< prev index next >

src/hotspot/share/opto/memnode.cpp

Print this page

   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"

  26 #include "classfile/javaClasses.hpp"

  27 #include "compiler/compileLog.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/c2/barrierSetC2.hpp"
  30 #include "gc/shared/tlab_globals.hpp"
  31 #include "memory/allocation.inline.hpp"
  32 #include "memory/resourceArea.hpp"
  33 #include "oops/objArrayKlass.hpp"
  34 #include "opto/addnode.hpp"
  35 #include "opto/arraycopynode.hpp"
  36 #include "opto/cfgnode.hpp"
  37 #include "opto/regalloc.hpp"
  38 #include "opto/compile.hpp"
  39 #include "opto/connode.hpp"
  40 #include "opto/convertnode.hpp"

  41 #include "opto/loopnode.hpp"
  42 #include "opto/machnode.hpp"
  43 #include "opto/matcher.hpp"
  44 #include "opto/memnode.hpp"
  45 #include "opto/mulnode.hpp"
  46 #include "opto/narrowptrnode.hpp"
  47 #include "opto/phaseX.hpp"
  48 #include "opto/regmask.hpp"
  49 #include "opto/rootnode.hpp"
  50 #include "opto/vectornode.hpp"
  51 #include "utilities/align.hpp"
  52 #include "utilities/copy.hpp"
  53 #include "utilities/macros.hpp"
  54 #include "utilities/powerOfTwo.hpp"
  55 #include "utilities/vmError.hpp"
  56 
  57 // Portions of code courtesy of Clifford Click
  58 
  59 // Optimization - Graph Style
  60 

 238       // clone the Phi with our address type
 239       result = mphi->split_out_instance(t_adr, igvn);
 240     } else {
 241       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 242     }
 243   }
 244   return result;
 245 }
 246 
 247 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 248   uint alias_idx = phase->C->get_alias_index(tp);
 249   Node *mem = mmem;
 250 #ifdef ASSERT
 251   {
 252     // Check that current type is consistent with the alias index used during graph construction
 253     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 254     bool consistent =  adr_check == NULL || adr_check->empty() ||
 255                        phase->C->must_alias(adr_check, alias_idx );
 256     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 257     if( !consistent && adr_check != NULL && !adr_check->empty() &&
 258                tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 259         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 260         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 261           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 262           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 263       // don't assert if it is dead code.
 264       consistent = true;
 265     }
 266     if( !consistent ) {
 267       st->print("alias_idx==%d, adr_check==", alias_idx);
 268       if( adr_check == NULL ) {
 269         st->print("NULL");
 270       } else {
 271         adr_check->dump();
 272       }
 273       st->cr();
 274       print_alias_types();
 275       assert(consistent, "adr_check must match alias idx");
 276     }
 277   }
 278 #endif

 906          "use LoadKlassNode instead");
 907   assert(!(adr_type->isa_aryptr() &&
 908            adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
 909          "use LoadRangeNode instead");
 910   // Check control edge of raw loads
 911   assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
 912           // oop will be recorded in oop map if load crosses safepoint
 913           rt->isa_oopptr() || is_immutable_value(adr),
 914           "raw memory operations should have control edge");
 915   LoadNode* load = NULL;
 916   switch (bt) {
 917   case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 918   case T_BYTE:    load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 919   case T_INT:     load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 920   case T_CHAR:    load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 921   case T_SHORT:   load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 922   case T_LONG:    load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
 923   case T_FLOAT:   load = new LoadFNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
 924   case T_DOUBLE:  load = new LoadDNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency, require_atomic_access); break;
 925   case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(),  mo, control_dependency); break;

 926   case T_OBJECT:
 927 #ifdef _LP64
 928     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 929       load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
 930     } else
 931 #endif
 932     {
 933       assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
 934       load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
 935     }
 936     break;
 937   default:
 938     ShouldNotReachHere();
 939     break;
 940   }
 941   assert(load != NULL, "LoadNode should have been created");
 942   if (unaligned) {
 943     load->set_unaligned_access();
 944   }
 945   if (mismatched) {

1001       assert(ld_alloc != NULL, "need an alloc");
1002       assert(addp->is_AddP(), "address must be addp");
1003       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1004       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1005       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1006       addp->set_req(AddPNode::Base, src);
1007       addp->set_req(AddPNode::Address, src);
1008     } else {
1009       assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1010              ac->as_ArrayCopy()->is_copyof_validated() ||
1011              ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1012       assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1013       addp->set_req(AddPNode::Base, src);
1014       addp->set_req(AddPNode::Address, src);
1015 
1016       const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1017       BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1018       if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1019 
1020       uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1021       uint shift  = exact_log2(type2aelembytes(ary_elem));
1022 
1023       Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1024 #ifdef _LP64
1025       diff = phase->transform(new ConvI2LNode(diff));
1026 #endif
1027       diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1028 
1029       Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1030       addp->set_req(AddPNode::Offset, offset);
1031     }
1032     addp = phase->transform(addp);
1033 #ifdef ASSERT
1034     const TypePtr* adr_type = phase->type(addp)->is_ptr();
1035     ld->_adr_type = adr_type;
1036 #endif
1037     ld->set_req(MemNode::Address, addp);
1038     ld->set_req(0, ctl);
1039     ld->set_req(MemNode::Memory, mem);
1040     // load depends on the tests that validate the arraycopy
1041     ld->_control_dependency = UnknownControl;

1122         // Same base, same offset.
1123         // Possible improvement for arrays: check index value instead of absolute offset.
1124 
1125         // At this point we have proven something like this setup:
1126         //   B = << base >>
1127         //   L =  LoadQ(AddP(Check/CastPP(B), #Off))
1128         //   S = StoreQ(AddP(             B , #Off), V)
1129         // (Actually, we haven't yet proven the Q's are the same.)
1130         // In other words, we are loading from a casted version of
1131         // the same pointer-and-offset that we stored to.
1132         // Casted version may carry a dependency and it is respected.
1133         // Thus, we are able to replace L by V.
1134       }
1135       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1136       if (store_Opcode() != st->Opcode()) {
1137         return NULL;
1138       }
1139       // LoadVector/StoreVector needs additional check to ensure the types match.
1140       if (st->is_StoreVector()) {
1141         const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
1142         const TypeVect* out_vt = as_LoadVector()->vect_type();
1143         if (in_vt != out_vt) {
1144           return NULL;
1145         }
1146       }
1147       return st->in(MemNode::ValueIn);
1148     }
1149 
1150     // A load from a freshly-created object always returns zero.
1151     // (This can happen after LoadNode::Ideal resets the load's memory input
1152     // to find_captured_store, which returned InitializeNode::zero_memory.)
1153     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1154         (st->in(0) == ld_alloc) &&
1155         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1156       // return a zero value for the load's basic type
1157       // (This is one of the few places where a generic PhaseTransform
1158       // can create new nodes.  Think of it as lazily manifesting
1159       // virtually pre-existing constants.)






1160       if (memory_type() != T_VOID) {
1161         if (ReduceBulkZeroing || find_array_copy_clone(phase, ld_alloc, in(MemNode::Memory)) == NULL) {
1162           // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1163           // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1164           // by the ArrayCopyNode.
1165           return phase->zerocon(memory_type());
1166         }
1167       } else {
1168         // TODO: materialize all-zero vector constant
1169         assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1170       }
1171     }
1172 
1173     // A load from an initialization barrier can match a captured store.
1174     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1175       InitializeNode* init = st->in(0)->as_Initialize();
1176       AllocateNode* alloc = init->allocation();
1177       if ((alloc != NULL) && (alloc == ld_alloc)) {
1178         // examine a captured store value
1179         st = init->find_captured_store(ld_off, memory_size(), phase);

1207 //----------------------is_instance_field_load_with_local_phi------------------
1208 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1209   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1210       in(Address)->is_AddP() ) {
1211     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1212     // Only instances and boxed values.
1213     if( t_oop != NULL &&
1214         (t_oop->is_ptr_to_boxed_value() ||
1215          t_oop->is_known_instance_field()) &&
1216         t_oop->offset() != Type::OffsetBot &&
1217         t_oop->offset() != Type::OffsetTop) {
1218       return true;
1219     }
1220   }
1221   return false;
1222 }
1223 
1224 //------------------------------Identity---------------------------------------
1225 // Loads are identity if previous store is to same address
1226 Node* LoadNode::Identity(PhaseGVN* phase) {

















1227   // If the previous store-maker is the right kind of Store, and the store is
1228   // to the same address, then we are equal to the value stored.
1229   Node* mem = in(Memory);
1230   Node* value = can_see_stored_value(mem, phase);
1231   if( value ) {
1232     // byte, short & char stores truncate naturally.
1233     // A load has to load the truncated value which requires
1234     // some sort of masking operation and that requires an
1235     // Ideal call instead of an Identity call.
1236     if (memory_size() < BytesPerInt) {
1237       // If the input to the store does not fit with the load's result type,
1238       // it must be truncated via an Ideal call.
1239       if (!phase->type(value)->higher_equal(phase->type(this)))
1240         return this;
1241     }
1242     // (This works even when value is a Con, but LoadNode::Value
1243     // usually runs first, producing the singleton type of the Con.)
1244     if (!has_pinned_control_dependency() || value->is_Con()) {
1245       return value;
1246     } else {

1931       }
1932     }
1933 
1934     // Don't do this for integer types. There is only potential profit if
1935     // the element type t is lower than _type; that is, for int types, if _type is
1936     // more restrictive than t.  This only happens here if one is short and the other
1937     // char (both 16 bits), and in those cases we've made an intentional decision
1938     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1939     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1940     //
1941     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1942     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
1943     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1944     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
1945     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1946     // In fact, that could have been the original type of p1, and p1 could have
1947     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1948     // expression (LShiftL quux 3) independently optimized to the constant 8.
1949     if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1950         && (_type->isa_vect() == NULL)

1951         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1952       // t might actually be lower than _type, if _type is a unique
1953       // concrete subclass of abstract class t.
1954       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
1955         const Type* jt = t->join_speculative(_type);
1956         // In any case, do not allow the join, per se, to empty out the type.
1957         if (jt->empty() && !t->empty()) {
1958           // This can happen if a interface-typed array narrows to a class type.
1959           jt = _type;
1960         }
1961 #ifdef ASSERT
1962         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1963           // The pointers in the autobox arrays are always non-null
1964           Node* base = adr->in(AddPNode::Base);
1965           if ((base != NULL) && base->is_DecodeN()) {
1966             // Get LoadN node which loads IntegerCache.cache field
1967             base = base->in(1);
1968           }
1969           if ((base != NULL) && base->is_Con()) {
1970             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1971             if ((base_type != NULL) && base_type->is_autobox_cache()) {
1972               // It could be narrow oop
1973               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1974             }
1975           }
1976         }
1977 #endif
1978         return jt;
1979       }
1980     }
1981   } else if (tp->base() == Type::InstPtr) {
1982     assert( off != Type::OffsetBot ||
1983             // arrays can be cast to Objects
1984             !tp->isa_instptr() ||
1985             tp->is_instptr()->instance_klass()->is_java_lang_Object() ||


1986             // unsafe field access may not have a constant offset
1987             C->has_unsafe_access(),
1988             "Field accesses must be precise" );
1989     // For oop loads, we expect the _type to be precise.
1990 
1991     // Optimize loads from constant fields.
1992     const TypeInstPtr* tinst = tp->is_instptr();



1993     ciObject* const_oop = tinst->const_oop();
1994     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1995       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());


















1996       if (con_type != NULL) {
1997         return con_type;
1998       }
1999     }
2000   } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2001     assert(off != Type::OffsetBot ||
2002             !tp->isa_instklassptr() ||
2003            // arrays can be cast to Objects
2004            tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2005            // also allow array-loading from the primary supertype
2006            // array during subtype checks
2007            Opcode() == Op_LoadKlass,
2008            "Field accesses must be precise");
2009     // For klass/static loads, we expect the _type to be precise
2010   } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2011     /* With mirrors being an indirect in the Klass*
2012      * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2013      * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2014      *
2015      * So check the type and klass of the node before the LoadP.
2016      */
2017     Node* adr2 = adr->in(MemNode::Address);
2018     const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2019     if (tkls != NULL && !StressReflectiveCode) {
2020       if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2021         ciKlass* klass = tkls->exact_klass();
2022         assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2023         assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2024         return TypeInstPtr::make(klass->java_mirror());
















2025       }
2026     }
2027   }
2028 
2029   const TypeKlassPtr *tkls = tp->isa_klassptr();
2030   if (tkls != NULL) {
2031     if (tkls->is_loaded() && tkls->klass_is_exact()) {
2032       ciKlass* klass = tkls->exact_klass();
2033       // We are loading a field from a Klass metaobject whose identity
2034       // is known at compile time (the type is "exact" or "precise").
2035       // Check for fields we know are maintained as constants by the VM.
2036       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2037         // The field is Klass::_super_check_offset.  Return its (constant) value.
2038         // (Folds up type checking code.)
2039         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2040         return TypeInt::make(klass->super_check_offset());
2041       }
2042       // Compute index into primary_supers array
2043       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2044       // Check for overflowing; use unsigned compare to handle the negative case.

2110   if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2111     Node* value = can_see_stored_value(mem,phase);
2112     if (value != NULL && value->is_Con()) {
2113       assert(value->bottom_type()->higher_equal(_type),"sanity");
2114       return value->bottom_type();
2115     }
2116   }
2117 
2118   bool is_vect = (_type->isa_vect() != NULL);
2119   if (is_instance && !is_vect) {
2120     // If we have an instance type and our memory input is the
2121     // programs's initial memory state, there is no matching store,
2122     // so just return a zero of the appropriate type -
2123     // except if it is vectorized - then we have no zero constant.
2124     Node *mem = in(MemNode::Memory);
2125     if (mem->is_Parm() && mem->in(0)->is_Start()) {
2126       assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2127       return Type::get_zero_type(_type->basic_type());
2128     }
2129   }
2130 
2131   Node* alloc = is_new_object_mark_load(phase);
2132   if (alloc != NULL) {
2133     return TypeX::make(markWord::prototype().value());









2134   }
2135 
2136   return _type;
2137 }
2138 
2139 //------------------------------match_edge-------------------------------------
2140 // Do we Match on this edge index or not?  Match only the address.
2141 uint LoadNode::match_edge(uint idx) const {
2142   return idx == MemNode::Address;
2143 }
2144 
2145 //--------------------------LoadBNode::Ideal--------------------------------------
2146 //
2147 //  If the previous store is to the same address as this load,
2148 //  and the value stored was larger than a byte, replace this load
2149 //  with the value stored truncated to a byte.  If no truncation is
2150 //  needed, the replacement is done in LoadNode::Identity().
2151 //
2152 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2153   Node* mem = in(MemNode::Memory);

2264   return LoadNode::Ideal(phase, can_reshape);
2265 }
2266 
2267 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2268   Node* mem = in(MemNode::Memory);
2269   Node* value = can_see_stored_value(mem,phase);
2270   if (value != NULL && value->is_Con() &&
2271       !value->bottom_type()->higher_equal(_type)) {
2272     // If the input to the store does not fit with the load's result type,
2273     // it must be truncated. We can't delay until Ideal call since
2274     // a singleton Value is needed for split_thru_phi optimization.
2275     int con = value->get_int();
2276     return TypeInt::make((con << 16) >> 16);
2277   }
2278   return LoadNode::Value(phase);
2279 }
2280 
2281 //=============================================================================
2282 //----------------------------LoadKlassNode::make------------------------------
2283 // Polymorphic factory method:
2284 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {

2285   // sanity check the alias category against the created node type
2286   const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2287   assert(adr_type != NULL, "expecting TypeKlassPtr");
2288 #ifdef _LP64
2289   if (adr_type->is_ptr_to_narrowklass()) {
2290     assert(UseCompressedClassPointers, "no compressed klasses");
2291     Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2292     return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2293   }
2294 #endif
2295   assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2296   return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2297 }
2298 
2299 //------------------------------Value------------------------------------------
2300 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2301   return klass_value_common(phase);
2302 }
2303 
2304 // In most cases, LoadKlassNode does not have the control input set. If the control

2311   // Either input is TOP ==> the result is TOP
2312   const Type *t1 = phase->type( in(MemNode::Memory) );
2313   if (t1 == Type::TOP)  return Type::TOP;
2314   Node *adr = in(MemNode::Address);
2315   const Type *t2 = phase->type( adr );
2316   if (t2 == Type::TOP)  return Type::TOP;
2317   const TypePtr *tp = t2->is_ptr();
2318   if (TypePtr::above_centerline(tp->ptr()) ||
2319       tp->ptr() == TypePtr::Null)  return Type::TOP;
2320 
2321   // Return a more precise klass, if possible
2322   const TypeInstPtr *tinst = tp->isa_instptr();
2323   if (tinst != NULL) {
2324     ciInstanceKlass* ik = tinst->instance_klass();
2325     int offset = tinst->offset();
2326     if (ik == phase->C->env()->Class_klass()
2327         && (offset == java_lang_Class::klass_offset() ||
2328             offset == java_lang_Class::array_klass_offset())) {
2329       // We are loading a special hidden field from a Class mirror object,
2330       // the field which points to the VM's Klass metaobject.
2331       ciType* t = tinst->java_mirror_type();

2332       // java_mirror_type returns non-null for compile-time Class constants.
2333       if (t != NULL) {
2334         // constant oop => constant klass
2335         if (offset == java_lang_Class::array_klass_offset()) {
2336           if (t->is_void()) {
2337             // We cannot create a void array.  Since void is a primitive type return null
2338             // klass.  Users of this result need to do a null check on the returned klass.
2339             return TypePtr::NULL_PTR;
2340           }
2341           return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2342         }
2343         if (!t->is_klass()) {
2344           // a primitive Class (e.g., int.class) has NULL for a klass field
2345           return TypePtr::NULL_PTR;
2346         }
2347         // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2348         return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2349       }
2350       // non-constant mirror, so we can't tell what's going on
2351     }
2352     if (!tinst->is_loaded())
2353       return _type;             // Bail out if not loaded
2354     if (offset == oopDesc::klass_offset_in_bytes()) {
2355       return tinst->as_klass_type(true);
2356     }
2357   }
2358 
2359   // Check for loading klass from an array
2360   const TypeAryPtr *tary = tp->isa_aryptr();
2361   if (tary != NULL && tary->elem() != Type::BOTTOM &&
2362       tary->offset() == oopDesc::klass_offset_in_bytes()) {
2363     return tary->as_klass_type(true);
2364   }
2365 
2366   // Check for loading klass from an array klass
2367   const TypeKlassPtr *tkls = tp->isa_klassptr();
2368   if (tkls != NULL && !StressReflectiveCode) {
2369     if (!tkls->is_loaded())
2370      return _type;             // Bail out if not loaded
2371     if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2372         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2373       // // Always returning precise element type is incorrect,
2374       // // e.g., element type could be object and array may contain strings
2375       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2376 
2377       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2378       // according to the element type's subclassing.
2379       return tkls->is_aryklassptr()->elem();
2380     }

2566 //=============================================================================
2567 //---------------------------StoreNode::make-----------------------------------
2568 // Polymorphic factory method:
2569 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2570   assert((mo == unordered || mo == release), "unexpected");
2571   Compile* C = gvn.C;
2572   assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2573          ctl != NULL, "raw memory operations should have control edge");
2574 
2575   switch (bt) {
2576   case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2577   case T_BYTE:    return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2578   case T_INT:     return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2579   case T_CHAR:
2580   case T_SHORT:   return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2581   case T_LONG:    return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2582   case T_FLOAT:   return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2583   case T_DOUBLE:  return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2584   case T_METADATA:
2585   case T_ADDRESS:

2586   case T_OBJECT:
2587 #ifdef _LP64
2588     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2589       val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2590       return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2591     } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2592                (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2593                 adr->bottom_type()->isa_rawptr())) {
2594       val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2595       return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2596     }
2597 #endif
2598     {
2599       return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2600     }
2601   default:
2602     ShouldNotReachHere();
2603     return (StoreNode*)NULL;
2604   }
2605 }

2616 
2617   // Since they are not commoned, do not hash them:
2618   return NO_HASH;
2619 }
2620 
2621 //------------------------------Ideal------------------------------------------
2622 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2623 // When a store immediately follows a relevant allocation/initialization,
2624 // try to capture it into the initialization, or hoist it above.
2625 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2626   Node* p = MemNode::Ideal_common(phase, can_reshape);
2627   if (p)  return (p == NodeSentinel) ? NULL : p;
2628 
2629   Node* mem     = in(MemNode::Memory);
2630   Node* address = in(MemNode::Address);
2631   Node* value   = in(MemNode::ValueIn);
2632   // Back-to-back stores to same address?  Fold em up.  Generally
2633   // unsafe if I have intervening uses...  Also disallowed for StoreCM
2634   // since they must follow each StoreP operation.  Redundant StoreCMs
2635   // are eliminated just before matching in final_graph_reshape.
2636   {
2637     Node* st = mem;
2638     // If Store 'st' has more than one use, we cannot fold 'st' away.
2639     // For example, 'st' might be the final state at a conditional
2640     // return.  Or, 'st' might be used by some node which is live at
2641     // the same time 'st' is live, which might be unschedulable.  So,
2642     // require exactly ONE user until such time as we clone 'mem' for
2643     // each of 'mem's uses (thus making the exactly-1-user-rule hold
2644     // true).
2645     while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2646       // Looking at a dead closed cycle of memory?
2647       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2648       assert(Opcode() == st->Opcode() ||
2649              st->Opcode() == Op_StoreVector ||
2650              Opcode() == Op_StoreVector ||
2651              st->Opcode() == Op_StoreVectorScatter ||
2652              Opcode() == Op_StoreVectorScatter ||
2653              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2654              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2655              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy

2656              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2657              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2658 
2659       if (st->in(MemNode::Address)->eqv_uncast(address) &&
2660           st->as_Store()->memory_size() <= this->memory_size()) {
2661         Node* use = st->raw_out(0);
2662         if (phase->is_IterGVN()) {
2663           phase->is_IterGVN()->rehash_node_delayed(use);
2664         }
2665         // It's OK to do this in the parser, since DU info is always accurate,
2666         // and the parser always refers to nodes via SafePointNode maps.
2667         use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2668         return this;
2669       }
2670       st = st->in(MemNode::Memory);
2671     }
2672   }
2673 
2674 
2675   // Capture an unaliased, unconditional, simple store into an initializer.

2732   // Load then Store?  Then the Store is useless
2733   if (val->is_Load() &&
2734       val->in(MemNode::Address)->eqv_uncast(adr) &&
2735       val->in(MemNode::Memory )->eqv_uncast(mem) &&
2736       val->as_Load()->store_Opcode() == Opcode()) {
2737     result = mem;
2738   }
2739 
2740   // Two stores in a row of the same value?
2741   if (result == this &&
2742       mem->is_Store() &&
2743       mem->in(MemNode::Address)->eqv_uncast(adr) &&
2744       mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2745       mem->Opcode() == Opcode()) {
2746     result = mem;
2747   }
2748 
2749   // Store of zero anywhere into a freshly-allocated object?
2750   // Then the store is useless.
2751   // (It must already have been captured by the InitializeNode.)
2752   if (result == this &&
2753       ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2754     // a newly allocated object is already all-zeroes everywhere
2755     if (mem->is_Proj() && mem->in(0)->is_Allocate()) {

2756       result = mem;
2757     }
2758 
2759     if (result == this) {
2760       // the store may also apply to zero-bits in an earlier object
2761       Node* prev_mem = find_previous_store(phase);
2762       // Steps (a), (b):  Walk past independent stores to find an exact match.
2763       if (prev_mem != NULL) {
2764         Node* prev_val = can_see_stored_value(prev_mem, phase);
2765         if (prev_val != NULL && prev_val == val) {
2766           // prev_val and val might differ by a cast; it would be good
2767           // to keep the more informative of the two.
2768           result = mem;
2769         }
2770       }
2771     }
2772   }
2773 
2774   PhaseIterGVN* igvn = phase->is_IterGVN();
2775   if (result != this && igvn != NULL) {
2776     MemBarNode* trailing = trailing_membar();
2777     if (trailing != NULL) {
2778 #ifdef ASSERT
2779       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();

2924 Node* StoreCMNode::Identity(PhaseGVN* phase) {
2925   // No need to card mark when storing a null ptr
2926   Node* my_store = in(MemNode::OopStore);
2927   if (my_store->is_Store()) {
2928     const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
2929     if( t1 == TypePtr::NULL_PTR ) {
2930       return in(MemNode::Memory);
2931     }
2932   }
2933   return this;
2934 }
2935 
2936 //=============================================================================
2937 //------------------------------Ideal---------------------------------------
2938 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
2939   Node* progress = StoreNode::Ideal(phase, can_reshape);
2940   if (progress != NULL) return progress;
2941 
2942   Node* my_store = in(MemNode::OopStore);
2943   if (my_store->is_MergeMem()) {
2944     Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
2945     set_req_X(MemNode::OopStore, mem, phase);
2946     return this;




2947   }
2948 
2949   return NULL;
2950 }
2951 
2952 //------------------------------Value-----------------------------------------
2953 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
2954   // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
2955   // If extra input is TOP ==> the result is TOP
2956   const Type* t = phase->type(in(MemNode::OopStore));
2957   if (t == Type::TOP) {
2958     return Type::TOP;
2959   }
2960   return StoreNode::Value(phase);
2961 }
2962 
2963 
2964 //=============================================================================
2965 //----------------------------------SCMemProjNode------------------------------
2966 const Type* SCMemProjNode::Value(PhaseGVN* phase) const

3085 // Clearing a short array is faster with stores
3086 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3087   // Already know this is a large node, do not try to ideal it
3088   if (_is_large) return NULL;
3089 
3090   const int unit = BytesPerLong;
3091   const TypeX* t = phase->type(in(2))->isa_intptr_t();
3092   if (!t)  return NULL;
3093   if (!t->is_con())  return NULL;
3094   intptr_t raw_count = t->get_con();
3095   intptr_t size = raw_count;
3096   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3097   // Clearing nothing uses the Identity call.
3098   // Negative clears are possible on dead ClearArrays
3099   // (see jck test stmt114.stmt11402.val).
3100   if (size <= 0 || size % unit != 0)  return NULL;
3101   intptr_t count = size / unit;
3102   // Length too long; communicate this to matchers and assemblers.
3103   // Assemblers are responsible to produce fast hardware clears for it.
3104   if (size > InitArrayShortSize) {
3105     return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3106   } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3107     return NULL;
3108   }
3109   if (!IdealizeClearArrayNode) return NULL;
3110   Node *mem = in(1);
3111   if( phase->type(mem)==Type::TOP ) return NULL;
3112   Node *adr = in(3);
3113   const Type* at = phase->type(adr);
3114   if( at==Type::TOP ) return NULL;
3115   const TypePtr* atp = at->isa_ptr();
3116   // adjust atp to be the correct array element address type
3117   if (atp == NULL)  atp = TypePtr::BOTTOM;
3118   else              atp = atp->add_offset(Type::OffsetBot);
3119   // Get base for derived pointer purposes
3120   if( adr->Opcode() != Op_AddP ) Unimplemented();
3121   Node *base = adr->in(1);
3122 
3123   Node *zero = phase->makecon(TypeLong::ZERO);
3124   Node *off  = phase->MakeConX(BytesPerLong);
3125   mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3126   count--;
3127   while( count-- ) {
3128     mem = phase->transform(mem);
3129     adr = phase->transform(new AddPNode(base,adr,off));
3130     mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3131   }
3132   return mem;
3133 }
3134 
3135 //----------------------------step_through----------------------------------
3136 // Return allocation input memory edge if it is different instance
3137 // or itself if it is the one we are looking for.
3138 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
3139   Node* n = *np;
3140   assert(n->is_ClearArray(), "sanity");
3141   intptr_t offset;
3142   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3143   // This method is called only before Allocate nodes are expanded
3144   // during macro nodes expansion. Before that ClearArray nodes are
3145   // only generated in PhaseMacroExpand::generate_arraycopy() (before
3146   // Allocate nodes are expanded) which follows allocations.
3147   assert(alloc != NULL, "should have allocation");
3148   if (alloc->_idx == instance_id) {
3149     // Can not bypass initialization of the instance we are looking for.
3150     return false;
3151   }
3152   // Otherwise skip it.
3153   InitializeNode* init = alloc->initialization();
3154   if (init != NULL)
3155     *np = init->in(TypeFunc::Memory);
3156   else
3157     *np = alloc->in(TypeFunc::Memory);
3158   return true;
3159 }
3160 
3161 //----------------------------clear_memory-------------------------------------
3162 // Generate code to initialize object storage to zero.
3163 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


3164                                    intptr_t start_offset,
3165                                    Node* end_offset,
3166                                    PhaseGVN* phase) {
3167   intptr_t offset = start_offset;
3168 
3169   int unit = BytesPerLong;
3170   if ((offset % unit) != 0) {
3171     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3172     adr = phase->transform(adr);
3173     const TypePtr* atp = TypeRawPtr::BOTTOM;
3174     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






3175     mem = phase->transform(mem);
3176     offset += BytesPerInt;
3177   }
3178   assert((offset % unit) == 0, "");
3179 
3180   // Initialize the remaining stuff, if any, with a ClearArray.
3181   return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
3182 }
3183 
3184 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,

3185                                    Node* start_offset,
3186                                    Node* end_offset,
3187                                    PhaseGVN* phase) {
3188   if (start_offset == end_offset) {
3189     // nothing to do
3190     return mem;
3191   }
3192 
3193   int unit = BytesPerLong;
3194   Node* zbase = start_offset;
3195   Node* zend  = end_offset;
3196 
3197   // Scale to the unit required by the CPU:
3198   if (!Matcher::init_array_count_is_in_bytes) {
3199     Node* shift = phase->intcon(exact_log2(unit));
3200     zbase = phase->transform(new URShiftXNode(zbase, shift) );
3201     zend  = phase->transform(new URShiftXNode(zend,  shift) );
3202   }
3203 
3204   // Bulk clear double-words
3205   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3206   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3207   mem = new ClearArrayNode(ctl, mem, zsize, adr, false);



3208   return phase->transform(mem);
3209 }
3210 
3211 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


3212                                    intptr_t start_offset,
3213                                    intptr_t end_offset,
3214                                    PhaseGVN* phase) {
3215   if (start_offset == end_offset) {
3216     // nothing to do
3217     return mem;
3218   }
3219 
3220   assert((end_offset % BytesPerInt) == 0, "odd end offset");
3221   intptr_t done_offset = end_offset;
3222   if ((done_offset % BytesPerLong) != 0) {
3223     done_offset -= BytesPerInt;
3224   }
3225   if (done_offset > start_offset) {
3226     mem = clear_memory(ctl, mem, dest,
3227                        start_offset, phase->MakeConX(done_offset), phase);
3228   }
3229   if (done_offset < end_offset) { // emit the final 32-bit store
3230     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3231     adr = phase->transform(adr);
3232     const TypePtr* atp = TypeRawPtr::BOTTOM;
3233     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






3234     mem = phase->transform(mem);
3235     done_offset += BytesPerInt;
3236   }
3237   assert(done_offset == end_offset, "");
3238   return mem;
3239 }
3240 
3241 //=============================================================================
3242 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3243   : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3244     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3245 #ifdef ASSERT
3246   , _pair_idx(0)
3247 #endif
3248 {
3249   init_class_id(Class_MemBar);
3250   Node* top = C->top();
3251   init_req(TypeFunc::I_O,top);
3252   init_req(TypeFunc::FramePtr,top);
3253   init_req(TypeFunc::ReturnAdr,top);

3358       PhaseIterGVN* igvn = phase->is_IterGVN();
3359       remove(igvn);
3360       // Must return either the original node (now dead) or a new node
3361       // (Do not return a top here, since that would break the uniqueness of top.)
3362       return new ConINode(TypeInt::ZERO);
3363     }
3364   }
3365   return progress ? this : NULL;
3366 }
3367 
3368 //------------------------------Value------------------------------------------
3369 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3370   if( !in(0) ) return Type::TOP;
3371   if( phase->type(in(0)) == Type::TOP )
3372     return Type::TOP;
3373   return TypeTuple::MEMBAR;
3374 }
3375 
3376 //------------------------------match------------------------------------------
3377 // Construct projections for memory.
3378 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3379   switch (proj->_con) {
3380   case TypeFunc::Control:
3381   case TypeFunc::Memory:
3382     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3383   }
3384   ShouldNotReachHere();
3385   return NULL;
3386 }
3387 
3388 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3389   trailing->_kind = TrailingStore;
3390   leading->_kind = LeadingStore;
3391 #ifdef ASSERT
3392   trailing->_pair_idx = leading->_idx;
3393   leading->_pair_idx = leading->_idx;
3394 #endif
3395 }
3396 
3397 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3398   trailing->_kind = TrailingLoadStore;

3645   return (req() > RawStores);
3646 }
3647 
3648 void InitializeNode::set_complete(PhaseGVN* phase) {
3649   assert(!is_complete(), "caller responsibility");
3650   _is_complete = Complete;
3651 
3652   // After this node is complete, it contains a bunch of
3653   // raw-memory initializations.  There is no need for
3654   // it to have anything to do with non-raw memory effects.
3655   // Therefore, tell all non-raw users to re-optimize themselves,
3656   // after skipping the memory effects of this initialization.
3657   PhaseIterGVN* igvn = phase->is_IterGVN();
3658   if (igvn)  igvn->add_users_to_worklist(this);
3659 }
3660 
3661 // convenience function
3662 // return false if the init contains any stores already
3663 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3664   InitializeNode* init = initialization();
3665   if (init == NULL || init->is_complete())  return false;


3666   init->remove_extra_zeroes();
3667   // for now, if this allocation has already collected any inits, bail:
3668   if (init->is_non_zero())  return false;
3669   init->set_complete(phase);
3670   return true;
3671 }
3672 
3673 void InitializeNode::remove_extra_zeroes() {
3674   if (req() == RawStores)  return;
3675   Node* zmem = zero_memory();
3676   uint fill = RawStores;
3677   for (uint i = fill; i < req(); i++) {
3678     Node* n = in(i);
3679     if (n->is_top() || n == zmem)  continue;  // skip
3680     if (fill < i)  set_req(fill, n);          // compact
3681     ++fill;
3682   }
3683   // delete any empty spaces created:
3684   while (fill < req()) {
3685     del_req(fill);

3823             // store node that we'd like to capture. We need to check
3824             // the uses of the MergeMemNode.
3825             mems.push(n);
3826           }
3827         } else if (n->is_Mem()) {
3828           Node* other_adr = n->in(MemNode::Address);
3829           if (other_adr == adr) {
3830             failed = true;
3831             break;
3832           } else {
3833             const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3834             if (other_t_adr != NULL) {
3835               int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3836               if (other_alias_idx == alias_idx) {
3837                 // A load from the same memory slice as the store right
3838                 // after the InitializeNode. We check the control of the
3839                 // object/array that is loaded from. If it's the same as
3840                 // the store control then we cannot capture the store.
3841                 assert(!n->is_Store(), "2 stores to same slice on same control?");
3842                 Node* base = other_adr;






3843                 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3844                 base = base->in(AddPNode::Base);
3845                 if (base != NULL) {
3846                   base = base->uncast();
3847                   if (base->is_Proj() && base->in(0) == alloc) {
3848                     failed = true;
3849                     break;
3850                   }
3851                 }
3852               }
3853             }
3854           }
3855         } else {
3856           failed = true;
3857           break;
3858         }
3859       }
3860     }
3861   }
3862   if (failed) {

4409         //   z's_done      12  16  16  16    12  16    12
4410         //   z's_needed    12  16  16  16    16  16    16
4411         //   zsize          0   0   0   0     4   0     4
4412         if (next_full_store < 0) {
4413           // Conservative tack:  Zero to end of current word.
4414           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4415         } else {
4416           // Zero to beginning of next fully initialized word.
4417           // Or, don't zero at all, if we are already in that word.
4418           assert(next_full_store >= zeroes_needed, "must go forward");
4419           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4420           zeroes_needed = next_full_store;
4421         }
4422       }
4423 
4424       if (zeroes_needed > zeroes_done) {
4425         intptr_t zsize = zeroes_needed - zeroes_done;
4426         // Do some incremental zeroing on rawmem, in parallel with inits.
4427         zeroes_done = align_down(zeroes_done, BytesPerInt);
4428         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


4429                                               zeroes_done, zeroes_needed,
4430                                               phase);
4431         zeroes_done = zeroes_needed;
4432         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4433           do_zeroing = false;   // leave the hole, next time
4434       }
4435     }
4436 
4437     // Collect the store and move on:
4438     phase->replace_input_of(st, MemNode::Memory, inits);
4439     inits = st;                 // put it on the linearized chain
4440     set_req(i, zmem);           // unhook from previous position
4441 
4442     if (zeroes_done == st_off)
4443       zeroes_done = next_init_off;
4444 
4445     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4446 
4447     #ifdef ASSERT
4448     // Various order invariants.  Weaker than stores_are_sane because

4468   remove_extra_zeroes();        // clear out all the zmems left over
4469   add_req(inits);
4470 
4471   if (!(UseTLAB && ZeroTLAB)) {
4472     // If anything remains to be zeroed, zero it all now.
4473     zeroes_done = align_down(zeroes_done, BytesPerInt);
4474     // if it is the last unused 4 bytes of an instance, forget about it
4475     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4476     if (zeroes_done + BytesPerLong >= size_limit) {
4477       AllocateNode* alloc = allocation();
4478       assert(alloc != NULL, "must be present");
4479       if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4480         Node* klass_node = alloc->in(AllocateNode::KlassNode);
4481         ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4482         if (zeroes_done == k->layout_helper())
4483           zeroes_done = size_limit;
4484       }
4485     }
4486     if (zeroes_done < size_limit) {
4487       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


4488                                             zeroes_done, size_in_bytes, phase);
4489     }
4490   }
4491 
4492   set_complete(phase);
4493   return rawmem;
4494 }
4495 
4496 
4497 #ifdef ASSERT
4498 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4499   if (is_complete())
4500     return true;                // stores could be anything at this point
4501   assert(allocation() != NULL, "must be present");
4502   intptr_t last_off = allocation()->minimum_header_size();
4503   for (uint i = InitializeNode::RawStores; i < req(); i++) {
4504     Node* st = in(i);
4505     intptr_t st_off = get_store_offset(st, phase);
4506     if (st_off < 0)  continue;  // ignore dead garbage
4507     if (last_off > st_off) {

   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciFlatArrayKlass.hpp"
  27 #include "classfile/javaClasses.hpp"
  28 #include "classfile/systemDictionary.hpp"
  29 #include "compiler/compileLog.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/c2/barrierSetC2.hpp"
  32 #include "gc/shared/tlab_globals.hpp"
  33 #include "memory/allocation.inline.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "oops/objArrayKlass.hpp"
  36 #include "opto/addnode.hpp"
  37 #include "opto/arraycopynode.hpp"
  38 #include "opto/cfgnode.hpp"
  39 #include "opto/regalloc.hpp"
  40 #include "opto/compile.hpp"
  41 #include "opto/connode.hpp"
  42 #include "opto/convertnode.hpp"
  43 #include "opto/inlinetypenode.hpp"
  44 #include "opto/loopnode.hpp"
  45 #include "opto/machnode.hpp"
  46 #include "opto/matcher.hpp"
  47 #include "opto/memnode.hpp"
  48 #include "opto/mulnode.hpp"
  49 #include "opto/narrowptrnode.hpp"
  50 #include "opto/phaseX.hpp"
  51 #include "opto/regmask.hpp"
  52 #include "opto/rootnode.hpp"
  53 #include "opto/vectornode.hpp"
  54 #include "utilities/align.hpp"
  55 #include "utilities/copy.hpp"
  56 #include "utilities/macros.hpp"
  57 #include "utilities/powerOfTwo.hpp"
  58 #include "utilities/vmError.hpp"
  59 
  60 // Portions of code courtesy of Clifford Click
  61 
  62 // Optimization - Graph Style
  63 

 241       // clone the Phi with our address type
 242       result = mphi->split_out_instance(t_adr, igvn);
 243     } else {
 244       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 245     }
 246   }
 247   return result;
 248 }
 249 
 250 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 251   uint alias_idx = phase->C->get_alias_index(tp);
 252   Node *mem = mmem;
 253 #ifdef ASSERT
 254   {
 255     // Check that current type is consistent with the alias index used during graph construction
 256     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 257     bool consistent =  adr_check == NULL || adr_check->empty() ||
 258                        phase->C->must_alias(adr_check, alias_idx );
 259     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 260     if( !consistent && adr_check != NULL && !adr_check->empty() &&
 261         tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 262         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 263         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 264           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 265           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 266       // don't assert if it is dead code.
 267       consistent = true;
 268     }
 269     if( !consistent ) {
 270       st->print("alias_idx==%d, adr_check==", alias_idx);
 271       if( adr_check == NULL ) {
 272         st->print("NULL");
 273       } else {
 274         adr_check->dump();
 275       }
 276       st->cr();
 277       print_alias_types();
 278       assert(consistent, "adr_check must match alias idx");
 279     }
 280   }
 281 #endif

 909          "use LoadKlassNode instead");
 910   assert(!(adr_type->isa_aryptr() &&
 911            adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
 912          "use LoadRangeNode instead");
 913   // Check control edge of raw loads
 914   assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
 915           // oop will be recorded in oop map if load crosses safepoint
 916           rt->isa_oopptr() || is_immutable_value(adr),
 917           "raw memory operations should have control edge");
 918   LoadNode* load = NULL;
 919   switch (bt) {
 920   case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 921   case T_BYTE:    load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 922   case T_INT:     load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 923   case T_CHAR:    load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 924   case T_SHORT:   load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 925   case T_LONG:    load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
 926   case T_FLOAT:   load = new LoadFNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
 927   case T_DOUBLE:  load = new LoadDNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency, require_atomic_access); break;
 928   case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(),  mo, control_dependency); break;
 929   case T_PRIMITIVE_OBJECT:
 930   case T_OBJECT:
 931 #ifdef _LP64
 932     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 933       load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
 934     } else
 935 #endif
 936     {
 937       assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
 938       load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
 939     }
 940     break;
 941   default:
 942     ShouldNotReachHere();
 943     break;
 944   }
 945   assert(load != NULL, "LoadNode should have been created");
 946   if (unaligned) {
 947     load->set_unaligned_access();
 948   }
 949   if (mismatched) {

1005       assert(ld_alloc != NULL, "need an alloc");
1006       assert(addp->is_AddP(), "address must be addp");
1007       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1008       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1009       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1010       addp->set_req(AddPNode::Base, src);
1011       addp->set_req(AddPNode::Address, src);
1012     } else {
1013       assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1014              ac->as_ArrayCopy()->is_copyof_validated() ||
1015              ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1016       assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1017       addp->set_req(AddPNode::Base, src);
1018       addp->set_req(AddPNode::Address, src);
1019 
1020       const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1021       BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1022       if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1023 
1024       uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1025       uint shift  = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1026 
1027       Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1028 #ifdef _LP64
1029       diff = phase->transform(new ConvI2LNode(diff));
1030 #endif
1031       diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1032 
1033       Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1034       addp->set_req(AddPNode::Offset, offset);
1035     }
1036     addp = phase->transform(addp);
1037 #ifdef ASSERT
1038     const TypePtr* adr_type = phase->type(addp)->is_ptr();
1039     ld->_adr_type = adr_type;
1040 #endif
1041     ld->set_req(MemNode::Address, addp);
1042     ld->set_req(0, ctl);
1043     ld->set_req(MemNode::Memory, mem);
1044     // load depends on the tests that validate the arraycopy
1045     ld->_control_dependency = UnknownControl;

1126         // Same base, same offset.
1127         // Possible improvement for arrays: check index value instead of absolute offset.
1128 
1129         // At this point we have proven something like this setup:
1130         //   B = << base >>
1131         //   L =  LoadQ(AddP(Check/CastPP(B), #Off))
1132         //   S = StoreQ(AddP(             B , #Off), V)
1133         // (Actually, we haven't yet proven the Q's are the same.)
1134         // In other words, we are loading from a casted version of
1135         // the same pointer-and-offset that we stored to.
1136         // Casted version may carry a dependency and it is respected.
1137         // Thus, we are able to replace L by V.
1138       }
1139       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1140       if (store_Opcode() != st->Opcode()) {
1141         return NULL;
1142       }
1143       // LoadVector/StoreVector needs additional check to ensure the types match.
1144       if (st->is_StoreVector()) {
1145         const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
1146         const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1147         if (in_vt != out_vt) {
1148           return NULL;
1149         }
1150       }
1151       return st->in(MemNode::ValueIn);
1152     }
1153 
1154     // A load from a freshly-created object always returns zero.
1155     // (This can happen after LoadNode::Ideal resets the load's memory input
1156     // to find_captured_store, which returned InitializeNode::zero_memory.)
1157     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1158         (st->in(0) == ld_alloc) &&
1159         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1160       // return a zero value for the load's basic type
1161       // (This is one of the few places where a generic PhaseTransform
1162       // can create new nodes.  Think of it as lazily manifesting
1163       // virtually pre-existing constants.)
1164       assert(memory_type() != T_PRIMITIVE_OBJECT, "should not be used for inline types");
1165       Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1166       if (default_value != NULL) {
1167         return default_value;
1168       }
1169       assert(ld_alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
1170       if (memory_type() != T_VOID) {
1171         if (ReduceBulkZeroing || find_array_copy_clone(phase, ld_alloc, in(MemNode::Memory)) == NULL) {
1172           // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1173           // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1174           // by the ArrayCopyNode.
1175           return phase->zerocon(memory_type());
1176         }
1177       } else {
1178         // TODO: materialize all-zero vector constant
1179         assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1180       }
1181     }
1182 
1183     // A load from an initialization barrier can match a captured store.
1184     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1185       InitializeNode* init = st->in(0)->as_Initialize();
1186       AllocateNode* alloc = init->allocation();
1187       if ((alloc != NULL) && (alloc == ld_alloc)) {
1188         // examine a captured store value
1189         st = init->find_captured_store(ld_off, memory_size(), phase);

1217 //----------------------is_instance_field_load_with_local_phi------------------
1218 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1219   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1220       in(Address)->is_AddP() ) {
1221     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1222     // Only instances and boxed values.
1223     if( t_oop != NULL &&
1224         (t_oop->is_ptr_to_boxed_value() ||
1225          t_oop->is_known_instance_field()) &&
1226         t_oop->offset() != Type::OffsetBot &&
1227         t_oop->offset() != Type::OffsetTop) {
1228       return true;
1229     }
1230   }
1231   return false;
1232 }
1233 
1234 //------------------------------Identity---------------------------------------
1235 // Loads are identity if previous store is to same address
1236 Node* LoadNode::Identity(PhaseGVN* phase) {
1237   // Loading from an InlineType? The InlineType has the values of
1238   // all fields as input. Look for the field with matching offset.
1239   Node* addr = in(Address);
1240   intptr_t offset;
1241   Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1242   if (base != NULL && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1243     Node* value = base->as_InlineType()->field_value_by_offset((int)offset, true);
1244     if (value != NULL) {
1245       if (Opcode() == Op_LoadN) {
1246         // Encode oop value if we are loading a narrow oop
1247         assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1248         value = phase->transform(new EncodePNode(value, bottom_type()));
1249       }
1250       return value;
1251     }
1252   }
1253 
1254   // If the previous store-maker is the right kind of Store, and the store is
1255   // to the same address, then we are equal to the value stored.
1256   Node* mem = in(Memory);
1257   Node* value = can_see_stored_value(mem, phase);
1258   if( value ) {
1259     // byte, short & char stores truncate naturally.
1260     // A load has to load the truncated value which requires
1261     // some sort of masking operation and that requires an
1262     // Ideal call instead of an Identity call.
1263     if (memory_size() < BytesPerInt) {
1264       // If the input to the store does not fit with the load's result type,
1265       // it must be truncated via an Ideal call.
1266       if (!phase->type(value)->higher_equal(phase->type(this)))
1267         return this;
1268     }
1269     // (This works even when value is a Con, but LoadNode::Value
1270     // usually runs first, producing the singleton type of the Con.)
1271     if (!has_pinned_control_dependency() || value->is_Con()) {
1272       return value;
1273     } else {

1958       }
1959     }
1960 
1961     // Don't do this for integer types. There is only potential profit if
1962     // the element type t is lower than _type; that is, for int types, if _type is
1963     // more restrictive than t.  This only happens here if one is short and the other
1964     // char (both 16 bits), and in those cases we've made an intentional decision
1965     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1966     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1967     //
1968     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1969     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
1970     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1971     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
1972     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1973     // In fact, that could have been the original type of p1, and p1 could have
1974     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1975     // expression (LShiftL quux 3) independently optimized to the constant 8.
1976     if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1977         && (_type->isa_vect() == NULL)
1978         && !ary->is_flat()
1979         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1980       // t might actually be lower than _type, if _type is a unique
1981       // concrete subclass of abstract class t.
1982       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
1983         const Type* jt = t->join_speculative(_type);
1984         // In any case, do not allow the join, per se, to empty out the type.
1985         if (jt->empty() && !t->empty()) {
1986           // This can happen if a interface-typed array narrows to a class type.
1987           jt = _type;
1988         }
1989 #ifdef ASSERT
1990         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1991           // The pointers in the autobox arrays are always non-null
1992           Node* base = adr->in(AddPNode::Base);
1993           if ((base != NULL) && base->is_DecodeN()) {
1994             // Get LoadN node which loads IntegerCache.cache field
1995             base = base->in(1);
1996           }
1997           if ((base != NULL) && base->is_Con()) {
1998             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1999             if ((base_type != NULL) && base_type->is_autobox_cache()) {
2000               // It could be narrow oop
2001               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2002             }
2003           }
2004         }
2005 #endif
2006         return jt;
2007       }
2008     }
2009   } else if (tp->base() == Type::InstPtr) {
2010     assert( off != Type::OffsetBot ||
2011             // arrays can be cast to Objects
2012             !tp->isa_instptr() ||
2013             tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2014             // Default value load
2015             tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2016             // unsafe field access may not have a constant offset
2017             C->has_unsafe_access(),
2018             "Field accesses must be precise" );
2019     // For oop loads, we expect the _type to be precise.
2020 

2021     const TypeInstPtr* tinst = tp->is_instptr();
2022     BasicType bt = memory_type();
2023 
2024     // Optimize loads from constant fields.
2025     ciObject* const_oop = tinst->const_oop();
2026     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
2027       ciType* mirror_type = const_oop->as_instance()->java_mirror_type();
2028       if (mirror_type != NULL) {
2029         const Type* const_oop = NULL;
2030         ciInlineKlass* vk = mirror_type->is_inlinetype() ? mirror_type->as_inline_klass() : NULL;
2031         // Fold default value loads
2032         if (vk != NULL && off == vk->default_value_offset()) {
2033           const_oop = TypeInstPtr::make(vk->default_instance());
2034         }
2035         // Fold class mirror loads
2036         if (off == java_lang_Class::primary_mirror_offset()) {
2037           const_oop = (vk == NULL) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->ref_instance());
2038         } else if (off == java_lang_Class::secondary_mirror_offset()) {
2039           const_oop = (vk == NULL) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->val_instance());
2040         }
2041         if (const_oop != NULL) {
2042           return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
2043         }
2044       }
2045       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2046       if (con_type != NULL) {
2047         return con_type;
2048       }
2049     }
2050   } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2051     assert(off != Type::OffsetBot ||
2052             !tp->isa_instklassptr() ||
2053            // arrays can be cast to Objects
2054            tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2055            // also allow array-loading from the primary supertype
2056            // array during subtype checks
2057            Opcode() == Op_LoadKlass,
2058            "Field accesses must be precise");
2059     // For klass/static loads, we expect the _type to be precise
2060   } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
2061     if (adr->is_Load() && off == 0) {
2062       /* With mirrors being an indirect in the Klass*
2063        * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2064        * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2065        *
2066        * So check the type and klass of the node before the LoadP.
2067        */
2068       Node* adr2 = adr->in(MemNode::Address);
2069       const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2070       if (tkls != NULL) {
2071         if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2072           ciKlass* klass = tkls->exact_klass();
2073           assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2074           assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2075           return TypeInstPtr::make(klass->java_mirror());
2076         }
2077       }
2078     } else {
2079       // Check for a load of the default value offset from the InlineKlassFixedBlock:
2080       // LoadI(LoadP(inline_klass, adr_inlineklass_fixed_block_offset), default_value_offset_offset)
2081       intptr_t offset = 0;
2082       Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2083       if (base != NULL && base->is_Load() && offset == in_bytes(InlineKlass::default_value_offset_offset())) {
2084         const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
2085         if (tkls != NULL && tkls->is_loaded() && tkls->klass_is_exact() && tkls->exact_klass()->is_inlinetype() &&
2086             tkls->offset() == in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())) {
2087           assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
2088           assert(Opcode() == Op_LoadI, "must load an int from fixed block");
2089           return TypeInt::make(tkls->exact_klass()->as_inline_klass()->default_value_offset());
2090         }
2091       }
2092     }
2093   }
2094 
2095   const TypeKlassPtr *tkls = tp->isa_klassptr();
2096   if (tkls != NULL) {
2097     if (tkls->is_loaded() && tkls->klass_is_exact()) {
2098       ciKlass* klass = tkls->exact_klass();
2099       // We are loading a field from a Klass metaobject whose identity
2100       // is known at compile time (the type is "exact" or "precise").
2101       // Check for fields we know are maintained as constants by the VM.
2102       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2103         // The field is Klass::_super_check_offset.  Return its (constant) value.
2104         // (Folds up type checking code.)
2105         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2106         return TypeInt::make(klass->super_check_offset());
2107       }
2108       // Compute index into primary_supers array
2109       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2110       // Check for overflowing; use unsigned compare to handle the negative case.

2176   if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2177     Node* value = can_see_stored_value(mem,phase);
2178     if (value != NULL && value->is_Con()) {
2179       assert(value->bottom_type()->higher_equal(_type),"sanity");
2180       return value->bottom_type();
2181     }
2182   }
2183 
2184   bool is_vect = (_type->isa_vect() != NULL);
2185   if (is_instance && !is_vect) {
2186     // If we have an instance type and our memory input is the
2187     // programs's initial memory state, there is no matching store,
2188     // so just return a zero of the appropriate type -
2189     // except if it is vectorized - then we have no zero constant.
2190     Node *mem = in(MemNode::Memory);
2191     if (mem->is_Parm() && mem->in(0)->is_Start()) {
2192       assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2193       return Type::get_zero_type(_type->basic_type());
2194     }
2195   }

2196   Node* alloc = is_new_object_mark_load(phase);
2197   if (alloc != NULL) {
2198     if (EnableValhalla) {
2199       // The mark word may contain property bits (inline, flat, null-free)
2200       Node* klass_node = alloc->in(AllocateNode::KlassNode);
2201       const TypeKlassPtr* tkls = phase->type(klass_node)->is_klassptr();
2202       if (tkls->is_loaded() && tkls->klass_is_exact()) {
2203         return TypeX::make(tkls->exact_klass()->prototype_header().value());
2204       }
2205     } else {
2206       return TypeX::make(markWord::prototype().value());
2207     }
2208   }
2209 
2210   return _type;
2211 }
2212 
2213 //------------------------------match_edge-------------------------------------
2214 // Do we Match on this edge index or not?  Match only the address.
2215 uint LoadNode::match_edge(uint idx) const {
2216   return idx == MemNode::Address;
2217 }
2218 
2219 //--------------------------LoadBNode::Ideal--------------------------------------
2220 //
2221 //  If the previous store is to the same address as this load,
2222 //  and the value stored was larger than a byte, replace this load
2223 //  with the value stored truncated to a byte.  If no truncation is
2224 //  needed, the replacement is done in LoadNode::Identity().
2225 //
2226 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2227   Node* mem = in(MemNode::Memory);

2338   return LoadNode::Ideal(phase, can_reshape);
2339 }
2340 
2341 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2342   Node* mem = in(MemNode::Memory);
2343   Node* value = can_see_stored_value(mem,phase);
2344   if (value != NULL && value->is_Con() &&
2345       !value->bottom_type()->higher_equal(_type)) {
2346     // If the input to the store does not fit with the load's result type,
2347     // it must be truncated. We can't delay until Ideal call since
2348     // a singleton Value is needed for split_thru_phi optimization.
2349     int con = value->get_int();
2350     return TypeInt::make((con << 16) >> 16);
2351   }
2352   return LoadNode::Value(phase);
2353 }
2354 
2355 //=============================================================================
2356 //----------------------------LoadKlassNode::make------------------------------
2357 // Polymorphic factory method:
2358 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2359                           const TypeKlassPtr* tk) {
2360   // sanity check the alias category against the created node type
2361   const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2362   assert(adr_type != NULL, "expecting TypeKlassPtr");
2363 #ifdef _LP64
2364   if (adr_type->is_ptr_to_narrowklass()) {
2365     assert(UseCompressedClassPointers, "no compressed klasses");
2366     Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2367     return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2368   }
2369 #endif
2370   assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2371   return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2372 }
2373 
2374 //------------------------------Value------------------------------------------
2375 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2376   return klass_value_common(phase);
2377 }
2378 
2379 // In most cases, LoadKlassNode does not have the control input set. If the control

2386   // Either input is TOP ==> the result is TOP
2387   const Type *t1 = phase->type( in(MemNode::Memory) );
2388   if (t1 == Type::TOP)  return Type::TOP;
2389   Node *adr = in(MemNode::Address);
2390   const Type *t2 = phase->type( adr );
2391   if (t2 == Type::TOP)  return Type::TOP;
2392   const TypePtr *tp = t2->is_ptr();
2393   if (TypePtr::above_centerline(tp->ptr()) ||
2394       tp->ptr() == TypePtr::Null)  return Type::TOP;
2395 
2396   // Return a more precise klass, if possible
2397   const TypeInstPtr *tinst = tp->isa_instptr();
2398   if (tinst != NULL) {
2399     ciInstanceKlass* ik = tinst->instance_klass();
2400     int offset = tinst->offset();
2401     if (ik == phase->C->env()->Class_klass()
2402         && (offset == java_lang_Class::klass_offset() ||
2403             offset == java_lang_Class::array_klass_offset())) {
2404       // We are loading a special hidden field from a Class mirror object,
2405       // the field which points to the VM's Klass metaobject.
2406       bool null_free = false;
2407       ciType* t = tinst->java_mirror_type(&null_free);
2408       // java_mirror_type returns non-null for compile-time Class constants.
2409       if (t != NULL) {
2410         // constant oop => constant klass
2411         if (offset == java_lang_Class::array_klass_offset()) {
2412           if (t->is_void()) {
2413             // We cannot create a void array.  Since void is a primitive type return null
2414             // klass.  Users of this result need to do a null check on the returned klass.
2415             return TypePtr::NULL_PTR;
2416           }
2417           return TypeKlassPtr::make(ciArrayKlass::make(t, null_free), Type::trust_interfaces);
2418         }
2419         if (!t->is_klass()) {
2420           // a primitive Class (e.g., int.class) has NULL for a klass field
2421           return TypePtr::NULL_PTR;
2422         }
2423         // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2424         return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2425       }
2426       // non-constant mirror, so we can't tell what's going on
2427     }
2428     if (!tinst->is_loaded())
2429       return _type;             // Bail out if not loaded
2430     if (offset == oopDesc::klass_offset_in_bytes()) {
2431       return tinst->as_klass_type(true);
2432     }
2433   }
2434 
2435   // Check for loading klass from an array
2436   const TypeAryPtr* tary = tp->isa_aryptr();
2437   if (tary != NULL && tary->elem() != Type::BOTTOM &&
2438       tary->offset() == oopDesc::klass_offset_in_bytes()) {
2439     return tary->as_klass_type(true);
2440   }
2441 
2442   // Check for loading klass from an array klass
2443   const TypeKlassPtr *tkls = tp->isa_klassptr();
2444   if (tkls != NULL && !StressReflectiveCode) {
2445     if (!tkls->is_loaded())
2446      return _type;             // Bail out if not loaded
2447     if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2448         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2449       // // Always returning precise element type is incorrect,
2450       // // e.g., element type could be object and array may contain strings
2451       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2452 
2453       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2454       // according to the element type's subclassing.
2455       return tkls->is_aryklassptr()->elem();
2456     }

2642 //=============================================================================
2643 //---------------------------StoreNode::make-----------------------------------
2644 // Polymorphic factory method:
2645 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2646   assert((mo == unordered || mo == release), "unexpected");
2647   Compile* C = gvn.C;
2648   assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2649          ctl != NULL, "raw memory operations should have control edge");
2650 
2651   switch (bt) {
2652   case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2653   case T_BYTE:    return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2654   case T_INT:     return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2655   case T_CHAR:
2656   case T_SHORT:   return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2657   case T_LONG:    return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2658   case T_FLOAT:   return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2659   case T_DOUBLE:  return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2660   case T_METADATA:
2661   case T_ADDRESS:
2662   case T_PRIMITIVE_OBJECT:
2663   case T_OBJECT:
2664 #ifdef _LP64
2665     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2666       val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2667       return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2668     } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2669                (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2670                 adr->bottom_type()->isa_rawptr())) {
2671       val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2672       return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2673     }
2674 #endif
2675     {
2676       return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2677     }
2678   default:
2679     ShouldNotReachHere();
2680     return (StoreNode*)NULL;
2681   }
2682 }

2693 
2694   // Since they are not commoned, do not hash them:
2695   return NO_HASH;
2696 }
2697 
2698 //------------------------------Ideal------------------------------------------
2699 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2700 // When a store immediately follows a relevant allocation/initialization,
2701 // try to capture it into the initialization, or hoist it above.
2702 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2703   Node* p = MemNode::Ideal_common(phase, can_reshape);
2704   if (p)  return (p == NodeSentinel) ? NULL : p;
2705 
2706   Node* mem     = in(MemNode::Memory);
2707   Node* address = in(MemNode::Address);
2708   Node* value   = in(MemNode::ValueIn);
2709   // Back-to-back stores to same address?  Fold em up.  Generally
2710   // unsafe if I have intervening uses...  Also disallowed for StoreCM
2711   // since they must follow each StoreP operation.  Redundant StoreCMs
2712   // are eliminated just before matching in final_graph_reshape.
2713   if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
2714     Node* st = mem;
2715     // If Store 'st' has more than one use, we cannot fold 'st' away.
2716     // For example, 'st' might be the final state at a conditional
2717     // return.  Or, 'st' might be used by some node which is live at
2718     // the same time 'st' is live, which might be unschedulable.  So,
2719     // require exactly ONE user until such time as we clone 'mem' for
2720     // each of 'mem's uses (thus making the exactly-1-user-rule hold
2721     // true).
2722     while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2723       // Looking at a dead closed cycle of memory?
2724       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2725       assert(Opcode() == st->Opcode() ||
2726              st->Opcode() == Op_StoreVector ||
2727              Opcode() == Op_StoreVector ||
2728              st->Opcode() == Op_StoreVectorScatter ||
2729              Opcode() == Op_StoreVectorScatter ||
2730              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2731              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2732              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2733              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
2734              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2735              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2736 
2737       if (st->in(MemNode::Address)->eqv_uncast(address) &&
2738           st->as_Store()->memory_size() <= this->memory_size()) {
2739         Node* use = st->raw_out(0);
2740         if (phase->is_IterGVN()) {
2741           phase->is_IterGVN()->rehash_node_delayed(use);
2742         }
2743         // It's OK to do this in the parser, since DU info is always accurate,
2744         // and the parser always refers to nodes via SafePointNode maps.
2745         use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2746         return this;
2747       }
2748       st = st->in(MemNode::Memory);
2749     }
2750   }
2751 
2752 
2753   // Capture an unaliased, unconditional, simple store into an initializer.

2810   // Load then Store?  Then the Store is useless
2811   if (val->is_Load() &&
2812       val->in(MemNode::Address)->eqv_uncast(adr) &&
2813       val->in(MemNode::Memory )->eqv_uncast(mem) &&
2814       val->as_Load()->store_Opcode() == Opcode()) {
2815     result = mem;
2816   }
2817 
2818   // Two stores in a row of the same value?
2819   if (result == this &&
2820       mem->is_Store() &&
2821       mem->in(MemNode::Address)->eqv_uncast(adr) &&
2822       mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2823       mem->Opcode() == Opcode()) {
2824     result = mem;
2825   }
2826 
2827   // Store of zero anywhere into a freshly-allocated object?
2828   // Then the store is useless.
2829   // (It must already have been captured by the InitializeNode.)
2830   if (result == this && ReduceFieldZeroing) {

2831     // a newly allocated object is already all-zeroes everywhere
2832     if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
2833         (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
2834       result = mem;
2835     }
2836 
2837     if (result == this && phase->type(val)->is_zero_type()) {
2838       // the store may also apply to zero-bits in an earlier object
2839       Node* prev_mem = find_previous_store(phase);
2840       // Steps (a), (b):  Walk past independent stores to find an exact match.
2841       if (prev_mem != NULL) {
2842         Node* prev_val = can_see_stored_value(prev_mem, phase);
2843         if (prev_val != NULL && prev_val == val) {
2844           // prev_val and val might differ by a cast; it would be good
2845           // to keep the more informative of the two.
2846           result = mem;
2847         }
2848       }
2849     }
2850   }
2851 
2852   PhaseIterGVN* igvn = phase->is_IterGVN();
2853   if (result != this && igvn != NULL) {
2854     MemBarNode* trailing = trailing_membar();
2855     if (trailing != NULL) {
2856 #ifdef ASSERT
2857       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();

3002 Node* StoreCMNode::Identity(PhaseGVN* phase) {
3003   // No need to card mark when storing a null ptr
3004   Node* my_store = in(MemNode::OopStore);
3005   if (my_store->is_Store()) {
3006     const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
3007     if( t1 == TypePtr::NULL_PTR ) {
3008       return in(MemNode::Memory);
3009     }
3010   }
3011   return this;
3012 }
3013 
3014 //=============================================================================
3015 //------------------------------Ideal---------------------------------------
3016 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
3017   Node* progress = StoreNode::Ideal(phase, can_reshape);
3018   if (progress != NULL) return progress;
3019 
3020   Node* my_store = in(MemNode::OopStore);
3021   if (my_store->is_MergeMem()) {
3022     if (oop_alias_idx() != phase->C->get_alias_index(TypeAryPtr::INLINES) ||
3023         phase->C->flattened_accesses_share_alias()) {
3024       // The alias that was recorded is no longer accurate enough.
3025       Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
3026       set_req_X(MemNode::OopStore, mem, phase);
3027       return this;
3028     }
3029   }
3030 
3031   return NULL;
3032 }
3033 
3034 //------------------------------Value-----------------------------------------
3035 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
3036   // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
3037   // If extra input is TOP ==> the result is TOP
3038   const Type* t = phase->type(in(MemNode::OopStore));
3039   if (t == Type::TOP) {
3040     return Type::TOP;
3041   }
3042   return StoreNode::Value(phase);
3043 }
3044 
3045 
3046 //=============================================================================
3047 //----------------------------------SCMemProjNode------------------------------
3048 const Type* SCMemProjNode::Value(PhaseGVN* phase) const

3167 // Clearing a short array is faster with stores
3168 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3169   // Already know this is a large node, do not try to ideal it
3170   if (_is_large) return NULL;
3171 
3172   const int unit = BytesPerLong;
3173   const TypeX* t = phase->type(in(2))->isa_intptr_t();
3174   if (!t)  return NULL;
3175   if (!t->is_con())  return NULL;
3176   intptr_t raw_count = t->get_con();
3177   intptr_t size = raw_count;
3178   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3179   // Clearing nothing uses the Identity call.
3180   // Negative clears are possible on dead ClearArrays
3181   // (see jck test stmt114.stmt11402.val).
3182   if (size <= 0 || size % unit != 0)  return NULL;
3183   intptr_t count = size / unit;
3184   // Length too long; communicate this to matchers and assemblers.
3185   // Assemblers are responsible to produce fast hardware clears for it.
3186   if (size > InitArrayShortSize) {
3187     return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3188   } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3189     return NULL;
3190   }
3191   if (!IdealizeClearArrayNode) return NULL;
3192   Node *mem = in(1);
3193   if( phase->type(mem)==Type::TOP ) return NULL;
3194   Node *adr = in(3);
3195   const Type* at = phase->type(adr);
3196   if( at==Type::TOP ) return NULL;
3197   const TypePtr* atp = at->isa_ptr();
3198   // adjust atp to be the correct array element address type
3199   if (atp == NULL)  atp = TypePtr::BOTTOM;
3200   else              atp = atp->add_offset(Type::OffsetBot);
3201   // Get base for derived pointer purposes
3202   if( adr->Opcode() != Op_AddP ) Unimplemented();
3203   Node *base = adr->in(1);
3204 
3205   Node *val = in(4);
3206   Node *off  = phase->MakeConX(BytesPerLong);
3207   mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3208   count--;
3209   while( count-- ) {
3210     mem = phase->transform(mem);
3211     adr = phase->transform(new AddPNode(base,adr,off));
3212     mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3213   }
3214   return mem;
3215 }
3216 
3217 //----------------------------step_through----------------------------------
3218 // Return allocation input memory edge if it is different instance
3219 // or itself if it is the one we are looking for.
3220 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
3221   Node* n = *np;
3222   assert(n->is_ClearArray(), "sanity");
3223   intptr_t offset;
3224   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3225   // This method is called only before Allocate nodes are expanded
3226   // during macro nodes expansion. Before that ClearArray nodes are
3227   // only generated in PhaseMacroExpand::generate_arraycopy() (before
3228   // Allocate nodes are expanded) which follows allocations.
3229   assert(alloc != NULL, "should have allocation");
3230   if (alloc->_idx == instance_id) {
3231     // Can not bypass initialization of the instance we are looking for.
3232     return false;
3233   }
3234   // Otherwise skip it.
3235   InitializeNode* init = alloc->initialization();
3236   if (init != NULL)
3237     *np = init->in(TypeFunc::Memory);
3238   else
3239     *np = alloc->in(TypeFunc::Memory);
3240   return true;
3241 }
3242 
3243 //----------------------------clear_memory-------------------------------------
3244 // Generate code to initialize object storage to zero.
3245 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3246                                    Node* val,
3247                                    Node* raw_val,
3248                                    intptr_t start_offset,
3249                                    Node* end_offset,
3250                                    PhaseGVN* phase) {
3251   intptr_t offset = start_offset;
3252 
3253   int unit = BytesPerLong;
3254   if ((offset % unit) != 0) {
3255     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3256     adr = phase->transform(adr);
3257     const TypePtr* atp = TypeRawPtr::BOTTOM;
3258     if (val != NULL) {
3259       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3260       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3261     } else {
3262       assert(raw_val == NULL, "val may not be null");
3263       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3264     }
3265     mem = phase->transform(mem);
3266     offset += BytesPerInt;
3267   }
3268   assert((offset % unit) == 0, "");
3269 
3270   // Initialize the remaining stuff, if any, with a ClearArray.
3271   return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3272 }
3273 
3274 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3275                                    Node* raw_val,
3276                                    Node* start_offset,
3277                                    Node* end_offset,
3278                                    PhaseGVN* phase) {
3279   if (start_offset == end_offset) {
3280     // nothing to do
3281     return mem;
3282   }
3283 
3284   int unit = BytesPerLong;
3285   Node* zbase = start_offset;
3286   Node* zend  = end_offset;
3287 
3288   // Scale to the unit required by the CPU:
3289   if (!Matcher::init_array_count_is_in_bytes) {
3290     Node* shift = phase->intcon(exact_log2(unit));
3291     zbase = phase->transform(new URShiftXNode(zbase, shift) );
3292     zend  = phase->transform(new URShiftXNode(zend,  shift) );
3293   }
3294 
3295   // Bulk clear double-words
3296   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3297   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3298   if (raw_val == NULL) {
3299     raw_val = phase->MakeConX(0);
3300   }
3301   mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3302   return phase->transform(mem);
3303 }
3304 
3305 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3306                                    Node* val,
3307                                    Node* raw_val,
3308                                    intptr_t start_offset,
3309                                    intptr_t end_offset,
3310                                    PhaseGVN* phase) {
3311   if (start_offset == end_offset) {
3312     // nothing to do
3313     return mem;
3314   }
3315 
3316   assert((end_offset % BytesPerInt) == 0, "odd end offset");
3317   intptr_t done_offset = end_offset;
3318   if ((done_offset % BytesPerLong) != 0) {
3319     done_offset -= BytesPerInt;
3320   }
3321   if (done_offset > start_offset) {
3322     mem = clear_memory(ctl, mem, dest, val, raw_val,
3323                        start_offset, phase->MakeConX(done_offset), phase);
3324   }
3325   if (done_offset < end_offset) { // emit the final 32-bit store
3326     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3327     adr = phase->transform(adr);
3328     const TypePtr* atp = TypeRawPtr::BOTTOM;
3329     if (val != NULL) {
3330       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3331       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3332     } else {
3333       assert(raw_val == NULL, "val may not be null");
3334       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3335     }
3336     mem = phase->transform(mem);
3337     done_offset += BytesPerInt;
3338   }
3339   assert(done_offset == end_offset, "");
3340   return mem;
3341 }
3342 
3343 //=============================================================================
3344 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3345   : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3346     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3347 #ifdef ASSERT
3348   , _pair_idx(0)
3349 #endif
3350 {
3351   init_class_id(Class_MemBar);
3352   Node* top = C->top();
3353   init_req(TypeFunc::I_O,top);
3354   init_req(TypeFunc::FramePtr,top);
3355   init_req(TypeFunc::ReturnAdr,top);

3460       PhaseIterGVN* igvn = phase->is_IterGVN();
3461       remove(igvn);
3462       // Must return either the original node (now dead) or a new node
3463       // (Do not return a top here, since that would break the uniqueness of top.)
3464       return new ConINode(TypeInt::ZERO);
3465     }
3466   }
3467   return progress ? this : NULL;
3468 }
3469 
3470 //------------------------------Value------------------------------------------
3471 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3472   if( !in(0) ) return Type::TOP;
3473   if( phase->type(in(0)) == Type::TOP )
3474     return Type::TOP;
3475   return TypeTuple::MEMBAR;
3476 }
3477 
3478 //------------------------------match------------------------------------------
3479 // Construct projections for memory.
3480 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
3481   switch (proj->_con) {
3482   case TypeFunc::Control:
3483   case TypeFunc::Memory:
3484     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3485   }
3486   ShouldNotReachHere();
3487   return NULL;
3488 }
3489 
3490 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3491   trailing->_kind = TrailingStore;
3492   leading->_kind = LeadingStore;
3493 #ifdef ASSERT
3494   trailing->_pair_idx = leading->_idx;
3495   leading->_pair_idx = leading->_idx;
3496 #endif
3497 }
3498 
3499 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3500   trailing->_kind = TrailingLoadStore;

3747   return (req() > RawStores);
3748 }
3749 
3750 void InitializeNode::set_complete(PhaseGVN* phase) {
3751   assert(!is_complete(), "caller responsibility");
3752   _is_complete = Complete;
3753 
3754   // After this node is complete, it contains a bunch of
3755   // raw-memory initializations.  There is no need for
3756   // it to have anything to do with non-raw memory effects.
3757   // Therefore, tell all non-raw users to re-optimize themselves,
3758   // after skipping the memory effects of this initialization.
3759   PhaseIterGVN* igvn = phase->is_IterGVN();
3760   if (igvn)  igvn->add_users_to_worklist(this);
3761 }
3762 
3763 // convenience function
3764 // return false if the init contains any stores already
3765 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3766   InitializeNode* init = initialization();
3767   if (init == NULL || init->is_complete()) {
3768     return false;
3769   }
3770   init->remove_extra_zeroes();
3771   // for now, if this allocation has already collected any inits, bail:
3772   if (init->is_non_zero())  return false;
3773   init->set_complete(phase);
3774   return true;
3775 }
3776 
3777 void InitializeNode::remove_extra_zeroes() {
3778   if (req() == RawStores)  return;
3779   Node* zmem = zero_memory();
3780   uint fill = RawStores;
3781   for (uint i = fill; i < req(); i++) {
3782     Node* n = in(i);
3783     if (n->is_top() || n == zmem)  continue;  // skip
3784     if (fill < i)  set_req(fill, n);          // compact
3785     ++fill;
3786   }
3787   // delete any empty spaces created:
3788   while (fill < req()) {
3789     del_req(fill);

3927             // store node that we'd like to capture. We need to check
3928             // the uses of the MergeMemNode.
3929             mems.push(n);
3930           }
3931         } else if (n->is_Mem()) {
3932           Node* other_adr = n->in(MemNode::Address);
3933           if (other_adr == adr) {
3934             failed = true;
3935             break;
3936           } else {
3937             const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3938             if (other_t_adr != NULL) {
3939               int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3940               if (other_alias_idx == alias_idx) {
3941                 // A load from the same memory slice as the store right
3942                 // after the InitializeNode. We check the control of the
3943                 // object/array that is loaded from. If it's the same as
3944                 // the store control then we cannot capture the store.
3945                 assert(!n->is_Store(), "2 stores to same slice on same control?");
3946                 Node* base = other_adr;
3947                 if (base->is_Phi()) {
3948                   // In rare case, base may be a PhiNode and it may read
3949                   // the same memory slice between InitializeNode and store.
3950                   failed = true;
3951                   break;
3952                 }
3953                 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3954                 base = base->in(AddPNode::Base);
3955                 if (base != NULL) {
3956                   base = base->uncast();
3957                   if (base->is_Proj() && base->in(0) == alloc) {
3958                     failed = true;
3959                     break;
3960                   }
3961                 }
3962               }
3963             }
3964           }
3965         } else {
3966           failed = true;
3967           break;
3968         }
3969       }
3970     }
3971   }
3972   if (failed) {

4519         //   z's_done      12  16  16  16    12  16    12
4520         //   z's_needed    12  16  16  16    16  16    16
4521         //   zsize          0   0   0   0     4   0     4
4522         if (next_full_store < 0) {
4523           // Conservative tack:  Zero to end of current word.
4524           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4525         } else {
4526           // Zero to beginning of next fully initialized word.
4527           // Or, don't zero at all, if we are already in that word.
4528           assert(next_full_store >= zeroes_needed, "must go forward");
4529           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4530           zeroes_needed = next_full_store;
4531         }
4532       }
4533 
4534       if (zeroes_needed > zeroes_done) {
4535         intptr_t zsize = zeroes_needed - zeroes_done;
4536         // Do some incremental zeroing on rawmem, in parallel with inits.
4537         zeroes_done = align_down(zeroes_done, BytesPerInt);
4538         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4539                                               allocation()->in(AllocateNode::DefaultValue),
4540                                               allocation()->in(AllocateNode::RawDefaultValue),
4541                                               zeroes_done, zeroes_needed,
4542                                               phase);
4543         zeroes_done = zeroes_needed;
4544         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4545           do_zeroing = false;   // leave the hole, next time
4546       }
4547     }
4548 
4549     // Collect the store and move on:
4550     phase->replace_input_of(st, MemNode::Memory, inits);
4551     inits = st;                 // put it on the linearized chain
4552     set_req(i, zmem);           // unhook from previous position
4553 
4554     if (zeroes_done == st_off)
4555       zeroes_done = next_init_off;
4556 
4557     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4558 
4559     #ifdef ASSERT
4560     // Various order invariants.  Weaker than stores_are_sane because

4580   remove_extra_zeroes();        // clear out all the zmems left over
4581   add_req(inits);
4582 
4583   if (!(UseTLAB && ZeroTLAB)) {
4584     // If anything remains to be zeroed, zero it all now.
4585     zeroes_done = align_down(zeroes_done, BytesPerInt);
4586     // if it is the last unused 4 bytes of an instance, forget about it
4587     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4588     if (zeroes_done + BytesPerLong >= size_limit) {
4589       AllocateNode* alloc = allocation();
4590       assert(alloc != NULL, "must be present");
4591       if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4592         Node* klass_node = alloc->in(AllocateNode::KlassNode);
4593         ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4594         if (zeroes_done == k->layout_helper())
4595           zeroes_done = size_limit;
4596       }
4597     }
4598     if (zeroes_done < size_limit) {
4599       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4600                                             allocation()->in(AllocateNode::DefaultValue),
4601                                             allocation()->in(AllocateNode::RawDefaultValue),
4602                                             zeroes_done, size_in_bytes, phase);
4603     }
4604   }
4605 
4606   set_complete(phase);
4607   return rawmem;
4608 }
4609 
4610 
4611 #ifdef ASSERT
4612 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4613   if (is_complete())
4614     return true;                // stores could be anything at this point
4615   assert(allocation() != NULL, "must be present");
4616   intptr_t last_off = allocation()->minimum_header_size();
4617   for (uint i = InitializeNode::RawStores; i < req(); i++) {
4618     Node* st = in(i);
4619     intptr_t st_off = get_store_offset(st, phase);
4620     if (st_off < 0)  continue;  // ignore dead garbage
4621     if (last_off > st_off) {
< prev index next >