< prev index next >

src/hotspot/share/opto/memnode.cpp

Print this page

   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"

  26 #include "classfile/javaClasses.hpp"

  27 #include "compiler/compileLog.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/c2/barrierSetC2.hpp"
  30 #include "gc/shared/tlab_globals.hpp"
  31 #include "memory/allocation.inline.hpp"
  32 #include "memory/resourceArea.hpp"
  33 #include "oops/objArrayKlass.hpp"
  34 #include "opto/addnode.hpp"
  35 #include "opto/arraycopynode.hpp"
  36 #include "opto/cfgnode.hpp"
  37 #include "opto/regalloc.hpp"
  38 #include "opto/compile.hpp"
  39 #include "opto/connode.hpp"
  40 #include "opto/convertnode.hpp"

  41 #include "opto/loopnode.hpp"
  42 #include "opto/machnode.hpp"
  43 #include "opto/matcher.hpp"
  44 #include "opto/memnode.hpp"
  45 #include "opto/mulnode.hpp"
  46 #include "opto/narrowptrnode.hpp"
  47 #include "opto/phaseX.hpp"
  48 #include "opto/regmask.hpp"
  49 #include "opto/rootnode.hpp"
  50 #include "opto/vectornode.hpp"
  51 #include "utilities/align.hpp"
  52 #include "utilities/copy.hpp"
  53 #include "utilities/macros.hpp"
  54 #include "utilities/powerOfTwo.hpp"
  55 #include "utilities/vmError.hpp"
  56 
  57 // Portions of code courtesy of Clifford Click
  58 
  59 // Optimization - Graph Style
  60 

 214   bool is_instance = t_oop->is_known_instance_field();
 215   PhaseIterGVN *igvn = phase->is_IterGVN();
 216   if (is_instance && igvn != nullptr && result->is_Phi()) {
 217     PhiNode *mphi = result->as_Phi();
 218     assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
 219     const TypePtr *t = mphi->adr_type();
 220     bool do_split = false;
 221     // In the following cases, Load memory input can be further optimized based on
 222     // its precise address type
 223     if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
 224       do_split = true;
 225     } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
 226       const TypeOopPtr* mem_t =
 227         t->is_oopptr()->cast_to_exactness(true)
 228         ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
 229         ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
 230       if (t_oop->is_aryptr()) {
 231         mem_t = mem_t->is_aryptr()
 232                      ->cast_to_stable(t_oop->is_aryptr()->is_stable())
 233                      ->cast_to_size(t_oop->is_aryptr()->size())


 234                      ->with_offset(t_oop->is_aryptr()->offset())
 235                      ->is_aryptr();
 236       }
 237       do_split = mem_t == t_oop;
 238     }
 239     if (do_split) {
 240       // clone the Phi with our address type
 241       result = mphi->split_out_instance(t_adr, igvn);
 242     } else {
 243       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 244     }
 245   }
 246   return result;
 247 }
 248 
 249 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 250   uint alias_idx = phase->C->get_alias_index(tp);
 251   Node *mem = mmem;
 252 #ifdef ASSERT
 253   {
 254     // Check that current type is consistent with the alias index used during graph construction
 255     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 256     bool consistent =  adr_check == nullptr || adr_check->empty() ||
 257                        phase->C->must_alias(adr_check, alias_idx );
 258     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 259     if( !consistent && adr_check != nullptr && !adr_check->empty() &&
 260                tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 261         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 262         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 263           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 264           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 265       // don't assert if it is dead code.
 266       consistent = true;
 267     }
 268     if( !consistent ) {
 269       st->print("alias_idx==%d, adr_check==", alias_idx);
 270       if( adr_check == nullptr ) {
 271         st->print("null");
 272       } else {
 273         adr_check->dump();
 274       }
 275       st->cr();
 276       print_alias_types();
 277       assert(consistent, "adr_check must match alias idx");
 278     }
 279   }
 280 #endif

1008       assert(ld_alloc != nullptr, "need an alloc");
1009       assert(addp->is_AddP(), "address must be addp");
1010       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1011       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1012       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1013       addp->set_req(AddPNode::Base, src);
1014       addp->set_req(AddPNode::Address, src);
1015     } else {
1016       assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1017              ac->as_ArrayCopy()->is_copyof_validated() ||
1018              ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1019       assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1020       addp->set_req(AddPNode::Base, src);
1021       addp->set_req(AddPNode::Address, src);
1022 
1023       const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1024       BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1025       if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1026 
1027       uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1028       uint shift  = exact_log2(type2aelembytes(ary_elem));
1029 
1030       Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1031 #ifdef _LP64
1032       diff = phase->transform(new ConvI2LNode(diff));
1033 #endif
1034       diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1035 
1036       Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1037       addp->set_req(AddPNode::Offset, offset);
1038     }
1039     addp = phase->transform(addp);
1040 #ifdef ASSERT
1041     const TypePtr* adr_type = phase->type(addp)->is_ptr();
1042     ld->_adr_type = adr_type;
1043 #endif
1044     ld->set_req(MemNode::Address, addp);
1045     ld->set_req(0, ctl);
1046     ld->set_req(MemNode::Memory, mem);
1047     // load depends on the tests that validate the arraycopy
1048     ld->_control_dependency = UnknownControl;

1129         // Same base, same offset.
1130         // Possible improvement for arrays: check index value instead of absolute offset.
1131 
1132         // At this point we have proven something like this setup:
1133         //   B = << base >>
1134         //   L =  LoadQ(AddP(Check/CastPP(B), #Off))
1135         //   S = StoreQ(AddP(             B , #Off), V)
1136         // (Actually, we haven't yet proven the Q's are the same.)
1137         // In other words, we are loading from a casted version of
1138         // the same pointer-and-offset that we stored to.
1139         // Casted version may carry a dependency and it is respected.
1140         // Thus, we are able to replace L by V.
1141       }
1142       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1143       if (store_Opcode() != st->Opcode()) {
1144         return nullptr;
1145       }
1146       // LoadVector/StoreVector needs additional check to ensure the types match.
1147       if (st->is_StoreVector()) {
1148         const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
1149         const TypeVect* out_vt = as_LoadVector()->vect_type();
1150         if (in_vt != out_vt) {
1151           return nullptr;
1152         }
1153       }
1154       return st->in(MemNode::ValueIn);
1155     }
1156 
1157     // A load from a freshly-created object always returns zero.
1158     // (This can happen after LoadNode::Ideal resets the load's memory input
1159     // to find_captured_store, which returned InitializeNode::zero_memory.)
1160     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1161         (st->in(0) == ld_alloc) &&
1162         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1163       // return a zero value for the load's basic type
1164       // (This is one of the few places where a generic PhaseTransform
1165       // can create new nodes.  Think of it as lazily manifesting
1166       // virtually pre-existing constants.)





1167       if (memory_type() != T_VOID) {
1168         if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1169           // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1170           // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1171           // by the ArrayCopyNode.
1172           return phase->zerocon(memory_type());
1173         }
1174       } else {
1175         // TODO: materialize all-zero vector constant
1176         assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1177       }
1178     }
1179 
1180     // A load from an initialization barrier can match a captured store.
1181     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1182       InitializeNode* init = st->in(0)->as_Initialize();
1183       AllocateNode* alloc = init->allocation();
1184       if ((alloc != nullptr) && (alloc == ld_alloc)) {
1185         // examine a captured store value
1186         st = init->find_captured_store(ld_off, memory_size(), phase);

1214 //----------------------is_instance_field_load_with_local_phi------------------
1215 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1216   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1217       in(Address)->is_AddP() ) {
1218     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1219     // Only instances and boxed values.
1220     if( t_oop != nullptr &&
1221         (t_oop->is_ptr_to_boxed_value() ||
1222          t_oop->is_known_instance_field()) &&
1223         t_oop->offset() != Type::OffsetBot &&
1224         t_oop->offset() != Type::OffsetTop) {
1225       return true;
1226     }
1227   }
1228   return false;
1229 }
1230 
1231 //------------------------------Identity---------------------------------------
1232 // Loads are identity if previous store is to same address
1233 Node* LoadNode::Identity(PhaseGVN* phase) {

















1234   // If the previous store-maker is the right kind of Store, and the store is
1235   // to the same address, then we are equal to the value stored.
1236   Node* mem = in(Memory);
1237   Node* value = can_see_stored_value(mem, phase);
1238   if( value ) {
1239     // byte, short & char stores truncate naturally.
1240     // A load has to load the truncated value which requires
1241     // some sort of masking operation and that requires an
1242     // Ideal call instead of an Identity call.
1243     if (memory_size() < BytesPerInt) {
1244       // If the input to the store does not fit with the load's result type,
1245       // it must be truncated via an Ideal call.
1246       if (!phase->type(value)->higher_equal(phase->type(this)))
1247         return this;
1248     }
1249     // (This works even when value is a Con, but LoadNode::Value
1250     // usually runs first, producing the singleton type of the Con.)
1251     if (!has_pinned_control_dependency() || value->is_Con()) {
1252       return value;
1253     } else {

1972       }
1973     }
1974 
1975     // Don't do this for integer types. There is only potential profit if
1976     // the element type t is lower than _type; that is, for int types, if _type is
1977     // more restrictive than t.  This only happens here if one is short and the other
1978     // char (both 16 bits), and in those cases we've made an intentional decision
1979     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1980     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1981     //
1982     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1983     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
1984     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1985     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
1986     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1987     // In fact, that could have been the original type of p1, and p1 could have
1988     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1989     // expression (LShiftL quux 3) independently optimized to the constant 8.
1990     if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
1991         && (_type->isa_vect() == nullptr)

1992         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1993       // t might actually be lower than _type, if _type is a unique
1994       // concrete subclass of abstract class t.
1995       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
1996         const Type* jt = t->join_speculative(_type);
1997         // In any case, do not allow the join, per se, to empty out the type.
1998         if (jt->empty() && !t->empty()) {
1999           // This can happen if a interface-typed array narrows to a class type.
2000           jt = _type;
2001         }
2002 #ifdef ASSERT
2003         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2004           // The pointers in the autobox arrays are always non-null
2005           Node* base = adr->in(AddPNode::Base);
2006           if ((base != nullptr) && base->is_DecodeN()) {
2007             // Get LoadN node which loads IntegerCache.cache field
2008             base = base->in(1);
2009           }
2010           if ((base != nullptr) && base->is_Con()) {
2011             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2012             if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2013               // It could be narrow oop
2014               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2015             }
2016           }
2017         }
2018 #endif
2019         return jt;
2020       }
2021     }
2022   } else if (tp->base() == Type::InstPtr) {
2023     assert( off != Type::OffsetBot ||
2024             // arrays can be cast to Objects
2025             !tp->isa_instptr() ||
2026             tp->is_instptr()->instance_klass()->is_java_lang_Object() ||


2027             // unsafe field access may not have a constant offset
2028             C->has_unsafe_access(),
2029             "Field accesses must be precise" );
2030     // For oop loads, we expect the _type to be precise.
2031 
2032     // Optimize loads from constant fields.
2033     const TypeInstPtr* tinst = tp->is_instptr();



2034     ciObject* const_oop = tinst->const_oop();
2035     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2036       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());


















2037       if (con_type != nullptr) {
2038         return con_type;
2039       }
2040     }
2041   } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2042     assert(off != Type::OffsetBot ||
2043             !tp->isa_instklassptr() ||
2044            // arrays can be cast to Objects
2045            tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2046            // also allow array-loading from the primary supertype
2047            // array during subtype checks
2048            Opcode() == Op_LoadKlass,
2049            "Field accesses must be precise");
2050     // For klass/static loads, we expect the _type to be precise
2051   } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2052     /* With mirrors being an indirect in the Klass*
2053      * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2054      * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2055      *
2056      * So check the type and klass of the node before the LoadP.
2057      */
2058     Node* adr2 = adr->in(MemNode::Address);
2059     const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2060     if (tkls != nullptr && !StressReflectiveCode) {
2061       if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2062         ciKlass* klass = tkls->exact_klass();
2063         assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2064         assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2065         return TypeInstPtr::make(klass->java_mirror());
















2066       }
2067     }
2068   }
2069 
2070   const TypeKlassPtr *tkls = tp->isa_klassptr();
2071   if (tkls != nullptr) {
2072     if (tkls->is_loaded() && tkls->klass_is_exact()) {
2073       ciKlass* klass = tkls->exact_klass();
2074       // We are loading a field from a Klass metaobject whose identity
2075       // is known at compile time (the type is "exact" or "precise").
2076       // Check for fields we know are maintained as constants by the VM.
2077       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2078         // The field is Klass::_super_check_offset.  Return its (constant) value.
2079         // (Folds up type checking code.)
2080         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2081         return TypeInt::make(klass->super_check_offset());
2082       }
2083       // Compute index into primary_supers array
2084       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2085       // Check for overflowing; use unsigned compare to handle the negative case.

2151   if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2152     Node* value = can_see_stored_value(mem,phase);
2153     if (value != nullptr && value->is_Con()) {
2154       assert(value->bottom_type()->higher_equal(_type),"sanity");
2155       return value->bottom_type();
2156     }
2157   }
2158 
2159   bool is_vect = (_type->isa_vect() != nullptr);
2160   if (is_instance && !is_vect) {
2161     // If we have an instance type and our memory input is the
2162     // programs's initial memory state, there is no matching store,
2163     // so just return a zero of the appropriate type -
2164     // except if it is vectorized - then we have no zero constant.
2165     Node *mem = in(MemNode::Memory);
2166     if (mem->is_Parm() && mem->in(0)->is_Start()) {
2167       assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2168       return Type::get_zero_type(_type->basic_type());
2169     }
2170   }
2171 
2172   Node* alloc = is_new_object_mark_load();
2173   if (alloc != nullptr) {
2174     return TypeX::make(markWord::prototype().value());









2175   }
2176 
2177   return _type;
2178 }
2179 
2180 //------------------------------match_edge-------------------------------------
2181 // Do we Match on this edge index or not?  Match only the address.
2182 uint LoadNode::match_edge(uint idx) const {
2183   return idx == MemNode::Address;
2184 }
2185 
2186 //--------------------------LoadBNode::Ideal--------------------------------------
2187 //
2188 //  If the previous store is to the same address as this load,
2189 //  and the value stored was larger than a byte, replace this load
2190 //  with the value stored truncated to a byte.  If no truncation is
2191 //  needed, the replacement is done in LoadNode::Identity().
2192 //
2193 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2194   Node* mem = in(MemNode::Memory);

2305   return LoadNode::Ideal(phase, can_reshape);
2306 }
2307 
2308 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2309   Node* mem = in(MemNode::Memory);
2310   Node* value = can_see_stored_value(mem,phase);
2311   if (value != nullptr && value->is_Con() &&
2312       !value->bottom_type()->higher_equal(_type)) {
2313     // If the input to the store does not fit with the load's result type,
2314     // it must be truncated. We can't delay until Ideal call since
2315     // a singleton Value is needed for split_thru_phi optimization.
2316     int con = value->get_int();
2317     return TypeInt::make((con << 16) >> 16);
2318   }
2319   return LoadNode::Value(phase);
2320 }
2321 
2322 //=============================================================================
2323 //----------------------------LoadKlassNode::make------------------------------
2324 // Polymorphic factory method:
2325 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {

2326   // sanity check the alias category against the created node type
2327   const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2328   assert(adr_type != nullptr, "expecting TypeKlassPtr");
2329 #ifdef _LP64
2330   if (adr_type->is_ptr_to_narrowklass()) {
2331     assert(UseCompressedClassPointers, "no compressed klasses");
2332     Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2333     return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2334   }
2335 #endif
2336   assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2337   return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2338 }
2339 
2340 //------------------------------Value------------------------------------------
2341 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2342   return klass_value_common(phase);
2343 }
2344 
2345 // In most cases, LoadKlassNode does not have the control input set. If the control

2352   // Either input is TOP ==> the result is TOP
2353   const Type *t1 = phase->type( in(MemNode::Memory) );
2354   if (t1 == Type::TOP)  return Type::TOP;
2355   Node *adr = in(MemNode::Address);
2356   const Type *t2 = phase->type( adr );
2357   if (t2 == Type::TOP)  return Type::TOP;
2358   const TypePtr *tp = t2->is_ptr();
2359   if (TypePtr::above_centerline(tp->ptr()) ||
2360       tp->ptr() == TypePtr::Null)  return Type::TOP;
2361 
2362   // Return a more precise klass, if possible
2363   const TypeInstPtr *tinst = tp->isa_instptr();
2364   if (tinst != nullptr) {
2365     ciInstanceKlass* ik = tinst->instance_klass();
2366     int offset = tinst->offset();
2367     if (ik == phase->C->env()->Class_klass()
2368         && (offset == java_lang_Class::klass_offset() ||
2369             offset == java_lang_Class::array_klass_offset())) {
2370       // We are loading a special hidden field from a Class mirror object,
2371       // the field which points to the VM's Klass metaobject.
2372       ciType* t = tinst->java_mirror_type();

2373       // java_mirror_type returns non-null for compile-time Class constants.
2374       if (t != nullptr) {
2375         // constant oop => constant klass
2376         if (offset == java_lang_Class::array_klass_offset()) {
2377           if (t->is_void()) {
2378             // We cannot create a void array.  Since void is a primitive type return null
2379             // klass.  Users of this result need to do a null check on the returned klass.
2380             return TypePtr::NULL_PTR;
2381           }
2382           return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2383         }
2384         if (!t->is_klass()) {
2385           // a primitive Class (e.g., int.class) has null for a klass field
2386           return TypePtr::NULL_PTR;
2387         }
2388         // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2389         return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2390       }
2391       // non-constant mirror, so we can't tell what's going on
2392     }
2393     if (!tinst->is_loaded())
2394       return _type;             // Bail out if not loaded
2395     if (offset == oopDesc::klass_offset_in_bytes()) {
2396       return tinst->as_klass_type(true);
2397     }
2398   }
2399 
2400   // Check for loading klass from an array
2401   const TypeAryPtr *tary = tp->isa_aryptr();
2402   if (tary != nullptr &&
2403       tary->offset() == oopDesc::klass_offset_in_bytes()) {
2404     return tary->as_klass_type(true);
2405   }
2406 
2407   // Check for loading klass from an array klass
2408   const TypeKlassPtr *tkls = tp->isa_klassptr();
2409   if (tkls != nullptr && !StressReflectiveCode) {
2410     if (!tkls->is_loaded())
2411      return _type;             // Bail out if not loaded
2412     if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2413         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2414       // // Always returning precise element type is incorrect,
2415       // // e.g., element type could be object and array may contain strings
2416       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2417 
2418       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2419       // according to the element type's subclassing.
2420       return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2421     }

2657 
2658   // Since they are not commoned, do not hash them:
2659   return NO_HASH;
2660 }
2661 
2662 //------------------------------Ideal------------------------------------------
2663 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2664 // When a store immediately follows a relevant allocation/initialization,
2665 // try to capture it into the initialization, or hoist it above.
2666 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2667   Node* p = MemNode::Ideal_common(phase, can_reshape);
2668   if (p)  return (p == NodeSentinel) ? nullptr : p;
2669 
2670   Node* mem     = in(MemNode::Memory);
2671   Node* address = in(MemNode::Address);
2672   Node* value   = in(MemNode::ValueIn);
2673   // Back-to-back stores to same address?  Fold em up.  Generally
2674   // unsafe if I have intervening uses...  Also disallowed for StoreCM
2675   // since they must follow each StoreP operation.  Redundant StoreCMs
2676   // are eliminated just before matching in final_graph_reshape.
2677   {
2678     Node* st = mem;
2679     // If Store 'st' has more than one use, we cannot fold 'st' away.
2680     // For example, 'st' might be the final state at a conditional
2681     // return.  Or, 'st' might be used by some node which is live at
2682     // the same time 'st' is live, which might be unschedulable.  So,
2683     // require exactly ONE user until such time as we clone 'mem' for
2684     // each of 'mem's uses (thus making the exactly-1-user-rule hold
2685     // true).
2686     while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2687       // Looking at a dead closed cycle of memory?
2688       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2689       assert(Opcode() == st->Opcode() ||
2690              st->Opcode() == Op_StoreVector ||
2691              Opcode() == Op_StoreVector ||
2692              st->Opcode() == Op_StoreVectorScatter ||
2693              Opcode() == Op_StoreVectorScatter ||
2694              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2695              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2696              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy

2697              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2698              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2699 
2700       if (st->in(MemNode::Address)->eqv_uncast(address) &&
2701           st->as_Store()->memory_size() <= this->memory_size()) {
2702         Node* use = st->raw_out(0);
2703         if (phase->is_IterGVN()) {
2704           phase->is_IterGVN()->rehash_node_delayed(use);
2705         }
2706         // It's OK to do this in the parser, since DU info is always accurate,
2707         // and the parser always refers to nodes via SafePointNode maps.
2708         use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2709         return this;
2710       }
2711       st = st->in(MemNode::Memory);
2712     }
2713   }
2714 
2715 
2716   // Capture an unaliased, unconditional, simple store into an initializer.

2773   // Load then Store?  Then the Store is useless
2774   if (val->is_Load() &&
2775       val->in(MemNode::Address)->eqv_uncast(adr) &&
2776       val->in(MemNode::Memory )->eqv_uncast(mem) &&
2777       val->as_Load()->store_Opcode() == Opcode()) {
2778     result = mem;
2779   }
2780 
2781   // Two stores in a row of the same value?
2782   if (result == this &&
2783       mem->is_Store() &&
2784       mem->in(MemNode::Address)->eqv_uncast(adr) &&
2785       mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2786       mem->Opcode() == Opcode()) {
2787     result = mem;
2788   }
2789 
2790   // Store of zero anywhere into a freshly-allocated object?
2791   // Then the store is useless.
2792   // (It must already have been captured by the InitializeNode.)
2793   if (result == this &&
2794       ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2795     // a newly allocated object is already all-zeroes everywhere
2796     if (mem->is_Proj() && mem->in(0)->is_Allocate()) {

2797       result = mem;
2798     }
2799 
2800     if (result == this) {
2801       // the store may also apply to zero-bits in an earlier object
2802       Node* prev_mem = find_previous_store(phase);
2803       // Steps (a), (b):  Walk past independent stores to find an exact match.
2804       if (prev_mem != nullptr) {
2805         Node* prev_val = can_see_stored_value(prev_mem, phase);
2806         if (prev_val != nullptr && prev_val == val) {
2807           // prev_val and val might differ by a cast; it would be good
2808           // to keep the more informative of the two.
2809           result = mem;
2810         }
2811       }
2812     }
2813   }
2814 
2815   PhaseIterGVN* igvn = phase->is_IterGVN();
2816   if (result != this && igvn != nullptr) {
2817     MemBarNode* trailing = trailing_membar();
2818     if (trailing != nullptr) {
2819 #ifdef ASSERT
2820       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();

2965 Node* StoreCMNode::Identity(PhaseGVN* phase) {
2966   // No need to card mark when storing a null ptr
2967   Node* my_store = in(MemNode::OopStore);
2968   if (my_store->is_Store()) {
2969     const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
2970     if( t1 == TypePtr::NULL_PTR ) {
2971       return in(MemNode::Memory);
2972     }
2973   }
2974   return this;
2975 }
2976 
2977 //=============================================================================
2978 //------------------------------Ideal---------------------------------------
2979 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
2980   Node* progress = StoreNode::Ideal(phase, can_reshape);
2981   if (progress != nullptr) return progress;
2982 
2983   Node* my_store = in(MemNode::OopStore);
2984   if (my_store->is_MergeMem()) {
2985     Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
2986     set_req_X(MemNode::OopStore, mem, phase);
2987     return this;




2988   }
2989 
2990   return nullptr;
2991 }
2992 
2993 //------------------------------Value-----------------------------------------
2994 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
2995   // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
2996   // If extra input is TOP ==> the result is TOP
2997   const Type* t = phase->type(in(MemNode::OopStore));
2998   if (t == Type::TOP) {
2999     return Type::TOP;
3000   }
3001   return StoreNode::Value(phase);
3002 }
3003 
3004 
3005 //=============================================================================
3006 //----------------------------------SCMemProjNode------------------------------
3007 const Type* SCMemProjNode::Value(PhaseGVN* phase) const

3138 // Clearing a short array is faster with stores
3139 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3140   // Already know this is a large node, do not try to ideal it
3141   if (_is_large) return nullptr;
3142 
3143   const int unit = BytesPerLong;
3144   const TypeX* t = phase->type(in(2))->isa_intptr_t();
3145   if (!t)  return nullptr;
3146   if (!t->is_con())  return nullptr;
3147   intptr_t raw_count = t->get_con();
3148   intptr_t size = raw_count;
3149   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3150   // Clearing nothing uses the Identity call.
3151   // Negative clears are possible on dead ClearArrays
3152   // (see jck test stmt114.stmt11402.val).
3153   if (size <= 0 || size % unit != 0)  return nullptr;
3154   intptr_t count = size / unit;
3155   // Length too long; communicate this to matchers and assemblers.
3156   // Assemblers are responsible to produce fast hardware clears for it.
3157   if (size > InitArrayShortSize) {
3158     return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3159   } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3160     return nullptr;
3161   }
3162   if (!IdealizeClearArrayNode) return nullptr;
3163   Node *mem = in(1);
3164   if( phase->type(mem)==Type::TOP ) return nullptr;
3165   Node *adr = in(3);
3166   const Type* at = phase->type(adr);
3167   if( at==Type::TOP ) return nullptr;
3168   const TypePtr* atp = at->isa_ptr();
3169   // adjust atp to be the correct array element address type
3170   if (atp == nullptr)  atp = TypePtr::BOTTOM;
3171   else              atp = atp->add_offset(Type::OffsetBot);
3172   // Get base for derived pointer purposes
3173   if( adr->Opcode() != Op_AddP ) Unimplemented();
3174   Node *base = adr->in(1);
3175 
3176   Node *zero = phase->makecon(TypeLong::ZERO);
3177   Node *off  = phase->MakeConX(BytesPerLong);
3178   mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3179   count--;
3180   while( count-- ) {
3181     mem = phase->transform(mem);
3182     adr = phase->transform(new AddPNode(base,adr,off));
3183     mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3184   }
3185   return mem;
3186 }
3187 
3188 //----------------------------step_through----------------------------------
3189 // Return allocation input memory edge if it is different instance
3190 // or itself if it is the one we are looking for.
3191 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3192   Node* n = *np;
3193   assert(n->is_ClearArray(), "sanity");
3194   intptr_t offset;
3195   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3196   // This method is called only before Allocate nodes are expanded
3197   // during macro nodes expansion. Before that ClearArray nodes are
3198   // only generated in PhaseMacroExpand::generate_arraycopy() (before
3199   // Allocate nodes are expanded) which follows allocations.
3200   assert(alloc != nullptr, "should have allocation");
3201   if (alloc->_idx == instance_id) {
3202     // Can not bypass initialization of the instance we are looking for.
3203     return false;
3204   }
3205   // Otherwise skip it.
3206   InitializeNode* init = alloc->initialization();
3207   if (init != nullptr)
3208     *np = init->in(TypeFunc::Memory);
3209   else
3210     *np = alloc->in(TypeFunc::Memory);
3211   return true;
3212 }
3213 
3214 //----------------------------clear_memory-------------------------------------
3215 // Generate code to initialize object storage to zero.
3216 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


3217                                    intptr_t start_offset,
3218                                    Node* end_offset,
3219                                    PhaseGVN* phase) {
3220   intptr_t offset = start_offset;
3221 
3222   int unit = BytesPerLong;
3223   if ((offset % unit) != 0) {
3224     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3225     adr = phase->transform(adr);
3226     const TypePtr* atp = TypeRawPtr::BOTTOM;
3227     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






3228     mem = phase->transform(mem);
3229     offset += BytesPerInt;
3230   }
3231   assert((offset % unit) == 0, "");
3232 
3233   // Initialize the remaining stuff, if any, with a ClearArray.
3234   return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
3235 }
3236 
3237 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,

3238                                    Node* start_offset,
3239                                    Node* end_offset,
3240                                    PhaseGVN* phase) {
3241   if (start_offset == end_offset) {
3242     // nothing to do
3243     return mem;
3244   }
3245 
3246   int unit = BytesPerLong;
3247   Node* zbase = start_offset;
3248   Node* zend  = end_offset;
3249 
3250   // Scale to the unit required by the CPU:
3251   if (!Matcher::init_array_count_is_in_bytes) {
3252     Node* shift = phase->intcon(exact_log2(unit));
3253     zbase = phase->transform(new URShiftXNode(zbase, shift) );
3254     zend  = phase->transform(new URShiftXNode(zend,  shift) );
3255   }
3256 
3257   // Bulk clear double-words
3258   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3259   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3260   mem = new ClearArrayNode(ctl, mem, zsize, adr, false);



3261   return phase->transform(mem);
3262 }
3263 
3264 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


3265                                    intptr_t start_offset,
3266                                    intptr_t end_offset,
3267                                    PhaseGVN* phase) {
3268   if (start_offset == end_offset) {
3269     // nothing to do
3270     return mem;
3271   }
3272 
3273   assert((end_offset % BytesPerInt) == 0, "odd end offset");
3274   intptr_t done_offset = end_offset;
3275   if ((done_offset % BytesPerLong) != 0) {
3276     done_offset -= BytesPerInt;
3277   }
3278   if (done_offset > start_offset) {
3279     mem = clear_memory(ctl, mem, dest,
3280                        start_offset, phase->MakeConX(done_offset), phase);
3281   }
3282   if (done_offset < end_offset) { // emit the final 32-bit store
3283     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3284     adr = phase->transform(adr);
3285     const TypePtr* atp = TypeRawPtr::BOTTOM;
3286     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






3287     mem = phase->transform(mem);
3288     done_offset += BytesPerInt;
3289   }
3290   assert(done_offset == end_offset, "");
3291   return mem;
3292 }
3293 
3294 //=============================================================================
3295 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3296   : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3297     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3298 #ifdef ASSERT
3299   , _pair_idx(0)
3300 #endif
3301 {
3302   init_class_id(Class_MemBar);
3303   Node* top = C->top();
3304   init_req(TypeFunc::I_O,top);
3305   init_req(TypeFunc::FramePtr,top);
3306   init_req(TypeFunc::ReturnAdr,top);

3411       PhaseIterGVN* igvn = phase->is_IterGVN();
3412       remove(igvn);
3413       // Must return either the original node (now dead) or a new node
3414       // (Do not return a top here, since that would break the uniqueness of top.)
3415       return new ConINode(TypeInt::ZERO);
3416     }
3417   }
3418   return progress ? this : nullptr;
3419 }
3420 
3421 //------------------------------Value------------------------------------------
3422 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3423   if( !in(0) ) return Type::TOP;
3424   if( phase->type(in(0)) == Type::TOP )
3425     return Type::TOP;
3426   return TypeTuple::MEMBAR;
3427 }
3428 
3429 //------------------------------match------------------------------------------
3430 // Construct projections for memory.
3431 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3432   switch (proj->_con) {
3433   case TypeFunc::Control:
3434   case TypeFunc::Memory:
3435     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3436   }
3437   ShouldNotReachHere();
3438   return nullptr;
3439 }
3440 
3441 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3442   trailing->_kind = TrailingStore;
3443   leading->_kind = LeadingStore;
3444 #ifdef ASSERT
3445   trailing->_pair_idx = leading->_idx;
3446   leading->_pair_idx = leading->_idx;
3447 #endif
3448 }
3449 
3450 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3451   trailing->_kind = TrailingLoadStore;

3698   return (req() > RawStores);
3699 }
3700 
3701 void InitializeNode::set_complete(PhaseGVN* phase) {
3702   assert(!is_complete(), "caller responsibility");
3703   _is_complete = Complete;
3704 
3705   // After this node is complete, it contains a bunch of
3706   // raw-memory initializations.  There is no need for
3707   // it to have anything to do with non-raw memory effects.
3708   // Therefore, tell all non-raw users to re-optimize themselves,
3709   // after skipping the memory effects of this initialization.
3710   PhaseIterGVN* igvn = phase->is_IterGVN();
3711   if (igvn)  igvn->add_users_to_worklist(this);
3712 }
3713 
3714 // convenience function
3715 // return false if the init contains any stores already
3716 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3717   InitializeNode* init = initialization();
3718   if (init == nullptr || init->is_complete())  return false;


3719   init->remove_extra_zeroes();
3720   // for now, if this allocation has already collected any inits, bail:
3721   if (init->is_non_zero())  return false;
3722   init->set_complete(phase);
3723   return true;
3724 }
3725 
3726 void InitializeNode::remove_extra_zeroes() {
3727   if (req() == RawStores)  return;
3728   Node* zmem = zero_memory();
3729   uint fill = RawStores;
3730   for (uint i = fill; i < req(); i++) {
3731     Node* n = in(i);
3732     if (n->is_top() || n == zmem)  continue;  // skip
3733     if (fill < i)  set_req(fill, n);          // compact
3734     ++fill;
3735   }
3736   // delete any empty spaces created:
3737   while (fill < req()) {
3738     del_req(fill);

3876             // store node that we'd like to capture. We need to check
3877             // the uses of the MergeMemNode.
3878             mems.push(n);
3879           }
3880         } else if (n->is_Mem()) {
3881           Node* other_adr = n->in(MemNode::Address);
3882           if (other_adr == adr) {
3883             failed = true;
3884             break;
3885           } else {
3886             const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3887             if (other_t_adr != nullptr) {
3888               int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3889               if (other_alias_idx == alias_idx) {
3890                 // A load from the same memory slice as the store right
3891                 // after the InitializeNode. We check the control of the
3892                 // object/array that is loaded from. If it's the same as
3893                 // the store control then we cannot capture the store.
3894                 assert(!n->is_Store(), "2 stores to same slice on same control?");
3895                 Node* base = other_adr;






3896                 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3897                 base = base->in(AddPNode::Base);
3898                 if (base != nullptr) {
3899                   base = base->uncast();
3900                   if (base->is_Proj() && base->in(0) == alloc) {
3901                     failed = true;
3902                     break;
3903                   }
3904                 }
3905               }
3906             }
3907           }
3908         } else {
3909           failed = true;
3910           break;
3911         }
3912       }
3913     }
3914   }
3915   if (failed) {

4462         //   z's_done      12  16  16  16    12  16    12
4463         //   z's_needed    12  16  16  16    16  16    16
4464         //   zsize          0   0   0   0     4   0     4
4465         if (next_full_store < 0) {
4466           // Conservative tack:  Zero to end of current word.
4467           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4468         } else {
4469           // Zero to beginning of next fully initialized word.
4470           // Or, don't zero at all, if we are already in that word.
4471           assert(next_full_store >= zeroes_needed, "must go forward");
4472           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4473           zeroes_needed = next_full_store;
4474         }
4475       }
4476 
4477       if (zeroes_needed > zeroes_done) {
4478         intptr_t zsize = zeroes_needed - zeroes_done;
4479         // Do some incremental zeroing on rawmem, in parallel with inits.
4480         zeroes_done = align_down(zeroes_done, BytesPerInt);
4481         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


4482                                               zeroes_done, zeroes_needed,
4483                                               phase);
4484         zeroes_done = zeroes_needed;
4485         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4486           do_zeroing = false;   // leave the hole, next time
4487       }
4488     }
4489 
4490     // Collect the store and move on:
4491     phase->replace_input_of(st, MemNode::Memory, inits);
4492     inits = st;                 // put it on the linearized chain
4493     set_req(i, zmem);           // unhook from previous position
4494 
4495     if (zeroes_done == st_off)
4496       zeroes_done = next_init_off;
4497 
4498     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4499 
4500     #ifdef ASSERT
4501     // Various order invariants.  Weaker than stores_are_sane because

4521   remove_extra_zeroes();        // clear out all the zmems left over
4522   add_req(inits);
4523 
4524   if (!(UseTLAB && ZeroTLAB)) {
4525     // If anything remains to be zeroed, zero it all now.
4526     zeroes_done = align_down(zeroes_done, BytesPerInt);
4527     // if it is the last unused 4 bytes of an instance, forget about it
4528     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4529     if (zeroes_done + BytesPerLong >= size_limit) {
4530       AllocateNode* alloc = allocation();
4531       assert(alloc != nullptr, "must be present");
4532       if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
4533         Node* klass_node = alloc->in(AllocateNode::KlassNode);
4534         ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4535         if (zeroes_done == k->layout_helper())
4536           zeroes_done = size_limit;
4537       }
4538     }
4539     if (zeroes_done < size_limit) {
4540       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


4541                                             zeroes_done, size_in_bytes, phase);
4542     }
4543   }
4544 
4545   set_complete(phase);
4546   return rawmem;
4547 }
4548 
4549 
4550 #ifdef ASSERT
4551 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
4552   if (is_complete())
4553     return true;                // stores could be anything at this point
4554   assert(allocation() != nullptr, "must be present");
4555   intptr_t last_off = allocation()->minimum_header_size();
4556   for (uint i = InitializeNode::RawStores; i < req(); i++) {
4557     Node* st = in(i);
4558     intptr_t st_off = get_store_offset(st, phase);
4559     if (st_off < 0)  continue;  // ignore dead garbage
4560     if (last_off > st_off) {

   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciFlatArrayKlass.hpp"
  27 #include "classfile/javaClasses.hpp"
  28 #include "classfile/systemDictionary.hpp"
  29 #include "compiler/compileLog.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/c2/barrierSetC2.hpp"
  32 #include "gc/shared/tlab_globals.hpp"
  33 #include "memory/allocation.inline.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "oops/objArrayKlass.hpp"
  36 #include "opto/addnode.hpp"
  37 #include "opto/arraycopynode.hpp"
  38 #include "opto/cfgnode.hpp"
  39 #include "opto/regalloc.hpp"
  40 #include "opto/compile.hpp"
  41 #include "opto/connode.hpp"
  42 #include "opto/convertnode.hpp"
  43 #include "opto/inlinetypenode.hpp"
  44 #include "opto/loopnode.hpp"
  45 #include "opto/machnode.hpp"
  46 #include "opto/matcher.hpp"
  47 #include "opto/memnode.hpp"
  48 #include "opto/mulnode.hpp"
  49 #include "opto/narrowptrnode.hpp"
  50 #include "opto/phaseX.hpp"
  51 #include "opto/regmask.hpp"
  52 #include "opto/rootnode.hpp"
  53 #include "opto/vectornode.hpp"
  54 #include "utilities/align.hpp"
  55 #include "utilities/copy.hpp"
  56 #include "utilities/macros.hpp"
  57 #include "utilities/powerOfTwo.hpp"
  58 #include "utilities/vmError.hpp"
  59 
  60 // Portions of code courtesy of Clifford Click
  61 
  62 // Optimization - Graph Style
  63 

 217   bool is_instance = t_oop->is_known_instance_field();
 218   PhaseIterGVN *igvn = phase->is_IterGVN();
 219   if (is_instance && igvn != nullptr && result->is_Phi()) {
 220     PhiNode *mphi = result->as_Phi();
 221     assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
 222     const TypePtr *t = mphi->adr_type();
 223     bool do_split = false;
 224     // In the following cases, Load memory input can be further optimized based on
 225     // its precise address type
 226     if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
 227       do_split = true;
 228     } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
 229       const TypeOopPtr* mem_t =
 230         t->is_oopptr()->cast_to_exactness(true)
 231         ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
 232         ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
 233       if (t_oop->is_aryptr()) {
 234         mem_t = mem_t->is_aryptr()
 235                      ->cast_to_stable(t_oop->is_aryptr()->is_stable())
 236                      ->cast_to_size(t_oop->is_aryptr()->size())
 237                      ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
 238                      ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
 239                      ->with_offset(t_oop->is_aryptr()->offset())
 240                      ->is_aryptr();
 241       }
 242       do_split = mem_t == t_oop;
 243     }
 244     if (do_split) {
 245       // clone the Phi with our address type
 246       result = mphi->split_out_instance(t_adr, igvn);
 247     } else {
 248       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 249     }
 250   }
 251   return result;
 252 }
 253 
 254 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 255   uint alias_idx = phase->C->get_alias_index(tp);
 256   Node *mem = mmem;
 257 #ifdef ASSERT
 258   {
 259     // Check that current type is consistent with the alias index used during graph construction
 260     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 261     bool consistent =  adr_check == nullptr || adr_check->empty() ||
 262                        phase->C->must_alias(adr_check, alias_idx );
 263     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 264     if( !consistent && adr_check != nullptr && !adr_check->empty() &&
 265         tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 266         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 267         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 268           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 269           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 270       // don't assert if it is dead code.
 271       consistent = true;
 272     }
 273     if( !consistent ) {
 274       st->print("alias_idx==%d, adr_check==", alias_idx);
 275       if( adr_check == nullptr ) {
 276         st->print("null");
 277       } else {
 278         adr_check->dump();
 279       }
 280       st->cr();
 281       print_alias_types();
 282       assert(consistent, "adr_check must match alias idx");
 283     }
 284   }
 285 #endif

1013       assert(ld_alloc != nullptr, "need an alloc");
1014       assert(addp->is_AddP(), "address must be addp");
1015       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1016       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1017       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1018       addp->set_req(AddPNode::Base, src);
1019       addp->set_req(AddPNode::Address, src);
1020     } else {
1021       assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1022              ac->as_ArrayCopy()->is_copyof_validated() ||
1023              ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1024       assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1025       addp->set_req(AddPNode::Base, src);
1026       addp->set_req(AddPNode::Address, src);
1027 
1028       const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1029       BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1030       if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1031 
1032       uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1033       uint shift  = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1034 
1035       Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1036 #ifdef _LP64
1037       diff = phase->transform(new ConvI2LNode(diff));
1038 #endif
1039       diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1040 
1041       Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1042       addp->set_req(AddPNode::Offset, offset);
1043     }
1044     addp = phase->transform(addp);
1045 #ifdef ASSERT
1046     const TypePtr* adr_type = phase->type(addp)->is_ptr();
1047     ld->_adr_type = adr_type;
1048 #endif
1049     ld->set_req(MemNode::Address, addp);
1050     ld->set_req(0, ctl);
1051     ld->set_req(MemNode::Memory, mem);
1052     // load depends on the tests that validate the arraycopy
1053     ld->_control_dependency = UnknownControl;

1134         // Same base, same offset.
1135         // Possible improvement for arrays: check index value instead of absolute offset.
1136 
1137         // At this point we have proven something like this setup:
1138         //   B = << base >>
1139         //   L =  LoadQ(AddP(Check/CastPP(B), #Off))
1140         //   S = StoreQ(AddP(             B , #Off), V)
1141         // (Actually, we haven't yet proven the Q's are the same.)
1142         // In other words, we are loading from a casted version of
1143         // the same pointer-and-offset that we stored to.
1144         // Casted version may carry a dependency and it is respected.
1145         // Thus, we are able to replace L by V.
1146       }
1147       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1148       if (store_Opcode() != st->Opcode()) {
1149         return nullptr;
1150       }
1151       // LoadVector/StoreVector needs additional check to ensure the types match.
1152       if (st->is_StoreVector()) {
1153         const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
1154         const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1155         if (in_vt != out_vt) {
1156           return nullptr;
1157         }
1158       }
1159       return st->in(MemNode::ValueIn);
1160     }
1161 
1162     // A load from a freshly-created object always returns zero.
1163     // (This can happen after LoadNode::Ideal resets the load's memory input
1164     // to find_captured_store, which returned InitializeNode::zero_memory.)
1165     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1166         (st->in(0) == ld_alloc) &&
1167         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1168       // return a zero value for the load's basic type
1169       // (This is one of the few places where a generic PhaseTransform
1170       // can create new nodes.  Think of it as lazily manifesting
1171       // virtually pre-existing constants.)
1172       Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1173       if (default_value != nullptr) {
1174         return default_value;
1175       }
1176       assert(ld_alloc->in(AllocateNode::RawDefaultValue) == nullptr, "default value may not be null");
1177       if (memory_type() != T_VOID) {
1178         if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1179           // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1180           // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1181           // by the ArrayCopyNode.
1182           return phase->zerocon(memory_type());
1183         }
1184       } else {
1185         // TODO: materialize all-zero vector constant
1186         assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1187       }
1188     }
1189 
1190     // A load from an initialization barrier can match a captured store.
1191     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1192       InitializeNode* init = st->in(0)->as_Initialize();
1193       AllocateNode* alloc = init->allocation();
1194       if ((alloc != nullptr) && (alloc == ld_alloc)) {
1195         // examine a captured store value
1196         st = init->find_captured_store(ld_off, memory_size(), phase);

1224 //----------------------is_instance_field_load_with_local_phi------------------
1225 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1226   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1227       in(Address)->is_AddP() ) {
1228     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1229     // Only instances and boxed values.
1230     if( t_oop != nullptr &&
1231         (t_oop->is_ptr_to_boxed_value() ||
1232          t_oop->is_known_instance_field()) &&
1233         t_oop->offset() != Type::OffsetBot &&
1234         t_oop->offset() != Type::OffsetTop) {
1235       return true;
1236     }
1237   }
1238   return false;
1239 }
1240 
1241 //------------------------------Identity---------------------------------------
1242 // Loads are identity if previous store is to same address
1243 Node* LoadNode::Identity(PhaseGVN* phase) {
1244   // Loading from an InlineType? The InlineType has the values of
1245   // all fields as input. Look for the field with matching offset.
1246   Node* addr = in(Address);
1247   intptr_t offset;
1248   Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1249   if (base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1250     Node* value = base->as_InlineType()->field_value_by_offset((int)offset, true);
1251     if (value != nullptr) {
1252       if (Opcode() == Op_LoadN) {
1253         // Encode oop value if we are loading a narrow oop
1254         assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1255         value = phase->transform(new EncodePNode(value, bottom_type()));
1256       }
1257       return value;
1258     }
1259   }
1260 
1261   // If the previous store-maker is the right kind of Store, and the store is
1262   // to the same address, then we are equal to the value stored.
1263   Node* mem = in(Memory);
1264   Node* value = can_see_stored_value(mem, phase);
1265   if( value ) {
1266     // byte, short & char stores truncate naturally.
1267     // A load has to load the truncated value which requires
1268     // some sort of masking operation and that requires an
1269     // Ideal call instead of an Identity call.
1270     if (memory_size() < BytesPerInt) {
1271       // If the input to the store does not fit with the load's result type,
1272       // it must be truncated via an Ideal call.
1273       if (!phase->type(value)->higher_equal(phase->type(this)))
1274         return this;
1275     }
1276     // (This works even when value is a Con, but LoadNode::Value
1277     // usually runs first, producing the singleton type of the Con.)
1278     if (!has_pinned_control_dependency() || value->is_Con()) {
1279       return value;
1280     } else {

1999       }
2000     }
2001 
2002     // Don't do this for integer types. There is only potential profit if
2003     // the element type t is lower than _type; that is, for int types, if _type is
2004     // more restrictive than t.  This only happens here if one is short and the other
2005     // char (both 16 bits), and in those cases we've made an intentional decision
2006     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2007     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2008     //
2009     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2010     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
2011     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2012     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
2013     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2014     // In fact, that could have been the original type of p1, and p1 could have
2015     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2016     // expression (LShiftL quux 3) independently optimized to the constant 8.
2017     if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2018         && (_type->isa_vect() == nullptr)
2019         && !ary->is_flat()
2020         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2021       // t might actually be lower than _type, if _type is a unique
2022       // concrete subclass of abstract class t.
2023       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
2024         const Type* jt = t->join_speculative(_type);
2025         // In any case, do not allow the join, per se, to empty out the type.
2026         if (jt->empty() && !t->empty()) {
2027           // This can happen if a interface-typed array narrows to a class type.
2028           jt = _type;
2029         }
2030 #ifdef ASSERT
2031         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2032           // The pointers in the autobox arrays are always non-null
2033           Node* base = adr->in(AddPNode::Base);
2034           if ((base != nullptr) && base->is_DecodeN()) {
2035             // Get LoadN node which loads IntegerCache.cache field
2036             base = base->in(1);
2037           }
2038           if ((base != nullptr) && base->is_Con()) {
2039             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2040             if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2041               // It could be narrow oop
2042               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2043             }
2044           }
2045         }
2046 #endif
2047         return jt;
2048       }
2049     }
2050   } else if (tp->base() == Type::InstPtr) {
2051     assert( off != Type::OffsetBot ||
2052             // arrays can be cast to Objects
2053             !tp->isa_instptr() ||
2054             tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2055             // Default value load
2056             tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2057             // unsafe field access may not have a constant offset
2058             C->has_unsafe_access(),
2059             "Field accesses must be precise" );
2060     // For oop loads, we expect the _type to be precise.
2061 

2062     const TypeInstPtr* tinst = tp->is_instptr();
2063     BasicType bt = memory_type();
2064 
2065     // Optimize loads from constant fields.
2066     ciObject* const_oop = tinst->const_oop();
2067     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2068       ciType* mirror_type = const_oop->as_instance()->java_mirror_type();
2069       if (mirror_type != nullptr) {
2070         const Type* const_oop = nullptr;
2071         ciInlineKlass* vk = mirror_type->is_inlinetype() ? mirror_type->as_inline_klass() : nullptr;
2072         // Fold default value loads
2073         if (vk != nullptr && off == vk->default_value_offset()) {
2074           const_oop = TypeInstPtr::make(vk->default_instance());
2075         }
2076         // Fold class mirror loads
2077         if (off == java_lang_Class::primary_mirror_offset()) {
2078           const_oop = (vk == nullptr) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->ref_instance());
2079         } else if (off == java_lang_Class::secondary_mirror_offset()) {
2080           const_oop = (vk == nullptr) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->val_instance());
2081         }
2082         if (const_oop != nullptr) {
2083           return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
2084         }
2085       }
2086       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2087       if (con_type != nullptr) {
2088         return con_type;
2089       }
2090     }
2091   } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2092     assert(off != Type::OffsetBot ||
2093             !tp->isa_instklassptr() ||
2094            // arrays can be cast to Objects
2095            tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2096            // also allow array-loading from the primary supertype
2097            // array during subtype checks
2098            Opcode() == Op_LoadKlass,
2099            "Field accesses must be precise");
2100     // For klass/static loads, we expect the _type to be precise
2101   } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
2102     if (adr->is_Load() && off == 0) {
2103       /* With mirrors being an indirect in the Klass*
2104        * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2105        * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2106        *
2107        * So check the type and klass of the node before the LoadP.
2108        */
2109       Node* adr2 = adr->in(MemNode::Address);
2110       const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2111       if (tkls != nullptr) {
2112         if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2113           ciKlass* klass = tkls->exact_klass();
2114           assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2115           assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2116           return TypeInstPtr::make(klass->java_mirror());
2117         }
2118       }
2119     } else {
2120       // Check for a load of the default value offset from the InlineKlassFixedBlock:
2121       // LoadI(LoadP(inline_klass, adr_inlineklass_fixed_block_offset), default_value_offset_offset)
2122       intptr_t offset = 0;
2123       Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2124       if (base != nullptr && base->is_Load() && offset == in_bytes(InlineKlass::default_value_offset_offset())) {
2125         const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
2126         if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact() && tkls->exact_klass()->is_inlinetype() &&
2127             tkls->offset() == in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())) {
2128           assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
2129           assert(Opcode() == Op_LoadI, "must load an int from fixed block");
2130           return TypeInt::make(tkls->exact_klass()->as_inline_klass()->default_value_offset());
2131         }
2132       }
2133     }
2134   }
2135 
2136   const TypeKlassPtr *tkls = tp->isa_klassptr();
2137   if (tkls != nullptr) {
2138     if (tkls->is_loaded() && tkls->klass_is_exact()) {
2139       ciKlass* klass = tkls->exact_klass();
2140       // We are loading a field from a Klass metaobject whose identity
2141       // is known at compile time (the type is "exact" or "precise").
2142       // Check for fields we know are maintained as constants by the VM.
2143       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2144         // The field is Klass::_super_check_offset.  Return its (constant) value.
2145         // (Folds up type checking code.)
2146         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2147         return TypeInt::make(klass->super_check_offset());
2148       }
2149       // Compute index into primary_supers array
2150       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2151       // Check for overflowing; use unsigned compare to handle the negative case.

2217   if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2218     Node* value = can_see_stored_value(mem,phase);
2219     if (value != nullptr && value->is_Con()) {
2220       assert(value->bottom_type()->higher_equal(_type),"sanity");
2221       return value->bottom_type();
2222     }
2223   }
2224 
2225   bool is_vect = (_type->isa_vect() != nullptr);
2226   if (is_instance && !is_vect) {
2227     // If we have an instance type and our memory input is the
2228     // programs's initial memory state, there is no matching store,
2229     // so just return a zero of the appropriate type -
2230     // except if it is vectorized - then we have no zero constant.
2231     Node *mem = in(MemNode::Memory);
2232     if (mem->is_Parm() && mem->in(0)->is_Start()) {
2233       assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2234       return Type::get_zero_type(_type->basic_type());
2235     }
2236   }

2237   Node* alloc = is_new_object_mark_load();
2238   if (alloc != nullptr) {
2239     if (EnableValhalla) {
2240       // The mark word may contain property bits (inline, flat, null-free)
2241       Node* klass_node = alloc->in(AllocateNode::KlassNode);
2242       const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2243       if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2244         return TypeX::make(tkls->exact_klass()->prototype_header().value());
2245       }
2246     } else {
2247       return TypeX::make(markWord::prototype().value());
2248     }
2249   }
2250 
2251   return _type;
2252 }
2253 
2254 //------------------------------match_edge-------------------------------------
2255 // Do we Match on this edge index or not?  Match only the address.
2256 uint LoadNode::match_edge(uint idx) const {
2257   return idx == MemNode::Address;
2258 }
2259 
2260 //--------------------------LoadBNode::Ideal--------------------------------------
2261 //
2262 //  If the previous store is to the same address as this load,
2263 //  and the value stored was larger than a byte, replace this load
2264 //  with the value stored truncated to a byte.  If no truncation is
2265 //  needed, the replacement is done in LoadNode::Identity().
2266 //
2267 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2268   Node* mem = in(MemNode::Memory);

2379   return LoadNode::Ideal(phase, can_reshape);
2380 }
2381 
2382 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2383   Node* mem = in(MemNode::Memory);
2384   Node* value = can_see_stored_value(mem,phase);
2385   if (value != nullptr && value->is_Con() &&
2386       !value->bottom_type()->higher_equal(_type)) {
2387     // If the input to the store does not fit with the load's result type,
2388     // it must be truncated. We can't delay until Ideal call since
2389     // a singleton Value is needed for split_thru_phi optimization.
2390     int con = value->get_int();
2391     return TypeInt::make((con << 16) >> 16);
2392   }
2393   return LoadNode::Value(phase);
2394 }
2395 
2396 //=============================================================================
2397 //----------------------------LoadKlassNode::make------------------------------
2398 // Polymorphic factory method:
2399 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2400                           const TypeKlassPtr* tk) {
2401   // sanity check the alias category against the created node type
2402   const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2403   assert(adr_type != nullptr, "expecting TypeKlassPtr");
2404 #ifdef _LP64
2405   if (adr_type->is_ptr_to_narrowklass()) {
2406     assert(UseCompressedClassPointers, "no compressed klasses");
2407     Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2408     return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2409   }
2410 #endif
2411   assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2412   return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2413 }
2414 
2415 //------------------------------Value------------------------------------------
2416 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2417   return klass_value_common(phase);
2418 }
2419 
2420 // In most cases, LoadKlassNode does not have the control input set. If the control

2427   // Either input is TOP ==> the result is TOP
2428   const Type *t1 = phase->type( in(MemNode::Memory) );
2429   if (t1 == Type::TOP)  return Type::TOP;
2430   Node *adr = in(MemNode::Address);
2431   const Type *t2 = phase->type( adr );
2432   if (t2 == Type::TOP)  return Type::TOP;
2433   const TypePtr *tp = t2->is_ptr();
2434   if (TypePtr::above_centerline(tp->ptr()) ||
2435       tp->ptr() == TypePtr::Null)  return Type::TOP;
2436 
2437   // Return a more precise klass, if possible
2438   const TypeInstPtr *tinst = tp->isa_instptr();
2439   if (tinst != nullptr) {
2440     ciInstanceKlass* ik = tinst->instance_klass();
2441     int offset = tinst->offset();
2442     if (ik == phase->C->env()->Class_klass()
2443         && (offset == java_lang_Class::klass_offset() ||
2444             offset == java_lang_Class::array_klass_offset())) {
2445       // We are loading a special hidden field from a Class mirror object,
2446       // the field which points to the VM's Klass metaobject.
2447       bool null_free = false;
2448       ciType* t = tinst->java_mirror_type(&null_free);
2449       // java_mirror_type returns non-null for compile-time Class constants.
2450       if (t != nullptr) {
2451         // constant oop => constant klass
2452         if (offset == java_lang_Class::array_klass_offset()) {
2453           if (t->is_void()) {
2454             // We cannot create a void array.  Since void is a primitive type return null
2455             // klass.  Users of this result need to do a null check on the returned klass.
2456             return TypePtr::NULL_PTR;
2457           }
2458           return TypeKlassPtr::make(ciArrayKlass::make(t, null_free), Type::trust_interfaces);
2459         }
2460         if (!t->is_klass()) {
2461           // a primitive Class (e.g., int.class) has null for a klass field
2462           return TypePtr::NULL_PTR;
2463         }
2464         // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2465         return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2466       }
2467       // non-constant mirror, so we can't tell what's going on
2468     }
2469     if (!tinst->is_loaded())
2470       return _type;             // Bail out if not loaded
2471     if (offset == oopDesc::klass_offset_in_bytes()) {
2472       return tinst->as_klass_type(true);
2473     }
2474   }
2475 
2476   // Check for loading klass from an array
2477   const TypeAryPtr* tary = tp->isa_aryptr();
2478   if (tary != nullptr &&
2479       tary->offset() == oopDesc::klass_offset_in_bytes()) {
2480     return tary->as_klass_type(true);
2481   }
2482 
2483   // Check for loading klass from an array klass
2484   const TypeKlassPtr *tkls = tp->isa_klassptr();
2485   if (tkls != nullptr && !StressReflectiveCode) {
2486     if (!tkls->is_loaded())
2487      return _type;             // Bail out if not loaded
2488     if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2489         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2490       // // Always returning precise element type is incorrect,
2491       // // e.g., element type could be object and array may contain strings
2492       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2493 
2494       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2495       // according to the element type's subclassing.
2496       return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2497     }

2733 
2734   // Since they are not commoned, do not hash them:
2735   return NO_HASH;
2736 }
2737 
2738 //------------------------------Ideal------------------------------------------
2739 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2740 // When a store immediately follows a relevant allocation/initialization,
2741 // try to capture it into the initialization, or hoist it above.
2742 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2743   Node* p = MemNode::Ideal_common(phase, can_reshape);
2744   if (p)  return (p == NodeSentinel) ? nullptr : p;
2745 
2746   Node* mem     = in(MemNode::Memory);
2747   Node* address = in(MemNode::Address);
2748   Node* value   = in(MemNode::ValueIn);
2749   // Back-to-back stores to same address?  Fold em up.  Generally
2750   // unsafe if I have intervening uses...  Also disallowed for StoreCM
2751   // since they must follow each StoreP operation.  Redundant StoreCMs
2752   // are eliminated just before matching in final_graph_reshape.
2753   if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
2754     Node* st = mem;
2755     // If Store 'st' has more than one use, we cannot fold 'st' away.
2756     // For example, 'st' might be the final state at a conditional
2757     // return.  Or, 'st' might be used by some node which is live at
2758     // the same time 'st' is live, which might be unschedulable.  So,
2759     // require exactly ONE user until such time as we clone 'mem' for
2760     // each of 'mem's uses (thus making the exactly-1-user-rule hold
2761     // true).
2762     while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2763       // Looking at a dead closed cycle of memory?
2764       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2765       assert(Opcode() == st->Opcode() ||
2766              st->Opcode() == Op_StoreVector ||
2767              Opcode() == Op_StoreVector ||
2768              st->Opcode() == Op_StoreVectorScatter ||
2769              Opcode() == Op_StoreVectorScatter ||
2770              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2771              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2772              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2773              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
2774              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2775              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2776 
2777       if (st->in(MemNode::Address)->eqv_uncast(address) &&
2778           st->as_Store()->memory_size() <= this->memory_size()) {
2779         Node* use = st->raw_out(0);
2780         if (phase->is_IterGVN()) {
2781           phase->is_IterGVN()->rehash_node_delayed(use);
2782         }
2783         // It's OK to do this in the parser, since DU info is always accurate,
2784         // and the parser always refers to nodes via SafePointNode maps.
2785         use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2786         return this;
2787       }
2788       st = st->in(MemNode::Memory);
2789     }
2790   }
2791 
2792 
2793   // Capture an unaliased, unconditional, simple store into an initializer.

2850   // Load then Store?  Then the Store is useless
2851   if (val->is_Load() &&
2852       val->in(MemNode::Address)->eqv_uncast(adr) &&
2853       val->in(MemNode::Memory )->eqv_uncast(mem) &&
2854       val->as_Load()->store_Opcode() == Opcode()) {
2855     result = mem;
2856   }
2857 
2858   // Two stores in a row of the same value?
2859   if (result == this &&
2860       mem->is_Store() &&
2861       mem->in(MemNode::Address)->eqv_uncast(adr) &&
2862       mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2863       mem->Opcode() == Opcode()) {
2864     result = mem;
2865   }
2866 
2867   // Store of zero anywhere into a freshly-allocated object?
2868   // Then the store is useless.
2869   // (It must already have been captured by the InitializeNode.)
2870   if (result == this && ReduceFieldZeroing) {

2871     // a newly allocated object is already all-zeroes everywhere
2872     if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
2873         (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
2874       result = mem;
2875     }
2876 
2877     if (result == this && phase->type(val)->is_zero_type()) {
2878       // the store may also apply to zero-bits in an earlier object
2879       Node* prev_mem = find_previous_store(phase);
2880       // Steps (a), (b):  Walk past independent stores to find an exact match.
2881       if (prev_mem != nullptr) {
2882         Node* prev_val = can_see_stored_value(prev_mem, phase);
2883         if (prev_val != nullptr && prev_val == val) {
2884           // prev_val and val might differ by a cast; it would be good
2885           // to keep the more informative of the two.
2886           result = mem;
2887         }
2888       }
2889     }
2890   }
2891 
2892   PhaseIterGVN* igvn = phase->is_IterGVN();
2893   if (result != this && igvn != nullptr) {
2894     MemBarNode* trailing = trailing_membar();
2895     if (trailing != nullptr) {
2896 #ifdef ASSERT
2897       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();

3042 Node* StoreCMNode::Identity(PhaseGVN* phase) {
3043   // No need to card mark when storing a null ptr
3044   Node* my_store = in(MemNode::OopStore);
3045   if (my_store->is_Store()) {
3046     const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
3047     if( t1 == TypePtr::NULL_PTR ) {
3048       return in(MemNode::Memory);
3049     }
3050   }
3051   return this;
3052 }
3053 
3054 //=============================================================================
3055 //------------------------------Ideal---------------------------------------
3056 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
3057   Node* progress = StoreNode::Ideal(phase, can_reshape);
3058   if (progress != nullptr) return progress;
3059 
3060   Node* my_store = in(MemNode::OopStore);
3061   if (my_store->is_MergeMem()) {
3062     if (oop_alias_idx() != phase->C->get_alias_index(TypeAryPtr::INLINES) ||
3063         phase->C->flat_accesses_share_alias()) {
3064       // The alias that was recorded is no longer accurate enough.
3065       Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
3066       set_req_X(MemNode::OopStore, mem, phase);
3067       return this;
3068     }
3069   }
3070 
3071   return nullptr;
3072 }
3073 
3074 //------------------------------Value-----------------------------------------
3075 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
3076   // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
3077   // If extra input is TOP ==> the result is TOP
3078   const Type* t = phase->type(in(MemNode::OopStore));
3079   if (t == Type::TOP) {
3080     return Type::TOP;
3081   }
3082   return StoreNode::Value(phase);
3083 }
3084 
3085 
3086 //=============================================================================
3087 //----------------------------------SCMemProjNode------------------------------
3088 const Type* SCMemProjNode::Value(PhaseGVN* phase) const

3219 // Clearing a short array is faster with stores
3220 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3221   // Already know this is a large node, do not try to ideal it
3222   if (_is_large) return nullptr;
3223 
3224   const int unit = BytesPerLong;
3225   const TypeX* t = phase->type(in(2))->isa_intptr_t();
3226   if (!t)  return nullptr;
3227   if (!t->is_con())  return nullptr;
3228   intptr_t raw_count = t->get_con();
3229   intptr_t size = raw_count;
3230   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3231   // Clearing nothing uses the Identity call.
3232   // Negative clears are possible on dead ClearArrays
3233   // (see jck test stmt114.stmt11402.val).
3234   if (size <= 0 || size % unit != 0)  return nullptr;
3235   intptr_t count = size / unit;
3236   // Length too long; communicate this to matchers and assemblers.
3237   // Assemblers are responsible to produce fast hardware clears for it.
3238   if (size > InitArrayShortSize) {
3239     return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3240   } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3241     return nullptr;
3242   }
3243   if (!IdealizeClearArrayNode) return nullptr;
3244   Node *mem = in(1);
3245   if( phase->type(mem)==Type::TOP ) return nullptr;
3246   Node *adr = in(3);
3247   const Type* at = phase->type(adr);
3248   if( at==Type::TOP ) return nullptr;
3249   const TypePtr* atp = at->isa_ptr();
3250   // adjust atp to be the correct array element address type
3251   if (atp == nullptr)  atp = TypePtr::BOTTOM;
3252   else              atp = atp->add_offset(Type::OffsetBot);
3253   // Get base for derived pointer purposes
3254   if( adr->Opcode() != Op_AddP ) Unimplemented();
3255   Node *base = adr->in(1);
3256 
3257   Node *val = in(4);
3258   Node *off  = phase->MakeConX(BytesPerLong);
3259   mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3260   count--;
3261   while( count-- ) {
3262     mem = phase->transform(mem);
3263     adr = phase->transform(new AddPNode(base,adr,off));
3264     mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3265   }
3266   return mem;
3267 }
3268 
3269 //----------------------------step_through----------------------------------
3270 // Return allocation input memory edge if it is different instance
3271 // or itself if it is the one we are looking for.
3272 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3273   Node* n = *np;
3274   assert(n->is_ClearArray(), "sanity");
3275   intptr_t offset;
3276   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3277   // This method is called only before Allocate nodes are expanded
3278   // during macro nodes expansion. Before that ClearArray nodes are
3279   // only generated in PhaseMacroExpand::generate_arraycopy() (before
3280   // Allocate nodes are expanded) which follows allocations.
3281   assert(alloc != nullptr, "should have allocation");
3282   if (alloc->_idx == instance_id) {
3283     // Can not bypass initialization of the instance we are looking for.
3284     return false;
3285   }
3286   // Otherwise skip it.
3287   InitializeNode* init = alloc->initialization();
3288   if (init != nullptr)
3289     *np = init->in(TypeFunc::Memory);
3290   else
3291     *np = alloc->in(TypeFunc::Memory);
3292   return true;
3293 }
3294 
3295 //----------------------------clear_memory-------------------------------------
3296 // Generate code to initialize object storage to zero.
3297 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3298                                    Node* val,
3299                                    Node* raw_val,
3300                                    intptr_t start_offset,
3301                                    Node* end_offset,
3302                                    PhaseGVN* phase) {
3303   intptr_t offset = start_offset;
3304 
3305   int unit = BytesPerLong;
3306   if ((offset % unit) != 0) {
3307     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3308     adr = phase->transform(adr);
3309     const TypePtr* atp = TypeRawPtr::BOTTOM;
3310     if (val != nullptr) {
3311       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3312       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3313     } else {
3314       assert(raw_val == nullptr, "val may not be null");
3315       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3316     }
3317     mem = phase->transform(mem);
3318     offset += BytesPerInt;
3319   }
3320   assert((offset % unit) == 0, "");
3321 
3322   // Initialize the remaining stuff, if any, with a ClearArray.
3323   return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3324 }
3325 
3326 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3327                                    Node* raw_val,
3328                                    Node* start_offset,
3329                                    Node* end_offset,
3330                                    PhaseGVN* phase) {
3331   if (start_offset == end_offset) {
3332     // nothing to do
3333     return mem;
3334   }
3335 
3336   int unit = BytesPerLong;
3337   Node* zbase = start_offset;
3338   Node* zend  = end_offset;
3339 
3340   // Scale to the unit required by the CPU:
3341   if (!Matcher::init_array_count_is_in_bytes) {
3342     Node* shift = phase->intcon(exact_log2(unit));
3343     zbase = phase->transform(new URShiftXNode(zbase, shift) );
3344     zend  = phase->transform(new URShiftXNode(zend,  shift) );
3345   }
3346 
3347   // Bulk clear double-words
3348   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3349   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3350   if (raw_val == nullptr) {
3351     raw_val = phase->MakeConX(0);
3352   }
3353   mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3354   return phase->transform(mem);
3355 }
3356 
3357 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3358                                    Node* val,
3359                                    Node* raw_val,
3360                                    intptr_t start_offset,
3361                                    intptr_t end_offset,
3362                                    PhaseGVN* phase) {
3363   if (start_offset == end_offset) {
3364     // nothing to do
3365     return mem;
3366   }
3367 
3368   assert((end_offset % BytesPerInt) == 0, "odd end offset");
3369   intptr_t done_offset = end_offset;
3370   if ((done_offset % BytesPerLong) != 0) {
3371     done_offset -= BytesPerInt;
3372   }
3373   if (done_offset > start_offset) {
3374     mem = clear_memory(ctl, mem, dest, val, raw_val,
3375                        start_offset, phase->MakeConX(done_offset), phase);
3376   }
3377   if (done_offset < end_offset) { // emit the final 32-bit store
3378     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3379     adr = phase->transform(adr);
3380     const TypePtr* atp = TypeRawPtr::BOTTOM;
3381     if (val != nullptr) {
3382       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3383       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3384     } else {
3385       assert(raw_val == nullptr, "val may not be null");
3386       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3387     }
3388     mem = phase->transform(mem);
3389     done_offset += BytesPerInt;
3390   }
3391   assert(done_offset == end_offset, "");
3392   return mem;
3393 }
3394 
3395 //=============================================================================
3396 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3397   : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3398     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3399 #ifdef ASSERT
3400   , _pair_idx(0)
3401 #endif
3402 {
3403   init_class_id(Class_MemBar);
3404   Node* top = C->top();
3405   init_req(TypeFunc::I_O,top);
3406   init_req(TypeFunc::FramePtr,top);
3407   init_req(TypeFunc::ReturnAdr,top);

3512       PhaseIterGVN* igvn = phase->is_IterGVN();
3513       remove(igvn);
3514       // Must return either the original node (now dead) or a new node
3515       // (Do not return a top here, since that would break the uniqueness of top.)
3516       return new ConINode(TypeInt::ZERO);
3517     }
3518   }
3519   return progress ? this : nullptr;
3520 }
3521 
3522 //------------------------------Value------------------------------------------
3523 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3524   if( !in(0) ) return Type::TOP;
3525   if( phase->type(in(0)) == Type::TOP )
3526     return Type::TOP;
3527   return TypeTuple::MEMBAR;
3528 }
3529 
3530 //------------------------------match------------------------------------------
3531 // Construct projections for memory.
3532 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
3533   switch (proj->_con) {
3534   case TypeFunc::Control:
3535   case TypeFunc::Memory:
3536     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3537   }
3538   ShouldNotReachHere();
3539   return nullptr;
3540 }
3541 
3542 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3543   trailing->_kind = TrailingStore;
3544   leading->_kind = LeadingStore;
3545 #ifdef ASSERT
3546   trailing->_pair_idx = leading->_idx;
3547   leading->_pair_idx = leading->_idx;
3548 #endif
3549 }
3550 
3551 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3552   trailing->_kind = TrailingLoadStore;

3799   return (req() > RawStores);
3800 }
3801 
3802 void InitializeNode::set_complete(PhaseGVN* phase) {
3803   assert(!is_complete(), "caller responsibility");
3804   _is_complete = Complete;
3805 
3806   // After this node is complete, it contains a bunch of
3807   // raw-memory initializations.  There is no need for
3808   // it to have anything to do with non-raw memory effects.
3809   // Therefore, tell all non-raw users to re-optimize themselves,
3810   // after skipping the memory effects of this initialization.
3811   PhaseIterGVN* igvn = phase->is_IterGVN();
3812   if (igvn)  igvn->add_users_to_worklist(this);
3813 }
3814 
3815 // convenience function
3816 // return false if the init contains any stores already
3817 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3818   InitializeNode* init = initialization();
3819   if (init == nullptr || init->is_complete()) {
3820     return false;
3821   }
3822   init->remove_extra_zeroes();
3823   // for now, if this allocation has already collected any inits, bail:
3824   if (init->is_non_zero())  return false;
3825   init->set_complete(phase);
3826   return true;
3827 }
3828 
3829 void InitializeNode::remove_extra_zeroes() {
3830   if (req() == RawStores)  return;
3831   Node* zmem = zero_memory();
3832   uint fill = RawStores;
3833   for (uint i = fill; i < req(); i++) {
3834     Node* n = in(i);
3835     if (n->is_top() || n == zmem)  continue;  // skip
3836     if (fill < i)  set_req(fill, n);          // compact
3837     ++fill;
3838   }
3839   // delete any empty spaces created:
3840   while (fill < req()) {
3841     del_req(fill);

3979             // store node that we'd like to capture. We need to check
3980             // the uses of the MergeMemNode.
3981             mems.push(n);
3982           }
3983         } else if (n->is_Mem()) {
3984           Node* other_adr = n->in(MemNode::Address);
3985           if (other_adr == adr) {
3986             failed = true;
3987             break;
3988           } else {
3989             const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3990             if (other_t_adr != nullptr) {
3991               int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3992               if (other_alias_idx == alias_idx) {
3993                 // A load from the same memory slice as the store right
3994                 // after the InitializeNode. We check the control of the
3995                 // object/array that is loaded from. If it's the same as
3996                 // the store control then we cannot capture the store.
3997                 assert(!n->is_Store(), "2 stores to same slice on same control?");
3998                 Node* base = other_adr;
3999                 if (base->is_Phi()) {
4000                   // In rare case, base may be a PhiNode and it may read
4001                   // the same memory slice between InitializeNode and store.
4002                   failed = true;
4003                   break;
4004                 }
4005                 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4006                 base = base->in(AddPNode::Base);
4007                 if (base != nullptr) {
4008                   base = base->uncast();
4009                   if (base->is_Proj() && base->in(0) == alloc) {
4010                     failed = true;
4011                     break;
4012                   }
4013                 }
4014               }
4015             }
4016           }
4017         } else {
4018           failed = true;
4019           break;
4020         }
4021       }
4022     }
4023   }
4024   if (failed) {

4571         //   z's_done      12  16  16  16    12  16    12
4572         //   z's_needed    12  16  16  16    16  16    16
4573         //   zsize          0   0   0   0     4   0     4
4574         if (next_full_store < 0) {
4575           // Conservative tack:  Zero to end of current word.
4576           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4577         } else {
4578           // Zero to beginning of next fully initialized word.
4579           // Or, don't zero at all, if we are already in that word.
4580           assert(next_full_store >= zeroes_needed, "must go forward");
4581           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4582           zeroes_needed = next_full_store;
4583         }
4584       }
4585 
4586       if (zeroes_needed > zeroes_done) {
4587         intptr_t zsize = zeroes_needed - zeroes_done;
4588         // Do some incremental zeroing on rawmem, in parallel with inits.
4589         zeroes_done = align_down(zeroes_done, BytesPerInt);
4590         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4591                                               allocation()->in(AllocateNode::DefaultValue),
4592                                               allocation()->in(AllocateNode::RawDefaultValue),
4593                                               zeroes_done, zeroes_needed,
4594                                               phase);
4595         zeroes_done = zeroes_needed;
4596         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4597           do_zeroing = false;   // leave the hole, next time
4598       }
4599     }
4600 
4601     // Collect the store and move on:
4602     phase->replace_input_of(st, MemNode::Memory, inits);
4603     inits = st;                 // put it on the linearized chain
4604     set_req(i, zmem);           // unhook from previous position
4605 
4606     if (zeroes_done == st_off)
4607       zeroes_done = next_init_off;
4608 
4609     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4610 
4611     #ifdef ASSERT
4612     // Various order invariants.  Weaker than stores_are_sane because

4632   remove_extra_zeroes();        // clear out all the zmems left over
4633   add_req(inits);
4634 
4635   if (!(UseTLAB && ZeroTLAB)) {
4636     // If anything remains to be zeroed, zero it all now.
4637     zeroes_done = align_down(zeroes_done, BytesPerInt);
4638     // if it is the last unused 4 bytes of an instance, forget about it
4639     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4640     if (zeroes_done + BytesPerLong >= size_limit) {
4641       AllocateNode* alloc = allocation();
4642       assert(alloc != nullptr, "must be present");
4643       if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
4644         Node* klass_node = alloc->in(AllocateNode::KlassNode);
4645         ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4646         if (zeroes_done == k->layout_helper())
4647           zeroes_done = size_limit;
4648       }
4649     }
4650     if (zeroes_done < size_limit) {
4651       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4652                                             allocation()->in(AllocateNode::DefaultValue),
4653                                             allocation()->in(AllocateNode::RawDefaultValue),
4654                                             zeroes_done, size_in_bytes, phase);
4655     }
4656   }
4657 
4658   set_complete(phase);
4659   return rawmem;
4660 }
4661 
4662 
4663 #ifdef ASSERT
4664 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
4665   if (is_complete())
4666     return true;                // stores could be anything at this point
4667   assert(allocation() != nullptr, "must be present");
4668   intptr_t last_off = allocation()->minimum_header_size();
4669   for (uint i = InitializeNode::RawStores; i < req(); i++) {
4670     Node* st = in(i);
4671     intptr_t st_off = get_store_offset(st, phase);
4672     if (st_off < 0)  continue;  // ignore dead garbage
4673     if (last_off > st_off) {
< prev index next >