< prev index next >

src/hotspot/share/opto/memnode.cpp

Print this page

   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 

  26 #include "classfile/javaClasses.hpp"

  27 #include "compiler/compileLog.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/c2/barrierSetC2.hpp"
  30 #include "gc/shared/tlab_globals.hpp"
  31 #include "memory/allocation.inline.hpp"
  32 #include "memory/resourceArea.hpp"
  33 #include "oops/objArrayKlass.hpp"
  34 #include "opto/addnode.hpp"
  35 #include "opto/arraycopynode.hpp"
  36 #include "opto/cfgnode.hpp"
  37 #include "opto/regalloc.hpp"
  38 #include "opto/compile.hpp"
  39 #include "opto/connode.hpp"
  40 #include "opto/convertnode.hpp"

  41 #include "opto/loopnode.hpp"
  42 #include "opto/machnode.hpp"
  43 #include "opto/matcher.hpp"
  44 #include "opto/memnode.hpp"
  45 #include "opto/mempointer.hpp"
  46 #include "opto/mulnode.hpp"
  47 #include "opto/narrowptrnode.hpp"
  48 #include "opto/phaseX.hpp"
  49 #include "opto/regmask.hpp"
  50 #include "opto/rootnode.hpp"
  51 #include "opto/traceMergeStoresTag.hpp"
  52 #include "opto/vectornode.hpp"
  53 #include "utilities/align.hpp"
  54 #include "utilities/copy.hpp"
  55 #include "utilities/macros.hpp"
  56 #include "utilities/powerOfTwo.hpp"
  57 #include "utilities/vmError.hpp"
  58 
  59 // Portions of code courtesy of Clifford Click
  60 

 216   bool is_instance = t_oop->is_known_instance_field();
 217   PhaseIterGVN *igvn = phase->is_IterGVN();
 218   if (is_instance && igvn != nullptr && result->is_Phi()) {
 219     PhiNode *mphi = result->as_Phi();
 220     assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
 221     const TypePtr *t = mphi->adr_type();
 222     bool do_split = false;
 223     // In the following cases, Load memory input can be further optimized based on
 224     // its precise address type
 225     if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
 226       do_split = true;
 227     } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
 228       const TypeOopPtr* mem_t =
 229         t->is_oopptr()->cast_to_exactness(true)
 230         ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
 231         ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
 232       if (t_oop->isa_aryptr()) {
 233         mem_t = mem_t->is_aryptr()
 234                      ->cast_to_stable(t_oop->is_aryptr()->is_stable())
 235                      ->cast_to_size(t_oop->is_aryptr()->size())


 236                      ->with_offset(t_oop->is_aryptr()->offset())
 237                      ->is_aryptr();
 238       }
 239       do_split = mem_t == t_oop;
 240     }
 241     if (do_split) {
 242       // clone the Phi with our address type
 243       result = mphi->split_out_instance(t_adr, igvn);
 244     } else {
 245       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 246     }
 247   }
 248   return result;
 249 }
 250 
 251 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 252   uint alias_idx = phase->C->get_alias_index(tp);
 253   Node *mem = mmem;
 254 #ifdef ASSERT
 255   {
 256     // Check that current type is consistent with the alias index used during graph construction
 257     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 258     bool consistent =  adr_check == nullptr || adr_check->empty() ||
 259                        phase->C->must_alias(adr_check, alias_idx );
 260     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 261     if( !consistent && adr_check != nullptr && !adr_check->empty() &&
 262                tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 263         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 264         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 265           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 266           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 267       // don't assert if it is dead code.
 268       consistent = true;
 269     }
 270     if( !consistent ) {
 271       st->print("alias_idx==%d, adr_check==", alias_idx);
 272       if( adr_check == nullptr ) {
 273         st->print("null");
 274       } else {
 275         adr_check->dump();
 276       }
 277       st->cr();
 278       print_alias_types();
 279       assert(consistent, "adr_check must match alias idx");
 280     }
 281   }
 282 #endif

 995     Node* ld = gvn.transform(load);
 996     return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
 997   }
 998 
 999   return load;
1000 }
1001 
1002 //------------------------------hash-------------------------------------------
1003 uint LoadNode::hash() const {
1004   // unroll addition of interesting fields
1005   return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1006 }
1007 
1008 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1009   if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1010     bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1011     bool is_stable_ary = FoldStableValues &&
1012                          (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1013                          tp->isa_aryptr()->is_stable();
1014 
1015     return (eliminate_boxing && non_volatile) || is_stable_ary;
1016   }
1017 
1018   return false;
1019 }
1020 
1021 LoadNode* LoadNode::pin_array_access_node() const {
1022   const TypePtr* adr_type = this->adr_type();
1023   if (adr_type != nullptr && adr_type->isa_aryptr()) {
1024     return clone_pinned();
1025   }
1026   return nullptr;
1027 }
1028 
1029 // Is the value loaded previously stored by an arraycopy? If so return
1030 // a load node that reads from the source array so we may be able to
1031 // optimize out the ArrayCopy node later.
1032 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1033   Node* ld_adr = in(MemNode::Address);
1034   intptr_t ld_off = 0;
1035   AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);

1052       assert(ld_alloc != nullptr, "need an alloc");
1053       assert(addp->is_AddP(), "address must be addp");
1054       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1055       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1056       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1057       addp->set_req(AddPNode::Base, src);
1058       addp->set_req(AddPNode::Address, src);
1059     } else {
1060       assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1061              ac->as_ArrayCopy()->is_copyof_validated() ||
1062              ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1063       assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1064       addp->set_req(AddPNode::Base, src);
1065       addp->set_req(AddPNode::Address, src);
1066 
1067       const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1068       BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1069       if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1070 
1071       uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1072       uint shift  = exact_log2(type2aelembytes(ary_elem));
1073 
1074       Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1075 #ifdef _LP64
1076       diff = phase->transform(new ConvI2LNode(diff));
1077 #endif
1078       diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1079 
1080       Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1081       addp->set_req(AddPNode::Offset, offset);
1082     }
1083     addp = phase->transform(addp);
1084 #ifdef ASSERT
1085     const TypePtr* adr_type = phase->type(addp)->is_ptr();
1086     ld->_adr_type = adr_type;
1087 #endif
1088     ld->set_req(MemNode::Address, addp);
1089     ld->set_req(0, ctl);
1090     ld->set_req(MemNode::Memory, mem);
1091     return ld;
1092   }

1171         // Same base, same offset.
1172         // Possible improvement for arrays: check index value instead of absolute offset.
1173 
1174         // At this point we have proven something like this setup:
1175         //   B = << base >>
1176         //   L =  LoadQ(AddP(Check/CastPP(B), #Off))
1177         //   S = StoreQ(AddP(             B , #Off), V)
1178         // (Actually, we haven't yet proven the Q's are the same.)
1179         // In other words, we are loading from a casted version of
1180         // the same pointer-and-offset that we stored to.
1181         // Casted version may carry a dependency and it is respected.
1182         // Thus, we are able to replace L by V.
1183       }
1184       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1185       if (store_Opcode() != st->Opcode()) {
1186         return nullptr;
1187       }
1188       // LoadVector/StoreVector needs additional check to ensure the types match.
1189       if (st->is_StoreVector()) {
1190         const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
1191         const TypeVect* out_vt = as_LoadVector()->vect_type();
1192         if (in_vt != out_vt) {
1193           return nullptr;
1194         }
1195       }
1196       return st->in(MemNode::ValueIn);
1197     }
1198 
1199     // A load from a freshly-created object always returns zero.
1200     // (This can happen after LoadNode::Ideal resets the load's memory input
1201     // to find_captured_store, which returned InitializeNode::zero_memory.)
1202     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1203         (st->in(0) == ld_alloc) &&
1204         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1205       // return a zero value for the load's basic type
1206       // (This is one of the few places where a generic PhaseTransform
1207       // can create new nodes.  Think of it as lazily manifesting
1208       // virtually pre-existing constants.)





1209       if (memory_type() != T_VOID) {
1210         if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1211           // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1212           // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1213           // by the ArrayCopyNode.
1214           return phase->zerocon(memory_type());
1215         }
1216       } else {
1217         // TODO: materialize all-zero vector constant
1218         assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1219       }
1220     }
1221 
1222     // A load from an initialization barrier can match a captured store.
1223     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1224       InitializeNode* init = st->in(0)->as_Initialize();
1225       AllocateNode* alloc = init->allocation();
1226       if ((alloc != nullptr) && (alloc == ld_alloc)) {
1227         // examine a captured store value
1228         st = init->find_captured_store(ld_off, memory_size(), phase);

1256 //----------------------is_instance_field_load_with_local_phi------------------
1257 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1258   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1259       in(Address)->is_AddP() ) {
1260     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1261     // Only instances and boxed values.
1262     if( t_oop != nullptr &&
1263         (t_oop->is_ptr_to_boxed_value() ||
1264          t_oop->is_known_instance_field()) &&
1265         t_oop->offset() != Type::OffsetBot &&
1266         t_oop->offset() != Type::OffsetTop) {
1267       return true;
1268     }
1269   }
1270   return false;
1271 }
1272 
1273 //------------------------------Identity---------------------------------------
1274 // Loads are identity if previous store is to same address
1275 Node* LoadNode::Identity(PhaseGVN* phase) {

















1276   // If the previous store-maker is the right kind of Store, and the store is
1277   // to the same address, then we are equal to the value stored.
1278   Node* mem = in(Memory);
1279   Node* value = can_see_stored_value(mem, phase);
1280   if( value ) {
1281     // byte, short & char stores truncate naturally.
1282     // A load has to load the truncated value which requires
1283     // some sort of masking operation and that requires an
1284     // Ideal call instead of an Identity call.
1285     if (memory_size() < BytesPerInt) {
1286       // If the input to the store does not fit with the load's result type,
1287       // it must be truncated via an Ideal call.
1288       if (!phase->type(value)->higher_equal(phase->type(this)))
1289         return this;
1290     }
1291     // (This works even when value is a Con, but LoadNode::Value
1292     // usually runs first, producing the singleton type of the Con.)
1293     if (!has_pinned_control_dependency() || value->is_Con()) {
1294       return value;
1295     } else {

1849   bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1850          phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1851 
1852   // Skip up past a SafePoint control.  Cannot do this for Stores because
1853   // pointer stores & cardmarks must stay on the same side of a SafePoint.
1854   if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1855       phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw  &&
1856       !addr_mark &&
1857       (depends_only_on_test() || has_unknown_control_dependency())) {
1858     ctrl = ctrl->in(0);
1859     set_req(MemNode::Control,ctrl);
1860     progress = true;
1861   }
1862 
1863   intptr_t ignore = 0;
1864   Node*    base   = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1865   if (base != nullptr
1866       && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1867     // Check for useless control edge in some common special cases
1868     if (in(MemNode::Control) != nullptr

1869         && can_remove_control()
1870         && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1871         && all_controls_dominate(base, phase->C->start())) {
1872       // A method-invariant, non-null address (constant or 'this' argument).
1873       set_req(MemNode::Control, nullptr);
1874       progress = true;
1875     }
1876   }
1877 
1878   Node* mem = in(MemNode::Memory);
1879   const TypePtr *addr_t = phase->type(address)->isa_ptr();
1880 
1881   if (can_reshape && (addr_t != nullptr)) {
1882     // try to optimize our memory input
1883     Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1884     if (opt_mem != mem) {
1885       set_req_X(MemNode::Memory, opt_mem, phase);
1886       if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1887       return this;
1888     }

2053       }
2054     }
2055 
2056     // Don't do this for integer types. There is only potential profit if
2057     // the element type t is lower than _type; that is, for int types, if _type is
2058     // more restrictive than t.  This only happens here if one is short and the other
2059     // char (both 16 bits), and in those cases we've made an intentional decision
2060     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2061     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2062     //
2063     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2064     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
2065     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2066     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
2067     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2068     // In fact, that could have been the original type of p1, and p1 could have
2069     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2070     // expression (LShiftL quux 3) independently optimized to the constant 8.
2071     if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2072         && (_type->isa_vect() == nullptr)

2073         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2074       // t might actually be lower than _type, if _type is a unique
2075       // concrete subclass of abstract class t.
2076       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
2077         const Type* jt = t->join_speculative(_type);
2078         // In any case, do not allow the join, per se, to empty out the type.
2079         if (jt->empty() && !t->empty()) {
2080           // This can happen if a interface-typed array narrows to a class type.
2081           jt = _type;
2082         }
2083 #ifdef ASSERT
2084         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2085           // The pointers in the autobox arrays are always non-null
2086           Node* base = adr->in(AddPNode::Base);
2087           if ((base != nullptr) && base->is_DecodeN()) {
2088             // Get LoadN node which loads IntegerCache.cache field
2089             base = base->in(1);
2090           }
2091           if ((base != nullptr) && base->is_Con()) {
2092             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2093             if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2094               // It could be narrow oop
2095               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2096             }
2097           }
2098         }
2099 #endif
2100         return jt;
2101       }
2102     }
2103   } else if (tp->base() == Type::InstPtr) {
2104     assert( off != Type::OffsetBot ||
2105             // arrays can be cast to Objects
2106             !tp->isa_instptr() ||
2107             tp->is_instptr()->instance_klass()->is_java_lang_Object() ||


2108             // unsafe field access may not have a constant offset
2109             C->has_unsafe_access(),
2110             "Field accesses must be precise" );
2111     // For oop loads, we expect the _type to be precise.
2112 
2113     // Optimize loads from constant fields.
2114     const TypeInstPtr* tinst = tp->is_instptr();



2115     ciObject* const_oop = tinst->const_oop();
2116     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2117       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
2118       if (con_type != nullptr) {
2119         return con_type;
2120       }
2121     }
2122   } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2123     assert(off != Type::OffsetBot ||
2124             !tp->isa_instklassptr() ||
2125            // arrays can be cast to Objects
2126            tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2127            // also allow array-loading from the primary supertype
2128            // array during subtype checks
2129            Opcode() == Op_LoadKlass,
2130            "Field accesses must be precise");
2131     // For klass/static loads, we expect the _type to be precise
2132   } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2133     /* With mirrors being an indirect in the Klass*
2134      * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2135      * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2136      *
2137      * So check the type and klass of the node before the LoadP.

2144         assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2145         assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2146         return TypeInstPtr::make(klass->java_mirror());
2147       }
2148     }
2149   }
2150 
2151   const TypeKlassPtr *tkls = tp->isa_klassptr();
2152   if (tkls != nullptr) {
2153     if (tkls->is_loaded() && tkls->klass_is_exact()) {
2154       ciKlass* klass = tkls->exact_klass();
2155       // We are loading a field from a Klass metaobject whose identity
2156       // is known at compile time (the type is "exact" or "precise").
2157       // Check for fields we know are maintained as constants by the VM.
2158       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2159         // The field is Klass::_super_check_offset.  Return its (constant) value.
2160         // (Folds up type checking code.)
2161         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2162         return TypeInt::make(klass->super_check_offset());
2163       }
2164       if (UseCompactObjectHeaders) {
2165         if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2166           // The field is Klass::_prototype_header. Return its (constant) value.
2167           assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2168           return TypeX::make(klass->prototype_header());
2169         }
2170       }
2171       // Compute index into primary_supers array
2172       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2173       // Check for overflowing; use unsigned compare to handle the negative case.
2174       if( depth < ciKlass::primary_super_limit() ) {
2175         // The field is an element of Klass::_primary_supers.  Return its (constant) value.
2176         // (Folds up type checking code.)
2177         assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2178         ciKlass *ss = klass->super_of_depth(depth);
2179         return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2180       }
2181       const Type* aift = load_array_final_field(tkls, klass);
2182       if (aift != nullptr)  return aift;
2183     }
2184 

2225       jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2226       // The key property of this type is that it folds up tests
2227       // for array-ness, since it proves that the layout_helper is positive.
2228       // Thus, a generic value like the basic object layout helper works fine.
2229       return TypeInt::make(min_size, max_jint, Type::WidenMin);
2230     }
2231   }
2232 
2233   bool is_vect = (_type->isa_vect() != nullptr);
2234   if (is_instance && !is_vect) {
2235     // If we have an instance type and our memory input is the
2236     // programs's initial memory state, there is no matching store,
2237     // so just return a zero of the appropriate type -
2238     // except if it is vectorized - then we have no zero constant.
2239     Node *mem = in(MemNode::Memory);
2240     if (mem->is_Parm() && mem->in(0)->is_Start()) {
2241       assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2242       return Type::get_zero_type(_type->basic_type());
2243     }
2244   }
2245 
2246   if (!UseCompactObjectHeaders) {
2247     Node* alloc = is_new_object_mark_load();
2248     if (alloc != nullptr) {
2249       return TypeX::make(markWord::prototype().value());









2250     }
2251   }
2252 
2253   return _type;
2254 }
2255 
2256 //------------------------------match_edge-------------------------------------
2257 // Do we Match on this edge index or not?  Match only the address.
2258 uint LoadNode::match_edge(uint idx) const {
2259   return idx == MemNode::Address;
2260 }
2261 
2262 //--------------------------LoadBNode::Ideal--------------------------------------
2263 //
2264 //  If the previous store is to the same address as this load,
2265 //  and the value stored was larger than a byte, replace this load
2266 //  with the value stored truncated to a byte.  If no truncation is
2267 //  needed, the replacement is done in LoadNode::Identity().
2268 //
2269 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {

2381   return LoadNode::Ideal(phase, can_reshape);
2382 }
2383 
2384 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2385   Node* mem = in(MemNode::Memory);
2386   Node* value = can_see_stored_value(mem,phase);
2387   if (value != nullptr && value->is_Con() &&
2388       !value->bottom_type()->higher_equal(_type)) {
2389     // If the input to the store does not fit with the load's result type,
2390     // it must be truncated. We can't delay until Ideal call since
2391     // a singleton Value is needed for split_thru_phi optimization.
2392     int con = value->get_int();
2393     return TypeInt::make((con << 16) >> 16);
2394   }
2395   return LoadNode::Value(phase);
2396 }
2397 
2398 //=============================================================================
2399 //----------------------------LoadKlassNode::make------------------------------
2400 // Polymorphic factory method:
2401 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {

2402   // sanity check the alias category against the created node type
2403   const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2404   assert(adr_type != nullptr, "expecting TypeKlassPtr");
2405 #ifdef _LP64
2406   if (adr_type->is_ptr_to_narrowklass()) {
2407     assert(UseCompressedClassPointers, "no compressed klasses");
2408     Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2409     return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2410   }
2411 #endif
2412   assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2413   return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2414 }
2415 
2416 //------------------------------Value------------------------------------------
2417 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2418   return klass_value_common(phase);
2419 }
2420 
2421 // In most cases, LoadKlassNode does not have the control input set. If the control

2428   // Either input is TOP ==> the result is TOP
2429   const Type *t1 = phase->type( in(MemNode::Memory) );
2430   if (t1 == Type::TOP)  return Type::TOP;
2431   Node *adr = in(MemNode::Address);
2432   const Type *t2 = phase->type( adr );
2433   if (t2 == Type::TOP)  return Type::TOP;
2434   const TypePtr *tp = t2->is_ptr();
2435   if (TypePtr::above_centerline(tp->ptr()) ||
2436       tp->ptr() == TypePtr::Null)  return Type::TOP;
2437 
2438   // Return a more precise klass, if possible
2439   const TypeInstPtr *tinst = tp->isa_instptr();
2440   if (tinst != nullptr) {
2441     ciInstanceKlass* ik = tinst->instance_klass();
2442     int offset = tinst->offset();
2443     if (ik == phase->C->env()->Class_klass()
2444         && (offset == java_lang_Class::klass_offset() ||
2445             offset == java_lang_Class::array_klass_offset())) {
2446       // We are loading a special hidden field from a Class mirror object,
2447       // the field which points to the VM's Klass metaobject.
2448       ciType* t = tinst->java_mirror_type();

2449       // java_mirror_type returns non-null for compile-time Class constants.
2450       if (t != nullptr) {
2451         // constant oop => constant klass
2452         if (offset == java_lang_Class::array_klass_offset()) {
2453           if (t->is_void()) {
2454             // We cannot create a void array.  Since void is a primitive type return null
2455             // klass.  Users of this result need to do a null check on the returned klass.
2456             return TypePtr::NULL_PTR;
2457           }
2458           return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);




2459         }
2460         if (!t->is_klass()) {
2461           // a primitive Class (e.g., int.class) has null for a klass field
2462           return TypePtr::NULL_PTR;
2463         }
2464         // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2465         return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);




2466       }
2467       // non-constant mirror, so we can't tell what's going on
2468     }
2469     if (!tinst->is_loaded())
2470       return _type;             // Bail out if not loaded
2471     if (offset == oopDesc::klass_offset_in_bytes()) {
2472       return tinst->as_klass_type(true);
2473     }
2474   }
2475 
2476   // Check for loading klass from an array
2477   const TypeAryPtr *tary = tp->isa_aryptr();
2478   if (tary != nullptr &&
2479       tary->offset() == oopDesc::klass_offset_in_bytes()) {
2480     return tary->as_klass_type(true);
2481   }
2482 
2483   // Check for loading klass from an array klass
2484   const TypeKlassPtr *tkls = tp->isa_klassptr();
2485   if (tkls != nullptr && !StressReflectiveCode) {
2486     if (!tkls->is_loaded())
2487      return _type;             // Bail out if not loaded
2488     if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2489         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2490       // // Always returning precise element type is incorrect,
2491       // // e.g., element type could be object and array may contain strings
2492       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2493 
2494       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2495       // according to the element type's subclassing.
2496       return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2497     }

3294   }
3295   ss.print_cr("[TraceMergeStores]: with");
3296   merged_input_value->dump("\n", false, &ss);
3297   merged_store->dump("\n", false, &ss);
3298   tty->print("%s", ss.as_string());
3299 }
3300 #endif
3301 
3302 //------------------------------Ideal------------------------------------------
3303 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3304 // When a store immediately follows a relevant allocation/initialization,
3305 // try to capture it into the initialization, or hoist it above.
3306 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3307   Node* p = MemNode::Ideal_common(phase, can_reshape);
3308   if (p)  return (p == NodeSentinel) ? nullptr : p;
3309 
3310   Node* mem     = in(MemNode::Memory);
3311   Node* address = in(MemNode::Address);
3312   Node* value   = in(MemNode::ValueIn);
3313   // Back-to-back stores to same address?  Fold em up.  Generally
3314   // unsafe if I have intervening uses.
3315   {
3316     Node* st = mem;
3317     // If Store 'st' has more than one use, we cannot fold 'st' away.
3318     // For example, 'st' might be the final state at a conditional
3319     // return.  Or, 'st' might be used by some node which is live at
3320     // the same time 'st' is live, which might be unschedulable.  So,
3321     // require exactly ONE user until such time as we clone 'mem' for
3322     // each of 'mem's uses (thus making the exactly-1-user-rule hold
3323     // true).
3324     while (st->is_Store() && st->outcnt() == 1) {
3325       // Looking at a dead closed cycle of memory?
3326       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3327       assert(Opcode() == st->Opcode() ||
3328              st->Opcode() == Op_StoreVector ||
3329              Opcode() == Op_StoreVector ||
3330              st->Opcode() == Op_StoreVectorScatter ||
3331              Opcode() == Op_StoreVectorScatter ||
3332              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3333              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3334              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy


3335              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3336              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3337 
3338       if (st->in(MemNode::Address)->eqv_uncast(address) &&
3339           st->as_Store()->memory_size() <= this->memory_size()) {
3340         Node* use = st->raw_out(0);
3341         if (phase->is_IterGVN()) {
3342           phase->is_IterGVN()->rehash_node_delayed(use);
3343         }
3344         // It's OK to do this in the parser, since DU info is always accurate,
3345         // and the parser always refers to nodes via SafePointNode maps.
3346         use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3347         return this;
3348       }
3349       st = st->in(MemNode::Memory);
3350     }
3351   }
3352 
3353 
3354   // Capture an unaliased, unconditional, simple store into an initializer.

3441       const StoreVectorNode* store_vector = as_StoreVector();
3442       const StoreVectorNode* mem_vector = mem->as_StoreVector();
3443       const Node* store_indices = store_vector->indices();
3444       const Node* mem_indices = mem_vector->indices();
3445       const Node* store_mask = store_vector->mask();
3446       const Node* mem_mask = mem_vector->mask();
3447       // Ensure types, indices, and masks match
3448       if (store_vector->vect_type() == mem_vector->vect_type() &&
3449           ((store_indices == nullptr) == (mem_indices == nullptr) &&
3450            (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3451           ((store_mask == nullptr) == (mem_mask == nullptr) &&
3452            (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3453         result = mem;
3454       }
3455     }
3456   }
3457 
3458   // Store of zero anywhere into a freshly-allocated object?
3459   // Then the store is useless.
3460   // (It must already have been captured by the InitializeNode.)
3461   if (result == this &&
3462       ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3463     // a newly allocated object is already all-zeroes everywhere
3464     if (mem->is_Proj() && mem->in(0)->is_Allocate()) {

3465       result = mem;
3466     }
3467 
3468     if (result == this) {
3469       // the store may also apply to zero-bits in an earlier object
3470       Node* prev_mem = find_previous_store(phase);
3471       // Steps (a), (b):  Walk past independent stores to find an exact match.
3472       if (prev_mem != nullptr) {
3473         Node* prev_val = can_see_stored_value(prev_mem, phase);
3474         if (prev_val != nullptr && prev_val == val) {
3475           // prev_val and val might differ by a cast; it would be good
3476           // to keep the more informative of the two.
3477           result = mem;
3478         }
3479       }
3480     }
3481   }
3482 
3483   PhaseIterGVN* igvn = phase->is_IterGVN();
3484   if (result != this && igvn != nullptr) {
3485     MemBarNode* trailing = trailing_membar();
3486     if (trailing != nullptr) {
3487 #ifdef ASSERT
3488       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();

3764 // Clearing a short array is faster with stores
3765 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3766   // Already know this is a large node, do not try to ideal it
3767   if (_is_large) return nullptr;
3768 
3769   const int unit = BytesPerLong;
3770   const TypeX* t = phase->type(in(2))->isa_intptr_t();
3771   if (!t)  return nullptr;
3772   if (!t->is_con())  return nullptr;
3773   intptr_t raw_count = t->get_con();
3774   intptr_t size = raw_count;
3775   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3776   // Clearing nothing uses the Identity call.
3777   // Negative clears are possible on dead ClearArrays
3778   // (see jck test stmt114.stmt11402.val).
3779   if (size <= 0 || size % unit != 0)  return nullptr;
3780   intptr_t count = size / unit;
3781   // Length too long; communicate this to matchers and assemblers.
3782   // Assemblers are responsible to produce fast hardware clears for it.
3783   if (size > InitArrayShortSize) {
3784     return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3785   } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3786     return nullptr;
3787   }
3788   if (!IdealizeClearArrayNode) return nullptr;
3789   Node *mem = in(1);
3790   if( phase->type(mem)==Type::TOP ) return nullptr;
3791   Node *adr = in(3);
3792   const Type* at = phase->type(adr);
3793   if( at==Type::TOP ) return nullptr;
3794   const TypePtr* atp = at->isa_ptr();
3795   // adjust atp to be the correct array element address type
3796   if (atp == nullptr)  atp = TypePtr::BOTTOM;
3797   else              atp = atp->add_offset(Type::OffsetBot);
3798   // Get base for derived pointer purposes
3799   if( adr->Opcode() != Op_AddP ) Unimplemented();
3800   Node *base = adr->in(1);
3801 
3802   Node *zero = phase->makecon(TypeLong::ZERO);
3803   Node *off  = phase->MakeConX(BytesPerLong);
3804   mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3805   count--;
3806   while( count-- ) {
3807     mem = phase->transform(mem);
3808     adr = phase->transform(new AddPNode(base,adr,off));
3809     mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3810   }
3811   return mem;
3812 }
3813 
3814 //----------------------------step_through----------------------------------
3815 // Return allocation input memory edge if it is different instance
3816 // or itself if it is the one we are looking for.
3817 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3818   Node* n = *np;
3819   assert(n->is_ClearArray(), "sanity");
3820   intptr_t offset;
3821   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3822   // This method is called only before Allocate nodes are expanded
3823   // during macro nodes expansion. Before that ClearArray nodes are
3824   // only generated in PhaseMacroExpand::generate_arraycopy() (before
3825   // Allocate nodes are expanded) which follows allocations.
3826   assert(alloc != nullptr, "should have allocation");
3827   if (alloc->_idx == instance_id) {
3828     // Can not bypass initialization of the instance we are looking for.
3829     return false;
3830   }
3831   // Otherwise skip it.
3832   InitializeNode* init = alloc->initialization();
3833   if (init != nullptr)
3834     *np = init->in(TypeFunc::Memory);
3835   else
3836     *np = alloc->in(TypeFunc::Memory);
3837   return true;
3838 }
3839 
3840 //----------------------------clear_memory-------------------------------------
3841 // Generate code to initialize object storage to zero.
3842 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


3843                                    intptr_t start_offset,
3844                                    Node* end_offset,
3845                                    PhaseGVN* phase) {
3846   intptr_t offset = start_offset;
3847 
3848   int unit = BytesPerLong;
3849   if ((offset % unit) != 0) {
3850     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3851     adr = phase->transform(adr);
3852     const TypePtr* atp = TypeRawPtr::BOTTOM;
3853     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






3854     mem = phase->transform(mem);
3855     offset += BytesPerInt;
3856   }
3857   assert((offset % unit) == 0, "");
3858 
3859   // Initialize the remaining stuff, if any, with a ClearArray.
3860   return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
3861 }
3862 
3863 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,

3864                                    Node* start_offset,
3865                                    Node* end_offset,
3866                                    PhaseGVN* phase) {
3867   if (start_offset == end_offset) {
3868     // nothing to do
3869     return mem;
3870   }
3871 
3872   int unit = BytesPerLong;
3873   Node* zbase = start_offset;
3874   Node* zend  = end_offset;
3875 
3876   // Scale to the unit required by the CPU:
3877   if (!Matcher::init_array_count_is_in_bytes) {
3878     Node* shift = phase->intcon(exact_log2(unit));
3879     zbase = phase->transform(new URShiftXNode(zbase, shift) );
3880     zend  = phase->transform(new URShiftXNode(zend,  shift) );
3881   }
3882 
3883   // Bulk clear double-words
3884   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3885   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3886   mem = new ClearArrayNode(ctl, mem, zsize, adr, false);



3887   return phase->transform(mem);
3888 }
3889 
3890 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


3891                                    intptr_t start_offset,
3892                                    intptr_t end_offset,
3893                                    PhaseGVN* phase) {
3894   if (start_offset == end_offset) {
3895     // nothing to do
3896     return mem;
3897   }
3898 
3899   assert((end_offset % BytesPerInt) == 0, "odd end offset");
3900   intptr_t done_offset = end_offset;
3901   if ((done_offset % BytesPerLong) != 0) {
3902     done_offset -= BytesPerInt;
3903   }
3904   if (done_offset > start_offset) {
3905     mem = clear_memory(ctl, mem, dest,
3906                        start_offset, phase->MakeConX(done_offset), phase);
3907   }
3908   if (done_offset < end_offset) { // emit the final 32-bit store
3909     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3910     adr = phase->transform(adr);
3911     const TypePtr* atp = TypeRawPtr::BOTTOM;
3912     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






3913     mem = phase->transform(mem);
3914     done_offset += BytesPerInt;
3915   }
3916   assert(done_offset == end_offset, "");
3917   return mem;
3918 }
3919 
3920 //=============================================================================
3921 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3922   : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3923     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3924 #ifdef ASSERT
3925   , _pair_idx(0)
3926 #endif
3927 {
3928   init_class_id(Class_MemBar);
3929   Node* top = C->top();
3930   init_req(TypeFunc::I_O,top);
3931   init_req(TypeFunc::FramePtr,top);
3932   init_req(TypeFunc::ReturnAdr,top);

4038       PhaseIterGVN* igvn = phase->is_IterGVN();
4039       remove(igvn);
4040       // Must return either the original node (now dead) or a new node
4041       // (Do not return a top here, since that would break the uniqueness of top.)
4042       return new ConINode(TypeInt::ZERO);
4043     }
4044   }
4045   return progress ? this : nullptr;
4046 }
4047 
4048 //------------------------------Value------------------------------------------
4049 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4050   if( !in(0) ) return Type::TOP;
4051   if( phase->type(in(0)) == Type::TOP )
4052     return Type::TOP;
4053   return TypeTuple::MEMBAR;
4054 }
4055 
4056 //------------------------------match------------------------------------------
4057 // Construct projections for memory.
4058 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4059   switch (proj->_con) {
4060   case TypeFunc::Control:
4061   case TypeFunc::Memory:
4062     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4063   }
4064   ShouldNotReachHere();
4065   return nullptr;
4066 }
4067 
4068 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4069   trailing->_kind = TrailingStore;
4070   leading->_kind = LeadingStore;
4071 #ifdef ASSERT
4072   trailing->_pair_idx = leading->_idx;
4073   leading->_pair_idx = leading->_idx;
4074 #endif
4075 }
4076 
4077 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4078   trailing->_kind = TrailingLoadStore;

4325   return (req() > RawStores);
4326 }
4327 
4328 void InitializeNode::set_complete(PhaseGVN* phase) {
4329   assert(!is_complete(), "caller responsibility");
4330   _is_complete = Complete;
4331 
4332   // After this node is complete, it contains a bunch of
4333   // raw-memory initializations.  There is no need for
4334   // it to have anything to do with non-raw memory effects.
4335   // Therefore, tell all non-raw users to re-optimize themselves,
4336   // after skipping the memory effects of this initialization.
4337   PhaseIterGVN* igvn = phase->is_IterGVN();
4338   if (igvn)  igvn->add_users_to_worklist(this);
4339 }
4340 
4341 // convenience function
4342 // return false if the init contains any stores already
4343 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4344   InitializeNode* init = initialization();
4345   if (init == nullptr || init->is_complete())  return false;


4346   init->remove_extra_zeroes();
4347   // for now, if this allocation has already collected any inits, bail:
4348   if (init->is_non_zero())  return false;
4349   init->set_complete(phase);
4350   return true;
4351 }
4352 
4353 void InitializeNode::remove_extra_zeroes() {
4354   if (req() == RawStores)  return;
4355   Node* zmem = zero_memory();
4356   uint fill = RawStores;
4357   for (uint i = fill; i < req(); i++) {
4358     Node* n = in(i);
4359     if (n->is_top() || n == zmem)  continue;  // skip
4360     if (fill < i)  set_req(fill, n);          // compact
4361     ++fill;
4362   }
4363   // delete any empty spaces created:
4364   while (fill < req()) {
4365     del_req(fill);

4509             // store node that we'd like to capture. We need to check
4510             // the uses of the MergeMemNode.
4511             mems.push(n);
4512           }
4513         } else if (n->is_Mem()) {
4514           Node* other_adr = n->in(MemNode::Address);
4515           if (other_adr == adr) {
4516             failed = true;
4517             break;
4518           } else {
4519             const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4520             if (other_t_adr != nullptr) {
4521               int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4522               if (other_alias_idx == alias_idx) {
4523                 // A load from the same memory slice as the store right
4524                 // after the InitializeNode. We check the control of the
4525                 // object/array that is loaded from. If it's the same as
4526                 // the store control then we cannot capture the store.
4527                 assert(!n->is_Store(), "2 stores to same slice on same control?");
4528                 Node* base = other_adr;






4529                 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4530                 base = base->in(AddPNode::Base);
4531                 if (base != nullptr) {
4532                   base = base->uncast();
4533                   if (base->is_Proj() && base->in(0) == alloc) {
4534                     failed = true;
4535                     break;
4536                   }
4537                 }
4538               }
4539             }
4540           }
4541         } else {
4542           failed = true;
4543           break;
4544         }
4545       }
4546     }
4547   }
4548   if (failed) {

5095         //   z's_done      12  16  16  16    12  16    12
5096         //   z's_needed    12  16  16  16    16  16    16
5097         //   zsize          0   0   0   0     4   0     4
5098         if (next_full_store < 0) {
5099           // Conservative tack:  Zero to end of current word.
5100           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5101         } else {
5102           // Zero to beginning of next fully initialized word.
5103           // Or, don't zero at all, if we are already in that word.
5104           assert(next_full_store >= zeroes_needed, "must go forward");
5105           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5106           zeroes_needed = next_full_store;
5107         }
5108       }
5109 
5110       if (zeroes_needed > zeroes_done) {
5111         intptr_t zsize = zeroes_needed - zeroes_done;
5112         // Do some incremental zeroing on rawmem, in parallel with inits.
5113         zeroes_done = align_down(zeroes_done, BytesPerInt);
5114         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


5115                                               zeroes_done, zeroes_needed,
5116                                               phase);
5117         zeroes_done = zeroes_needed;
5118         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5119           do_zeroing = false;   // leave the hole, next time
5120       }
5121     }
5122 
5123     // Collect the store and move on:
5124     phase->replace_input_of(st, MemNode::Memory, inits);
5125     inits = st;                 // put it on the linearized chain
5126     set_req(i, zmem);           // unhook from previous position
5127 
5128     if (zeroes_done == st_off)
5129       zeroes_done = next_init_off;
5130 
5131     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5132 
5133     #ifdef ASSERT
5134     // Various order invariants.  Weaker than stores_are_sane because

5154   remove_extra_zeroes();        // clear out all the zmems left over
5155   add_req(inits);
5156 
5157   if (!(UseTLAB && ZeroTLAB)) {
5158     // If anything remains to be zeroed, zero it all now.
5159     zeroes_done = align_down(zeroes_done, BytesPerInt);
5160     // if it is the last unused 4 bytes of an instance, forget about it
5161     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5162     if (zeroes_done + BytesPerLong >= size_limit) {
5163       AllocateNode* alloc = allocation();
5164       assert(alloc != nullptr, "must be present");
5165       if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5166         Node* klass_node = alloc->in(AllocateNode::KlassNode);
5167         ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5168         if (zeroes_done == k->layout_helper())
5169           zeroes_done = size_limit;
5170       }
5171     }
5172     if (zeroes_done < size_limit) {
5173       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


5174                                             zeroes_done, size_in_bytes, phase);
5175     }
5176   }
5177 
5178   set_complete(phase);
5179   return rawmem;
5180 }
5181 
5182 
5183 #ifdef ASSERT
5184 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5185   if (is_complete())
5186     return true;                // stores could be anything at this point
5187   assert(allocation() != nullptr, "must be present");
5188   intptr_t last_off = allocation()->minimum_header_size();
5189   for (uint i = InitializeNode::RawStores; i < req(); i++) {
5190     Node* st = in(i);
5191     intptr_t st_off = get_store_offset(st, phase);
5192     if (st_off < 0)  continue;  // ignore dead garbage
5193     if (last_off > st_off) {

   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "ci/ciFlatArrayKlass.hpp"
  27 #include "classfile/javaClasses.hpp"
  28 #include "classfile/systemDictionary.hpp"
  29 #include "compiler/compileLog.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/c2/barrierSetC2.hpp"
  32 #include "gc/shared/tlab_globals.hpp"
  33 #include "memory/allocation.inline.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "oops/objArrayKlass.hpp"
  36 #include "opto/addnode.hpp"
  37 #include "opto/arraycopynode.hpp"
  38 #include "opto/cfgnode.hpp"
  39 #include "opto/regalloc.hpp"
  40 #include "opto/compile.hpp"
  41 #include "opto/connode.hpp"
  42 #include "opto/convertnode.hpp"
  43 #include "opto/inlinetypenode.hpp"
  44 #include "opto/loopnode.hpp"
  45 #include "opto/machnode.hpp"
  46 #include "opto/matcher.hpp"
  47 #include "opto/memnode.hpp"
  48 #include "opto/mempointer.hpp"
  49 #include "opto/mulnode.hpp"
  50 #include "opto/narrowptrnode.hpp"
  51 #include "opto/phaseX.hpp"
  52 #include "opto/regmask.hpp"
  53 #include "opto/rootnode.hpp"
  54 #include "opto/traceMergeStoresTag.hpp"
  55 #include "opto/vectornode.hpp"
  56 #include "utilities/align.hpp"
  57 #include "utilities/copy.hpp"
  58 #include "utilities/macros.hpp"
  59 #include "utilities/powerOfTwo.hpp"
  60 #include "utilities/vmError.hpp"
  61 
  62 // Portions of code courtesy of Clifford Click
  63 

 219   bool is_instance = t_oop->is_known_instance_field();
 220   PhaseIterGVN *igvn = phase->is_IterGVN();
 221   if (is_instance && igvn != nullptr && result->is_Phi()) {
 222     PhiNode *mphi = result->as_Phi();
 223     assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
 224     const TypePtr *t = mphi->adr_type();
 225     bool do_split = false;
 226     // In the following cases, Load memory input can be further optimized based on
 227     // its precise address type
 228     if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
 229       do_split = true;
 230     } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
 231       const TypeOopPtr* mem_t =
 232         t->is_oopptr()->cast_to_exactness(true)
 233         ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
 234         ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
 235       if (t_oop->isa_aryptr()) {
 236         mem_t = mem_t->is_aryptr()
 237                      ->cast_to_stable(t_oop->is_aryptr()->is_stable())
 238                      ->cast_to_size(t_oop->is_aryptr()->size())
 239                      ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
 240                      ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
 241                      ->with_offset(t_oop->is_aryptr()->offset())
 242                      ->is_aryptr();
 243       }
 244       do_split = mem_t == t_oop;
 245     }
 246     if (do_split) {
 247       // clone the Phi with our address type
 248       result = mphi->split_out_instance(t_adr, igvn);
 249     } else {
 250       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 251     }
 252   }
 253   return result;
 254 }
 255 
 256 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 257   uint alias_idx = phase->C->get_alias_index(tp);
 258   Node *mem = mmem;
 259 #ifdef ASSERT
 260   {
 261     // Check that current type is consistent with the alias index used during graph construction
 262     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 263     bool consistent =  adr_check == nullptr || adr_check->empty() ||
 264                        phase->C->must_alias(adr_check, alias_idx );
 265     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 266     if( !consistent && adr_check != nullptr && !adr_check->empty() &&
 267         tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 268         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 269         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 270           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 271           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 272       // don't assert if it is dead code.
 273       consistent = true;
 274     }
 275     if( !consistent ) {
 276       st->print("alias_idx==%d, adr_check==", alias_idx);
 277       if( adr_check == nullptr ) {
 278         st->print("null");
 279       } else {
 280         adr_check->dump();
 281       }
 282       st->cr();
 283       print_alias_types();
 284       assert(consistent, "adr_check must match alias idx");
 285     }
 286   }
 287 #endif

1000     Node* ld = gvn.transform(load);
1001     return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1002   }
1003 
1004   return load;
1005 }
1006 
1007 //------------------------------hash-------------------------------------------
1008 uint LoadNode::hash() const {
1009   // unroll addition of interesting fields
1010   return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1011 }
1012 
1013 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1014   if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1015     bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1016     bool is_stable_ary = FoldStableValues &&
1017                          (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1018                          tp->isa_aryptr()->is_stable();
1019 
1020     return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1021   }
1022 
1023   return false;
1024 }
1025 
1026 LoadNode* LoadNode::pin_array_access_node() const {
1027   const TypePtr* adr_type = this->adr_type();
1028   if (adr_type != nullptr && adr_type->isa_aryptr()) {
1029     return clone_pinned();
1030   }
1031   return nullptr;
1032 }
1033 
1034 // Is the value loaded previously stored by an arraycopy? If so return
1035 // a load node that reads from the source array so we may be able to
1036 // optimize out the ArrayCopy node later.
1037 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1038   Node* ld_adr = in(MemNode::Address);
1039   intptr_t ld_off = 0;
1040   AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);

1057       assert(ld_alloc != nullptr, "need an alloc");
1058       assert(addp->is_AddP(), "address must be addp");
1059       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1060       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1061       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1062       addp->set_req(AddPNode::Base, src);
1063       addp->set_req(AddPNode::Address, src);
1064     } else {
1065       assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1066              ac->as_ArrayCopy()->is_copyof_validated() ||
1067              ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1068       assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1069       addp->set_req(AddPNode::Base, src);
1070       addp->set_req(AddPNode::Address, src);
1071 
1072       const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1073       BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1074       if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1075 
1076       uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1077       uint shift  = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1078 
1079       Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1080 #ifdef _LP64
1081       diff = phase->transform(new ConvI2LNode(diff));
1082 #endif
1083       diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1084 
1085       Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1086       addp->set_req(AddPNode::Offset, offset);
1087     }
1088     addp = phase->transform(addp);
1089 #ifdef ASSERT
1090     const TypePtr* adr_type = phase->type(addp)->is_ptr();
1091     ld->_adr_type = adr_type;
1092 #endif
1093     ld->set_req(MemNode::Address, addp);
1094     ld->set_req(0, ctl);
1095     ld->set_req(MemNode::Memory, mem);
1096     return ld;
1097   }

1176         // Same base, same offset.
1177         // Possible improvement for arrays: check index value instead of absolute offset.
1178 
1179         // At this point we have proven something like this setup:
1180         //   B = << base >>
1181         //   L =  LoadQ(AddP(Check/CastPP(B), #Off))
1182         //   S = StoreQ(AddP(             B , #Off), V)
1183         // (Actually, we haven't yet proven the Q's are the same.)
1184         // In other words, we are loading from a casted version of
1185         // the same pointer-and-offset that we stored to.
1186         // Casted version may carry a dependency and it is respected.
1187         // Thus, we are able to replace L by V.
1188       }
1189       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1190       if (store_Opcode() != st->Opcode()) {
1191         return nullptr;
1192       }
1193       // LoadVector/StoreVector needs additional check to ensure the types match.
1194       if (st->is_StoreVector()) {
1195         const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
1196         const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1197         if (in_vt != out_vt) {
1198           return nullptr;
1199         }
1200       }
1201       return st->in(MemNode::ValueIn);
1202     }
1203 
1204     // A load from a freshly-created object always returns zero.
1205     // (This can happen after LoadNode::Ideal resets the load's memory input
1206     // to find_captured_store, which returned InitializeNode::zero_memory.)
1207     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1208         (st->in(0) == ld_alloc) &&
1209         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1210       // return a zero value for the load's basic type
1211       // (This is one of the few places where a generic PhaseTransform
1212       // can create new nodes.  Think of it as lazily manifesting
1213       // virtually pre-existing constants.)
1214       Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1215       if (default_value != nullptr) {
1216         return default_value;
1217       }
1218       assert(ld_alloc->in(AllocateNode::RawDefaultValue) == nullptr, "default value may not be null");
1219       if (memory_type() != T_VOID) {
1220         if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1221           // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1222           // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1223           // by the ArrayCopyNode.
1224           return phase->zerocon(memory_type());
1225         }
1226       } else {
1227         // TODO: materialize all-zero vector constant
1228         assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1229       }
1230     }
1231 
1232     // A load from an initialization barrier can match a captured store.
1233     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1234       InitializeNode* init = st->in(0)->as_Initialize();
1235       AllocateNode* alloc = init->allocation();
1236       if ((alloc != nullptr) && (alloc == ld_alloc)) {
1237         // examine a captured store value
1238         st = init->find_captured_store(ld_off, memory_size(), phase);

1266 //----------------------is_instance_field_load_with_local_phi------------------
1267 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1268   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1269       in(Address)->is_AddP() ) {
1270     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1271     // Only instances and boxed values.
1272     if( t_oop != nullptr &&
1273         (t_oop->is_ptr_to_boxed_value() ||
1274          t_oop->is_known_instance_field()) &&
1275         t_oop->offset() != Type::OffsetBot &&
1276         t_oop->offset() != Type::OffsetTop) {
1277       return true;
1278     }
1279   }
1280   return false;
1281 }
1282 
1283 //------------------------------Identity---------------------------------------
1284 // Loads are identity if previous store is to same address
1285 Node* LoadNode::Identity(PhaseGVN* phase) {
1286   // Loading from an InlineType? The InlineType has the values of
1287   // all fields as input. Look for the field with matching offset.
1288   Node* addr = in(Address);
1289   intptr_t offset;
1290   Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1291   if (!is_mismatched_access() && base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1292     Node* value = base->as_InlineType()->field_value_by_offset((int)offset, true);
1293     if (value != nullptr) {
1294       if (Opcode() == Op_LoadN) {
1295         // Encode oop value if we are loading a narrow oop
1296         assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1297         value = phase->transform(new EncodePNode(value, bottom_type()));
1298       }
1299       return value;
1300     }
1301   }
1302 
1303   // If the previous store-maker is the right kind of Store, and the store is
1304   // to the same address, then we are equal to the value stored.
1305   Node* mem = in(Memory);
1306   Node* value = can_see_stored_value(mem, phase);
1307   if( value ) {
1308     // byte, short & char stores truncate naturally.
1309     // A load has to load the truncated value which requires
1310     // some sort of masking operation and that requires an
1311     // Ideal call instead of an Identity call.
1312     if (memory_size() < BytesPerInt) {
1313       // If the input to the store does not fit with the load's result type,
1314       // it must be truncated via an Ideal call.
1315       if (!phase->type(value)->higher_equal(phase->type(this)))
1316         return this;
1317     }
1318     // (This works even when value is a Con, but LoadNode::Value
1319     // usually runs first, producing the singleton type of the Con.)
1320     if (!has_pinned_control_dependency() || value->is_Con()) {
1321       return value;
1322     } else {

1876   bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1877          phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1878 
1879   // Skip up past a SafePoint control.  Cannot do this for Stores because
1880   // pointer stores & cardmarks must stay on the same side of a SafePoint.
1881   if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1882       phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw  &&
1883       !addr_mark &&
1884       (depends_only_on_test() || has_unknown_control_dependency())) {
1885     ctrl = ctrl->in(0);
1886     set_req(MemNode::Control,ctrl);
1887     progress = true;
1888   }
1889 
1890   intptr_t ignore = 0;
1891   Node*    base   = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1892   if (base != nullptr
1893       && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1894     // Check for useless control edge in some common special cases
1895     if (in(MemNode::Control) != nullptr
1896         && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
1897         && can_remove_control()
1898         && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1899         && all_controls_dominate(base, phase->C->start())) {
1900       // A method-invariant, non-null address (constant or 'this' argument).
1901       set_req(MemNode::Control, nullptr);
1902       progress = true;
1903     }
1904   }
1905 
1906   Node* mem = in(MemNode::Memory);
1907   const TypePtr *addr_t = phase->type(address)->isa_ptr();
1908 
1909   if (can_reshape && (addr_t != nullptr)) {
1910     // try to optimize our memory input
1911     Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1912     if (opt_mem != mem) {
1913       set_req_X(MemNode::Memory, opt_mem, phase);
1914       if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1915       return this;
1916     }

2081       }
2082     }
2083 
2084     // Don't do this for integer types. There is only potential profit if
2085     // the element type t is lower than _type; that is, for int types, if _type is
2086     // more restrictive than t.  This only happens here if one is short and the other
2087     // char (both 16 bits), and in those cases we've made an intentional decision
2088     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2089     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2090     //
2091     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2092     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
2093     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2094     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
2095     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2096     // In fact, that could have been the original type of p1, and p1 could have
2097     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2098     // expression (LShiftL quux 3) independently optimized to the constant 8.
2099     if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2100         && (_type->isa_vect() == nullptr)
2101         && !ary->is_flat()
2102         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2103       // t might actually be lower than _type, if _type is a unique
2104       // concrete subclass of abstract class t.
2105       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
2106         const Type* jt = t->join_speculative(_type);
2107         // In any case, do not allow the join, per se, to empty out the type.
2108         if (jt->empty() && !t->empty()) {
2109           // This can happen if a interface-typed array narrows to a class type.
2110           jt = _type;
2111         }
2112 #ifdef ASSERT
2113         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2114           // The pointers in the autobox arrays are always non-null
2115           Node* base = adr->in(AddPNode::Base);
2116           if ((base != nullptr) && base->is_DecodeN()) {
2117             // Get LoadN node which loads IntegerCache.cache field
2118             base = base->in(1);
2119           }
2120           if ((base != nullptr) && base->is_Con()) {
2121             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2122             if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2123               // It could be narrow oop
2124               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2125             }
2126           }
2127         }
2128 #endif
2129         return jt;
2130       }
2131     }
2132   } else if (tp->base() == Type::InstPtr) {
2133     assert( off != Type::OffsetBot ||
2134             // arrays can be cast to Objects
2135             !tp->isa_instptr() ||
2136             tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2137             // Default value load
2138             tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2139             // unsafe field access may not have a constant offset
2140             C->has_unsafe_access(),
2141             "Field accesses must be precise" );
2142     // For oop loads, we expect the _type to be precise.
2143 

2144     const TypeInstPtr* tinst = tp->is_instptr();
2145     BasicType bt = memory_type();
2146 
2147     // Optimize loads from constant fields.
2148     ciObject* const_oop = tinst->const_oop();
2149     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2150       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2151       if (con_type != nullptr) {
2152         return con_type;
2153       }
2154     }
2155   } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2156     assert(off != Type::OffsetBot ||
2157             !tp->isa_instklassptr() ||
2158            // arrays can be cast to Objects
2159            tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2160            // also allow array-loading from the primary supertype
2161            // array during subtype checks
2162            Opcode() == Op_LoadKlass,
2163            "Field accesses must be precise");
2164     // For klass/static loads, we expect the _type to be precise
2165   } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2166     /* With mirrors being an indirect in the Klass*
2167      * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2168      * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2169      *
2170      * So check the type and klass of the node before the LoadP.

2177         assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2178         assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2179         return TypeInstPtr::make(klass->java_mirror());
2180       }
2181     }
2182   }
2183 
2184   const TypeKlassPtr *tkls = tp->isa_klassptr();
2185   if (tkls != nullptr) {
2186     if (tkls->is_loaded() && tkls->klass_is_exact()) {
2187       ciKlass* klass = tkls->exact_klass();
2188       // We are loading a field from a Klass metaobject whose identity
2189       // is known at compile time (the type is "exact" or "precise").
2190       // Check for fields we know are maintained as constants by the VM.
2191       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2192         // The field is Klass::_super_check_offset.  Return its (constant) value.
2193         // (Folds up type checking code.)
2194         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2195         return TypeInt::make(klass->super_check_offset());
2196       }
2197       if (UseCompactObjectHeaders) { // TODO: Should EnableValhalla also take this path ?
2198         if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2199           // The field is Klass::_prototype_header. Return its (constant) value.
2200           assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2201           return TypeX::make(klass->prototype_header());
2202         }
2203       }
2204       // Compute index into primary_supers array
2205       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2206       // Check for overflowing; use unsigned compare to handle the negative case.
2207       if( depth < ciKlass::primary_super_limit() ) {
2208         // The field is an element of Klass::_primary_supers.  Return its (constant) value.
2209         // (Folds up type checking code.)
2210         assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2211         ciKlass *ss = klass->super_of_depth(depth);
2212         return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2213       }
2214       const Type* aift = load_array_final_field(tkls, klass);
2215       if (aift != nullptr)  return aift;
2216     }
2217 

2258       jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2259       // The key property of this type is that it folds up tests
2260       // for array-ness, since it proves that the layout_helper is positive.
2261       // Thus, a generic value like the basic object layout helper works fine.
2262       return TypeInt::make(min_size, max_jint, Type::WidenMin);
2263     }
2264   }
2265 
2266   bool is_vect = (_type->isa_vect() != nullptr);
2267   if (is_instance && !is_vect) {
2268     // If we have an instance type and our memory input is the
2269     // programs's initial memory state, there is no matching store,
2270     // so just return a zero of the appropriate type -
2271     // except if it is vectorized - then we have no zero constant.
2272     Node *mem = in(MemNode::Memory);
2273     if (mem->is_Parm() && mem->in(0)->is_Start()) {
2274       assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2275       return Type::get_zero_type(_type->basic_type());
2276     }
2277   }

2278   if (!UseCompactObjectHeaders) {
2279     Node* alloc = is_new_object_mark_load();
2280     if (alloc != nullptr) {
2281       if (EnableValhalla) {
2282         // The mark word may contain property bits (inline, flat, null-free)
2283         Node* klass_node = alloc->in(AllocateNode::KlassNode);
2284         const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2285         if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2286           return TypeX::make(tkls->exact_klass()->prototype_header());
2287         }
2288       } else {
2289         return TypeX::make(markWord::prototype().value());
2290       }
2291     }
2292   }
2293 
2294   return _type;
2295 }
2296 
2297 //------------------------------match_edge-------------------------------------
2298 // Do we Match on this edge index or not?  Match only the address.
2299 uint LoadNode::match_edge(uint idx) const {
2300   return idx == MemNode::Address;
2301 }
2302 
2303 //--------------------------LoadBNode::Ideal--------------------------------------
2304 //
2305 //  If the previous store is to the same address as this load,
2306 //  and the value stored was larger than a byte, replace this load
2307 //  with the value stored truncated to a byte.  If no truncation is
2308 //  needed, the replacement is done in LoadNode::Identity().
2309 //
2310 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {

2422   return LoadNode::Ideal(phase, can_reshape);
2423 }
2424 
2425 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2426   Node* mem = in(MemNode::Memory);
2427   Node* value = can_see_stored_value(mem,phase);
2428   if (value != nullptr && value->is_Con() &&
2429       !value->bottom_type()->higher_equal(_type)) {
2430     // If the input to the store does not fit with the load's result type,
2431     // it must be truncated. We can't delay until Ideal call since
2432     // a singleton Value is needed for split_thru_phi optimization.
2433     int con = value->get_int();
2434     return TypeInt::make((con << 16) >> 16);
2435   }
2436   return LoadNode::Value(phase);
2437 }
2438 
2439 //=============================================================================
2440 //----------------------------LoadKlassNode::make------------------------------
2441 // Polymorphic factory method:
2442 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2443                           const TypeKlassPtr* tk) {
2444   // sanity check the alias category against the created node type
2445   const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2446   assert(adr_type != nullptr, "expecting TypeKlassPtr");
2447 #ifdef _LP64
2448   if (adr_type->is_ptr_to_narrowklass()) {
2449     assert(UseCompressedClassPointers, "no compressed klasses");
2450     Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2451     return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2452   }
2453 #endif
2454   assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2455   return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2456 }
2457 
2458 //------------------------------Value------------------------------------------
2459 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2460   return klass_value_common(phase);
2461 }
2462 
2463 // In most cases, LoadKlassNode does not have the control input set. If the control

2470   // Either input is TOP ==> the result is TOP
2471   const Type *t1 = phase->type( in(MemNode::Memory) );
2472   if (t1 == Type::TOP)  return Type::TOP;
2473   Node *adr = in(MemNode::Address);
2474   const Type *t2 = phase->type( adr );
2475   if (t2 == Type::TOP)  return Type::TOP;
2476   const TypePtr *tp = t2->is_ptr();
2477   if (TypePtr::above_centerline(tp->ptr()) ||
2478       tp->ptr() == TypePtr::Null)  return Type::TOP;
2479 
2480   // Return a more precise klass, if possible
2481   const TypeInstPtr *tinst = tp->isa_instptr();
2482   if (tinst != nullptr) {
2483     ciInstanceKlass* ik = tinst->instance_klass();
2484     int offset = tinst->offset();
2485     if (ik == phase->C->env()->Class_klass()
2486         && (offset == java_lang_Class::klass_offset() ||
2487             offset == java_lang_Class::array_klass_offset())) {
2488       // We are loading a special hidden field from a Class mirror object,
2489       // the field which points to the VM's Klass metaobject.
2490       bool is_null_free_array = false;
2491       ciType* t = tinst->java_mirror_type(&is_null_free_array);
2492       // java_mirror_type returns non-null for compile-time Class constants.
2493       if (t != nullptr) {
2494         // constant oop => constant klass
2495         if (offset == java_lang_Class::array_klass_offset()) {
2496           if (t->is_void()) {
2497             // We cannot create a void array.  Since void is a primitive type return null
2498             // klass.  Users of this result need to do a null check on the returned klass.
2499             return TypePtr::NULL_PTR;
2500           }
2501           const TypeKlassPtr* tklass = TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2502           if (is_null_free_array) {
2503             tklass = tklass->is_aryklassptr()->cast_to_null_free();
2504           }
2505           return tklass;
2506         }
2507         if (!t->is_klass()) {
2508           // a primitive Class (e.g., int.class) has null for a klass field
2509           return TypePtr::NULL_PTR;
2510         }
2511         // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2512         const TypeKlassPtr* tklass = TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2513         if (is_null_free_array) {
2514           tklass = tklass->is_aryklassptr()->cast_to_null_free();
2515         }
2516         return tklass;
2517       }
2518       // non-constant mirror, so we can't tell what's going on
2519     }
2520     if (!tinst->is_loaded())
2521       return _type;             // Bail out if not loaded
2522     if (offset == oopDesc::klass_offset_in_bytes()) {
2523       return tinst->as_klass_type(true);
2524     }
2525   }
2526 
2527   // Check for loading klass from an array
2528   const TypeAryPtr* tary = tp->isa_aryptr();
2529   if (tary != nullptr &&
2530       tary->offset() == oopDesc::klass_offset_in_bytes()) {
2531     return tary->as_klass_type(true);
2532   }
2533 
2534   // Check for loading klass from an array klass
2535   const TypeKlassPtr *tkls = tp->isa_klassptr();
2536   if (tkls != nullptr && !StressReflectiveCode) {
2537     if (!tkls->is_loaded())
2538      return _type;             // Bail out if not loaded
2539     if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2540         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2541       // // Always returning precise element type is incorrect,
2542       // // e.g., element type could be object and array may contain strings
2543       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2544 
2545       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2546       // according to the element type's subclassing.
2547       return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2548     }

3345   }
3346   ss.print_cr("[TraceMergeStores]: with");
3347   merged_input_value->dump("\n", false, &ss);
3348   merged_store->dump("\n", false, &ss);
3349   tty->print("%s", ss.as_string());
3350 }
3351 #endif
3352 
3353 //------------------------------Ideal------------------------------------------
3354 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3355 // When a store immediately follows a relevant allocation/initialization,
3356 // try to capture it into the initialization, or hoist it above.
3357 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3358   Node* p = MemNode::Ideal_common(phase, can_reshape);
3359   if (p)  return (p == NodeSentinel) ? nullptr : p;
3360 
3361   Node* mem     = in(MemNode::Memory);
3362   Node* address = in(MemNode::Address);
3363   Node* value   = in(MemNode::ValueIn);
3364   // Back-to-back stores to same address?  Fold em up.  Generally
3365   // unsafe if I have intervening uses...
3366   if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3367     Node* st = mem;
3368     // If Store 'st' has more than one use, we cannot fold 'st' away.
3369     // For example, 'st' might be the final state at a conditional
3370     // return.  Or, 'st' might be used by some node which is live at
3371     // the same time 'st' is live, which might be unschedulable.  So,
3372     // require exactly ONE user until such time as we clone 'mem' for
3373     // each of 'mem's uses (thus making the exactly-1-user-rule hold
3374     // true).
3375     while (st->is_Store() && st->outcnt() == 1) {
3376       // Looking at a dead closed cycle of memory?
3377       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3378       assert(Opcode() == st->Opcode() ||
3379              st->Opcode() == Op_StoreVector ||
3380              Opcode() == Op_StoreVector ||
3381              st->Opcode() == Op_StoreVectorScatter ||
3382              Opcode() == Op_StoreVectorScatter ||
3383              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3384              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3385              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3386              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3387              (st->adr_type()->isa_aryptr() && st->adr_type()->is_aryptr()->is_flat()) || // TODO 8343835
3388              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3389              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3390 
3391       if (st->in(MemNode::Address)->eqv_uncast(address) &&
3392           st->as_Store()->memory_size() <= this->memory_size()) {
3393         Node* use = st->raw_out(0);
3394         if (phase->is_IterGVN()) {
3395           phase->is_IterGVN()->rehash_node_delayed(use);
3396         }
3397         // It's OK to do this in the parser, since DU info is always accurate,
3398         // and the parser always refers to nodes via SafePointNode maps.
3399         use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3400         return this;
3401       }
3402       st = st->in(MemNode::Memory);
3403     }
3404   }
3405 
3406 
3407   // Capture an unaliased, unconditional, simple store into an initializer.

3494       const StoreVectorNode* store_vector = as_StoreVector();
3495       const StoreVectorNode* mem_vector = mem->as_StoreVector();
3496       const Node* store_indices = store_vector->indices();
3497       const Node* mem_indices = mem_vector->indices();
3498       const Node* store_mask = store_vector->mask();
3499       const Node* mem_mask = mem_vector->mask();
3500       // Ensure types, indices, and masks match
3501       if (store_vector->vect_type() == mem_vector->vect_type() &&
3502           ((store_indices == nullptr) == (mem_indices == nullptr) &&
3503            (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3504           ((store_mask == nullptr) == (mem_mask == nullptr) &&
3505            (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3506         result = mem;
3507       }
3508     }
3509   }
3510 
3511   // Store of zero anywhere into a freshly-allocated object?
3512   // Then the store is useless.
3513   // (It must already have been captured by the InitializeNode.)
3514   if (result == this && ReduceFieldZeroing) {

3515     // a newly allocated object is already all-zeroes everywhere
3516     if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3517         (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
3518       result = mem;
3519     }
3520 
3521     if (result == this && phase->type(val)->is_zero_type()) {
3522       // the store may also apply to zero-bits in an earlier object
3523       Node* prev_mem = find_previous_store(phase);
3524       // Steps (a), (b):  Walk past independent stores to find an exact match.
3525       if (prev_mem != nullptr) {
3526         Node* prev_val = can_see_stored_value(prev_mem, phase);
3527         if (prev_val != nullptr && prev_val == val) {
3528           // prev_val and val might differ by a cast; it would be good
3529           // to keep the more informative of the two.
3530           result = mem;
3531         }
3532       }
3533     }
3534   }
3535 
3536   PhaseIterGVN* igvn = phase->is_IterGVN();
3537   if (result != this && igvn != nullptr) {
3538     MemBarNode* trailing = trailing_membar();
3539     if (trailing != nullptr) {
3540 #ifdef ASSERT
3541       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();

3817 // Clearing a short array is faster with stores
3818 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3819   // Already know this is a large node, do not try to ideal it
3820   if (_is_large) return nullptr;
3821 
3822   const int unit = BytesPerLong;
3823   const TypeX* t = phase->type(in(2))->isa_intptr_t();
3824   if (!t)  return nullptr;
3825   if (!t->is_con())  return nullptr;
3826   intptr_t raw_count = t->get_con();
3827   intptr_t size = raw_count;
3828   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3829   // Clearing nothing uses the Identity call.
3830   // Negative clears are possible on dead ClearArrays
3831   // (see jck test stmt114.stmt11402.val).
3832   if (size <= 0 || size % unit != 0)  return nullptr;
3833   intptr_t count = size / unit;
3834   // Length too long; communicate this to matchers and assemblers.
3835   // Assemblers are responsible to produce fast hardware clears for it.
3836   if (size > InitArrayShortSize) {
3837     return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3838   } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3839     return nullptr;
3840   }
3841   if (!IdealizeClearArrayNode) return nullptr;
3842   Node *mem = in(1);
3843   if( phase->type(mem)==Type::TOP ) return nullptr;
3844   Node *adr = in(3);
3845   const Type* at = phase->type(adr);
3846   if( at==Type::TOP ) return nullptr;
3847   const TypePtr* atp = at->isa_ptr();
3848   // adjust atp to be the correct array element address type
3849   if (atp == nullptr)  atp = TypePtr::BOTTOM;
3850   else              atp = atp->add_offset(Type::OffsetBot);
3851   // Get base for derived pointer purposes
3852   if( adr->Opcode() != Op_AddP ) Unimplemented();
3853   Node *base = adr->in(1);
3854 
3855   Node *val = in(4);
3856   Node *off  = phase->MakeConX(BytesPerLong);
3857   mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3858   count--;
3859   while( count-- ) {
3860     mem = phase->transform(mem);
3861     adr = phase->transform(new AddPNode(base,adr,off));
3862     mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3863   }
3864   return mem;
3865 }
3866 
3867 //----------------------------step_through----------------------------------
3868 // Return allocation input memory edge if it is different instance
3869 // or itself if it is the one we are looking for.
3870 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3871   Node* n = *np;
3872   assert(n->is_ClearArray(), "sanity");
3873   intptr_t offset;
3874   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3875   // This method is called only before Allocate nodes are expanded
3876   // during macro nodes expansion. Before that ClearArray nodes are
3877   // only generated in PhaseMacroExpand::generate_arraycopy() (before
3878   // Allocate nodes are expanded) which follows allocations.
3879   assert(alloc != nullptr, "should have allocation");
3880   if (alloc->_idx == instance_id) {
3881     // Can not bypass initialization of the instance we are looking for.
3882     return false;
3883   }
3884   // Otherwise skip it.
3885   InitializeNode* init = alloc->initialization();
3886   if (init != nullptr)
3887     *np = init->in(TypeFunc::Memory);
3888   else
3889     *np = alloc->in(TypeFunc::Memory);
3890   return true;
3891 }
3892 
3893 //----------------------------clear_memory-------------------------------------
3894 // Generate code to initialize object storage to zero.
3895 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3896                                    Node* val,
3897                                    Node* raw_val,
3898                                    intptr_t start_offset,
3899                                    Node* end_offset,
3900                                    PhaseGVN* phase) {
3901   intptr_t offset = start_offset;
3902 
3903   int unit = BytesPerLong;
3904   if ((offset % unit) != 0) {
3905     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3906     adr = phase->transform(adr);
3907     const TypePtr* atp = TypeRawPtr::BOTTOM;
3908     if (val != nullptr) {
3909       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3910       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3911     } else {
3912       assert(raw_val == nullptr, "val may not be null");
3913       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3914     }
3915     mem = phase->transform(mem);
3916     offset += BytesPerInt;
3917   }
3918   assert((offset % unit) == 0, "");
3919 
3920   // Initialize the remaining stuff, if any, with a ClearArray.
3921   return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3922 }
3923 
3924 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3925                                    Node* raw_val,
3926                                    Node* start_offset,
3927                                    Node* end_offset,
3928                                    PhaseGVN* phase) {
3929   if (start_offset == end_offset) {
3930     // nothing to do
3931     return mem;
3932   }
3933 
3934   int unit = BytesPerLong;
3935   Node* zbase = start_offset;
3936   Node* zend  = end_offset;
3937 
3938   // Scale to the unit required by the CPU:
3939   if (!Matcher::init_array_count_is_in_bytes) {
3940     Node* shift = phase->intcon(exact_log2(unit));
3941     zbase = phase->transform(new URShiftXNode(zbase, shift) );
3942     zend  = phase->transform(new URShiftXNode(zend,  shift) );
3943   }
3944 
3945   // Bulk clear double-words
3946   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3947   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3948   if (raw_val == nullptr) {
3949     raw_val = phase->MakeConX(0);
3950   }
3951   mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3952   return phase->transform(mem);
3953 }
3954 
3955 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3956                                    Node* val,
3957                                    Node* raw_val,
3958                                    intptr_t start_offset,
3959                                    intptr_t end_offset,
3960                                    PhaseGVN* phase) {
3961   if (start_offset == end_offset) {
3962     // nothing to do
3963     return mem;
3964   }
3965 
3966   assert((end_offset % BytesPerInt) == 0, "odd end offset");
3967   intptr_t done_offset = end_offset;
3968   if ((done_offset % BytesPerLong) != 0) {
3969     done_offset -= BytesPerInt;
3970   }
3971   if (done_offset > start_offset) {
3972     mem = clear_memory(ctl, mem, dest, val, raw_val,
3973                        start_offset, phase->MakeConX(done_offset), phase);
3974   }
3975   if (done_offset < end_offset) { // emit the final 32-bit store
3976     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3977     adr = phase->transform(adr);
3978     const TypePtr* atp = TypeRawPtr::BOTTOM;
3979     if (val != nullptr) {
3980       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3981       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3982     } else {
3983       assert(raw_val == nullptr, "val may not be null");
3984       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3985     }
3986     mem = phase->transform(mem);
3987     done_offset += BytesPerInt;
3988   }
3989   assert(done_offset == end_offset, "");
3990   return mem;
3991 }
3992 
3993 //=============================================================================
3994 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3995   : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3996     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3997 #ifdef ASSERT
3998   , _pair_idx(0)
3999 #endif
4000 {
4001   init_class_id(Class_MemBar);
4002   Node* top = C->top();
4003   init_req(TypeFunc::I_O,top);
4004   init_req(TypeFunc::FramePtr,top);
4005   init_req(TypeFunc::ReturnAdr,top);

4111       PhaseIterGVN* igvn = phase->is_IterGVN();
4112       remove(igvn);
4113       // Must return either the original node (now dead) or a new node
4114       // (Do not return a top here, since that would break the uniqueness of top.)
4115       return new ConINode(TypeInt::ZERO);
4116     }
4117   }
4118   return progress ? this : nullptr;
4119 }
4120 
4121 //------------------------------Value------------------------------------------
4122 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4123   if( !in(0) ) return Type::TOP;
4124   if( phase->type(in(0)) == Type::TOP )
4125     return Type::TOP;
4126   return TypeTuple::MEMBAR;
4127 }
4128 
4129 //------------------------------match------------------------------------------
4130 // Construct projections for memory.
4131 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4132   switch (proj->_con) {
4133   case TypeFunc::Control:
4134   case TypeFunc::Memory:
4135     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4136   }
4137   ShouldNotReachHere();
4138   return nullptr;
4139 }
4140 
4141 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4142   trailing->_kind = TrailingStore;
4143   leading->_kind = LeadingStore;
4144 #ifdef ASSERT
4145   trailing->_pair_idx = leading->_idx;
4146   leading->_pair_idx = leading->_idx;
4147 #endif
4148 }
4149 
4150 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4151   trailing->_kind = TrailingLoadStore;

4398   return (req() > RawStores);
4399 }
4400 
4401 void InitializeNode::set_complete(PhaseGVN* phase) {
4402   assert(!is_complete(), "caller responsibility");
4403   _is_complete = Complete;
4404 
4405   // After this node is complete, it contains a bunch of
4406   // raw-memory initializations.  There is no need for
4407   // it to have anything to do with non-raw memory effects.
4408   // Therefore, tell all non-raw users to re-optimize themselves,
4409   // after skipping the memory effects of this initialization.
4410   PhaseIterGVN* igvn = phase->is_IterGVN();
4411   if (igvn)  igvn->add_users_to_worklist(this);
4412 }
4413 
4414 // convenience function
4415 // return false if the init contains any stores already
4416 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4417   InitializeNode* init = initialization();
4418   if (init == nullptr || init->is_complete()) {
4419     return false;
4420   }
4421   init->remove_extra_zeroes();
4422   // for now, if this allocation has already collected any inits, bail:
4423   if (init->is_non_zero())  return false;
4424   init->set_complete(phase);
4425   return true;
4426 }
4427 
4428 void InitializeNode::remove_extra_zeroes() {
4429   if (req() == RawStores)  return;
4430   Node* zmem = zero_memory();
4431   uint fill = RawStores;
4432   for (uint i = fill; i < req(); i++) {
4433     Node* n = in(i);
4434     if (n->is_top() || n == zmem)  continue;  // skip
4435     if (fill < i)  set_req(fill, n);          // compact
4436     ++fill;
4437   }
4438   // delete any empty spaces created:
4439   while (fill < req()) {
4440     del_req(fill);

4584             // store node that we'd like to capture. We need to check
4585             // the uses of the MergeMemNode.
4586             mems.push(n);
4587           }
4588         } else if (n->is_Mem()) {
4589           Node* other_adr = n->in(MemNode::Address);
4590           if (other_adr == adr) {
4591             failed = true;
4592             break;
4593           } else {
4594             const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4595             if (other_t_adr != nullptr) {
4596               int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4597               if (other_alias_idx == alias_idx) {
4598                 // A load from the same memory slice as the store right
4599                 // after the InitializeNode. We check the control of the
4600                 // object/array that is loaded from. If it's the same as
4601                 // the store control then we cannot capture the store.
4602                 assert(!n->is_Store(), "2 stores to same slice on same control?");
4603                 Node* base = other_adr;
4604                 if (base->is_Phi()) {
4605                   // In rare case, base may be a PhiNode and it may read
4606                   // the same memory slice between InitializeNode and store.
4607                   failed = true;
4608                   break;
4609                 }
4610                 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4611                 base = base->in(AddPNode::Base);
4612                 if (base != nullptr) {
4613                   base = base->uncast();
4614                   if (base->is_Proj() && base->in(0) == alloc) {
4615                     failed = true;
4616                     break;
4617                   }
4618                 }
4619               }
4620             }
4621           }
4622         } else {
4623           failed = true;
4624           break;
4625         }
4626       }
4627     }
4628   }
4629   if (failed) {

5176         //   z's_done      12  16  16  16    12  16    12
5177         //   z's_needed    12  16  16  16    16  16    16
5178         //   zsize          0   0   0   0     4   0     4
5179         if (next_full_store < 0) {
5180           // Conservative tack:  Zero to end of current word.
5181           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5182         } else {
5183           // Zero to beginning of next fully initialized word.
5184           // Or, don't zero at all, if we are already in that word.
5185           assert(next_full_store >= zeroes_needed, "must go forward");
5186           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5187           zeroes_needed = next_full_store;
5188         }
5189       }
5190 
5191       if (zeroes_needed > zeroes_done) {
5192         intptr_t zsize = zeroes_needed - zeroes_done;
5193         // Do some incremental zeroing on rawmem, in parallel with inits.
5194         zeroes_done = align_down(zeroes_done, BytesPerInt);
5195         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5196                                               allocation()->in(AllocateNode::DefaultValue),
5197                                               allocation()->in(AllocateNode::RawDefaultValue),
5198                                               zeroes_done, zeroes_needed,
5199                                               phase);
5200         zeroes_done = zeroes_needed;
5201         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5202           do_zeroing = false;   // leave the hole, next time
5203       }
5204     }
5205 
5206     // Collect the store and move on:
5207     phase->replace_input_of(st, MemNode::Memory, inits);
5208     inits = st;                 // put it on the linearized chain
5209     set_req(i, zmem);           // unhook from previous position
5210 
5211     if (zeroes_done == st_off)
5212       zeroes_done = next_init_off;
5213 
5214     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5215 
5216     #ifdef ASSERT
5217     // Various order invariants.  Weaker than stores_are_sane because

5237   remove_extra_zeroes();        // clear out all the zmems left over
5238   add_req(inits);
5239 
5240   if (!(UseTLAB && ZeroTLAB)) {
5241     // If anything remains to be zeroed, zero it all now.
5242     zeroes_done = align_down(zeroes_done, BytesPerInt);
5243     // if it is the last unused 4 bytes of an instance, forget about it
5244     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5245     if (zeroes_done + BytesPerLong >= size_limit) {
5246       AllocateNode* alloc = allocation();
5247       assert(alloc != nullptr, "must be present");
5248       if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5249         Node* klass_node = alloc->in(AllocateNode::KlassNode);
5250         ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5251         if (zeroes_done == k->layout_helper())
5252           zeroes_done = size_limit;
5253       }
5254     }
5255     if (zeroes_done < size_limit) {
5256       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5257                                             allocation()->in(AllocateNode::DefaultValue),
5258                                             allocation()->in(AllocateNode::RawDefaultValue),
5259                                             zeroes_done, size_in_bytes, phase);
5260     }
5261   }
5262 
5263   set_complete(phase);
5264   return rawmem;
5265 }
5266 
5267 
5268 #ifdef ASSERT
5269 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5270   if (is_complete())
5271     return true;                // stores could be anything at this point
5272   assert(allocation() != nullptr, "must be present");
5273   intptr_t last_off = allocation()->minimum_header_size();
5274   for (uint i = InitializeNode::RawStores; i < req(); i++) {
5275     Node* st = in(i);
5276     intptr_t st_off = get_store_offset(st, phase);
5277     if (st_off < 0)  continue;  // ignore dead garbage
5278     if (last_off > st_off) {
< prev index next >