7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "compiler/compileLog.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/c2/barrierSetC2.hpp"
31 #include "gc/shared/tlab_globals.hpp"
32 #include "memory/allocation.inline.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "oops/objArrayKlass.hpp"
35 #include "opto/addnode.hpp"
36 #include "opto/arraycopynode.hpp"
37 #include "opto/cfgnode.hpp"
38 #include "opto/regalloc.hpp"
39 #include "opto/compile.hpp"
40 #include "opto/connode.hpp"
41 #include "opto/convertnode.hpp"
42 #include "opto/loopnode.hpp"
43 #include "opto/machnode.hpp"
44 #include "opto/matcher.hpp"
45 #include "opto/memnode.hpp"
46 #include "opto/mulnode.hpp"
47 #include "opto/narrowptrnode.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regmask.hpp"
50 #include "opto/rootnode.hpp"
51 #include "opto/vectornode.hpp"
52 #include "utilities/align.hpp"
53 #include "utilities/copy.hpp"
54 #include "utilities/macros.hpp"
55 #include "utilities/powerOfTwo.hpp"
56 #include "utilities/vmError.hpp"
57
58 // Portions of code courtesy of Clifford Click
59
60 // Optimization - Graph Style
61
215 bool is_instance = t_oop->is_known_instance_field();
216 PhaseIterGVN *igvn = phase->is_IterGVN();
217 if (is_instance && igvn != nullptr && result->is_Phi()) {
218 PhiNode *mphi = result->as_Phi();
219 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
220 const TypePtr *t = mphi->adr_type();
221 bool do_split = false;
222 // In the following cases, Load memory input can be further optimized based on
223 // its precise address type
224 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
225 do_split = true;
226 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
227 const TypeOopPtr* mem_t =
228 t->is_oopptr()->cast_to_exactness(true)
229 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
230 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
231 if (t_oop->isa_aryptr()) {
232 mem_t = mem_t->is_aryptr()
233 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
234 ->cast_to_size(t_oop->is_aryptr()->size())
235 ->with_offset(t_oop->is_aryptr()->offset())
236 ->is_aryptr();
237 }
238 do_split = mem_t == t_oop;
239 }
240 if (do_split) {
241 // clone the Phi with our address type
242 result = mphi->split_out_instance(t_adr, igvn);
243 } else {
244 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
245 }
246 }
247 return result;
248 }
249
250 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
251 uint alias_idx = phase->C->get_alias_index(tp);
252 Node *mem = mmem;
253 #ifdef ASSERT
254 {
255 // Check that current type is consistent with the alias index used during graph construction
256 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
257 bool consistent = adr_check == nullptr || adr_check->empty() ||
258 phase->C->must_alias(adr_check, alias_idx );
259 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
260 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
261 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
262 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
263 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
264 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
265 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
266 // don't assert if it is dead code.
267 consistent = true;
268 }
269 if( !consistent ) {
270 st->print("alias_idx==%d, adr_check==", alias_idx);
271 if( adr_check == nullptr ) {
272 st->print("null");
273 } else {
274 adr_check->dump();
275 }
276 st->cr();
277 print_alias_types();
278 assert(consistent, "adr_check must match alias idx");
279 }
280 }
281 #endif
994 Node* ld = gvn.transform(load);
995 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
996 }
997
998 return load;
999 }
1000
1001 //------------------------------hash-------------------------------------------
1002 uint LoadNode::hash() const {
1003 // unroll addition of interesting fields
1004 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1005 }
1006
1007 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1008 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1009 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1010 bool is_stable_ary = FoldStableValues &&
1011 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1012 tp->isa_aryptr()->is_stable();
1013
1014 return (eliminate_boxing && non_volatile) || is_stable_ary;
1015 }
1016
1017 return false;
1018 }
1019
1020 LoadNode* LoadNode::pin_array_access_node() const {
1021 const TypePtr* adr_type = this->adr_type();
1022 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1023 return clone_pinned();
1024 }
1025 return nullptr;
1026 }
1027
1028 // Is the value loaded previously stored by an arraycopy? If so return
1029 // a load node that reads from the source array so we may be able to
1030 // optimize out the ArrayCopy node later.
1031 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1032 Node* ld_adr = in(MemNode::Address);
1033 intptr_t ld_off = 0;
1034 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1051 assert(ld_alloc != nullptr, "need an alloc");
1052 assert(addp->is_AddP(), "address must be addp");
1053 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1054 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1055 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1056 addp->set_req(AddPNode::Base, src);
1057 addp->set_req(AddPNode::Address, src);
1058 } else {
1059 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1060 ac->as_ArrayCopy()->is_copyof_validated() ||
1061 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1062 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1063 addp->set_req(AddPNode::Base, src);
1064 addp->set_req(AddPNode::Address, src);
1065
1066 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1067 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1068 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1069
1070 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1071 uint shift = exact_log2(type2aelembytes(ary_elem));
1072
1073 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1074 #ifdef _LP64
1075 diff = phase->transform(new ConvI2LNode(diff));
1076 #endif
1077 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1078
1079 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1080 addp->set_req(AddPNode::Offset, offset);
1081 }
1082 addp = phase->transform(addp);
1083 #ifdef ASSERT
1084 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1085 ld->_adr_type = adr_type;
1086 #endif
1087 ld->set_req(MemNode::Address, addp);
1088 ld->set_req(0, ctl);
1089 ld->set_req(MemNode::Memory, mem);
1090 return ld;
1091 }
1170 // Same base, same offset.
1171 // Possible improvement for arrays: check index value instead of absolute offset.
1172
1173 // At this point we have proven something like this setup:
1174 // B = << base >>
1175 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1176 // S = StoreQ(AddP( B , #Off), V)
1177 // (Actually, we haven't yet proven the Q's are the same.)
1178 // In other words, we are loading from a casted version of
1179 // the same pointer-and-offset that we stored to.
1180 // Casted version may carry a dependency and it is respected.
1181 // Thus, we are able to replace L by V.
1182 }
1183 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1184 if (store_Opcode() != st->Opcode()) {
1185 return nullptr;
1186 }
1187 // LoadVector/StoreVector needs additional check to ensure the types match.
1188 if (st->is_StoreVector()) {
1189 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1190 const TypeVect* out_vt = as_LoadVector()->vect_type();
1191 if (in_vt != out_vt) {
1192 return nullptr;
1193 }
1194 }
1195 return st->in(MemNode::ValueIn);
1196 }
1197
1198 // A load from a freshly-created object always returns zero.
1199 // (This can happen after LoadNode::Ideal resets the load's memory input
1200 // to find_captured_store, which returned InitializeNode::zero_memory.)
1201 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1202 (st->in(0) == ld_alloc) &&
1203 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1204 // return a zero value for the load's basic type
1205 // (This is one of the few places where a generic PhaseTransform
1206 // can create new nodes. Think of it as lazily manifesting
1207 // virtually pre-existing constants.)
1208 if (memory_type() != T_VOID) {
1209 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1210 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1211 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1212 // by the ArrayCopyNode.
1213 return phase->zerocon(memory_type());
1214 }
1215 } else {
1216 // TODO: materialize all-zero vector constant
1217 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1218 }
1219 }
1220
1221 // A load from an initialization barrier can match a captured store.
1222 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1223 InitializeNode* init = st->in(0)->as_Initialize();
1224 AllocateNode* alloc = init->allocation();
1225 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1226 // examine a captured store value
1227 st = init->find_captured_store(ld_off, memory_size(), phase);
1255 //----------------------is_instance_field_load_with_local_phi------------------
1256 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1257 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1258 in(Address)->is_AddP() ) {
1259 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1260 // Only instances and boxed values.
1261 if( t_oop != nullptr &&
1262 (t_oop->is_ptr_to_boxed_value() ||
1263 t_oop->is_known_instance_field()) &&
1264 t_oop->offset() != Type::OffsetBot &&
1265 t_oop->offset() != Type::OffsetTop) {
1266 return true;
1267 }
1268 }
1269 return false;
1270 }
1271
1272 //------------------------------Identity---------------------------------------
1273 // Loads are identity if previous store is to same address
1274 Node* LoadNode::Identity(PhaseGVN* phase) {
1275 // If the previous store-maker is the right kind of Store, and the store is
1276 // to the same address, then we are equal to the value stored.
1277 Node* mem = in(Memory);
1278 Node* value = can_see_stored_value(mem, phase);
1279 if( value ) {
1280 // byte, short & char stores truncate naturally.
1281 // A load has to load the truncated value which requires
1282 // some sort of masking operation and that requires an
1283 // Ideal call instead of an Identity call.
1284 if (memory_size() < BytesPerInt) {
1285 // If the input to the store does not fit with the load's result type,
1286 // it must be truncated via an Ideal call.
1287 if (!phase->type(value)->higher_equal(phase->type(this)))
1288 return this;
1289 }
1290 // (This works even when value is a Con, but LoadNode::Value
1291 // usually runs first, producing the singleton type of the Con.)
1292 if (!has_pinned_control_dependency() || value->is_Con()) {
1293 return value;
1294 } else {
2039 }
2040 }
2041
2042 // Don't do this for integer types. There is only potential profit if
2043 // the element type t is lower than _type; that is, for int types, if _type is
2044 // more restrictive than t. This only happens here if one is short and the other
2045 // char (both 16 bits), and in those cases we've made an intentional decision
2046 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2047 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2048 //
2049 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2050 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2051 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2052 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2053 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2054 // In fact, that could have been the original type of p1, and p1 could have
2055 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2056 // expression (LShiftL quux 3) independently optimized to the constant 8.
2057 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2058 && (_type->isa_vect() == nullptr)
2059 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2060 // t might actually be lower than _type, if _type is a unique
2061 // concrete subclass of abstract class t.
2062 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2063 const Type* jt = t->join_speculative(_type);
2064 // In any case, do not allow the join, per se, to empty out the type.
2065 if (jt->empty() && !t->empty()) {
2066 // This can happen if a interface-typed array narrows to a class type.
2067 jt = _type;
2068 }
2069 #ifdef ASSERT
2070 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2071 // The pointers in the autobox arrays are always non-null
2072 Node* base = adr->in(AddPNode::Base);
2073 if ((base != nullptr) && base->is_DecodeN()) {
2074 // Get LoadN node which loads IntegerCache.cache field
2075 base = base->in(1);
2076 }
2077 if ((base != nullptr) && base->is_Con()) {
2078 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2079 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2080 // It could be narrow oop
2081 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2082 }
2083 }
2084 }
2085 #endif
2086 return jt;
2087 }
2088 }
2089 } else if (tp->base() == Type::InstPtr) {
2090 assert( off != Type::OffsetBot ||
2091 // arrays can be cast to Objects
2092 !tp->isa_instptr() ||
2093 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2094 // unsafe field access may not have a constant offset
2095 C->has_unsafe_access(),
2096 "Field accesses must be precise" );
2097 // For oop loads, we expect the _type to be precise.
2098
2099 // Optimize loads from constant fields.
2100 const TypeInstPtr* tinst = tp->is_instptr();
2101 ciObject* const_oop = tinst->const_oop();
2102 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2103 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
2104 if (con_type != nullptr) {
2105 return con_type;
2106 }
2107 }
2108 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2109 assert(off != Type::OffsetBot ||
2110 !tp->isa_instklassptr() ||
2111 // arrays can be cast to Objects
2112 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2113 // also allow array-loading from the primary supertype
2114 // array during subtype checks
2115 Opcode() == Op_LoadKlass,
2116 "Field accesses must be precise");
2117 // For klass/static loads, we expect the _type to be precise
2118 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2119 /* With mirrors being an indirect in the Klass*
2120 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2121 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2122 *
2123 * So check the type and klass of the node before the LoadP.
2218 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2219 Node* value = can_see_stored_value(mem,phase);
2220 if (value != nullptr && value->is_Con()) {
2221 assert(value->bottom_type()->higher_equal(_type),"sanity");
2222 return value->bottom_type();
2223 }
2224 }
2225
2226 bool is_vect = (_type->isa_vect() != nullptr);
2227 if (is_instance && !is_vect) {
2228 // If we have an instance type and our memory input is the
2229 // programs's initial memory state, there is no matching store,
2230 // so just return a zero of the appropriate type -
2231 // except if it is vectorized - then we have no zero constant.
2232 Node *mem = in(MemNode::Memory);
2233 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2234 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2235 return Type::get_zero_type(_type->basic_type());
2236 }
2237 }
2238
2239 Node* alloc = is_new_object_mark_load();
2240 if (alloc != nullptr) {
2241 return TypeX::make(markWord::prototype().value());
2242 }
2243
2244 return _type;
2245 }
2246
2247 //------------------------------match_edge-------------------------------------
2248 // Do we Match on this edge index or not? Match only the address.
2249 uint LoadNode::match_edge(uint idx) const {
2250 return idx == MemNode::Address;
2251 }
2252
2253 //--------------------------LoadBNode::Ideal--------------------------------------
2254 //
2255 // If the previous store is to the same address as this load,
2256 // and the value stored was larger than a byte, replace this load
2257 // with the value stored truncated to a byte. If no truncation is
2258 // needed, the replacement is done in LoadNode::Identity().
2259 //
2260 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2261 Node* mem = in(MemNode::Memory);
2372 return LoadNode::Ideal(phase, can_reshape);
2373 }
2374
2375 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2376 Node* mem = in(MemNode::Memory);
2377 Node* value = can_see_stored_value(mem,phase);
2378 if (value != nullptr && value->is_Con() &&
2379 !value->bottom_type()->higher_equal(_type)) {
2380 // If the input to the store does not fit with the load's result type,
2381 // it must be truncated. We can't delay until Ideal call since
2382 // a singleton Value is needed for split_thru_phi optimization.
2383 int con = value->get_int();
2384 return TypeInt::make((con << 16) >> 16);
2385 }
2386 return LoadNode::Value(phase);
2387 }
2388
2389 //=============================================================================
2390 //----------------------------LoadKlassNode::make------------------------------
2391 // Polymorphic factory method:
2392 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2393 // sanity check the alias category against the created node type
2394 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2395 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2396 #ifdef _LP64
2397 if (adr_type->is_ptr_to_narrowklass()) {
2398 assert(UseCompressedClassPointers, "no compressed klasses");
2399 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2400 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2401 }
2402 #endif
2403 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2404 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2405 }
2406
2407 //------------------------------Value------------------------------------------
2408 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2409 return klass_value_common(phase);
2410 }
2411
2412 // In most cases, LoadKlassNode does not have the control input set. If the control
2419 // Either input is TOP ==> the result is TOP
2420 const Type *t1 = phase->type( in(MemNode::Memory) );
2421 if (t1 == Type::TOP) return Type::TOP;
2422 Node *adr = in(MemNode::Address);
2423 const Type *t2 = phase->type( adr );
2424 if (t2 == Type::TOP) return Type::TOP;
2425 const TypePtr *tp = t2->is_ptr();
2426 if (TypePtr::above_centerline(tp->ptr()) ||
2427 tp->ptr() == TypePtr::Null) return Type::TOP;
2428
2429 // Return a more precise klass, if possible
2430 const TypeInstPtr *tinst = tp->isa_instptr();
2431 if (tinst != nullptr) {
2432 ciInstanceKlass* ik = tinst->instance_klass();
2433 int offset = tinst->offset();
2434 if (ik == phase->C->env()->Class_klass()
2435 && (offset == java_lang_Class::klass_offset() ||
2436 offset == java_lang_Class::array_klass_offset())) {
2437 // We are loading a special hidden field from a Class mirror object,
2438 // the field which points to the VM's Klass metaobject.
2439 ciType* t = tinst->java_mirror_type();
2440 // java_mirror_type returns non-null for compile-time Class constants.
2441 if (t != nullptr) {
2442 // constant oop => constant klass
2443 if (offset == java_lang_Class::array_klass_offset()) {
2444 if (t->is_void()) {
2445 // We cannot create a void array. Since void is a primitive type return null
2446 // klass. Users of this result need to do a null check on the returned klass.
2447 return TypePtr::NULL_PTR;
2448 }
2449 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2450 }
2451 if (!t->is_klass()) {
2452 // a primitive Class (e.g., int.class) has null for a klass field
2453 return TypePtr::NULL_PTR;
2454 }
2455 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2456 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2457 }
2458 // non-constant mirror, so we can't tell what's going on
2459 }
2460 if (!tinst->is_loaded())
2461 return _type; // Bail out if not loaded
2462 if (offset == oopDesc::klass_offset_in_bytes()) {
2463 return tinst->as_klass_type(true);
2464 }
2465 }
2466
2467 // Check for loading klass from an array
2468 const TypeAryPtr *tary = tp->isa_aryptr();
2469 if (tary != nullptr &&
2470 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2471 return tary->as_klass_type(true);
2472 }
2473
2474 // Check for loading klass from an array klass
2475 const TypeKlassPtr *tkls = tp->isa_klassptr();
2476 if (tkls != nullptr && !StressReflectiveCode) {
2477 if (!tkls->is_loaded())
2478 return _type; // Bail out if not loaded
2479 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2480 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2481 // // Always returning precise element type is incorrect,
2482 // // e.g., element type could be object and array may contain strings
2483 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2484
2485 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2486 // according to the element type's subclassing.
2487 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2488 }
3448 merged_store->dump("\n", false, &ss);
3449 tty->print("%s", ss.as_string());
3450 }
3451 #endif
3452
3453 //------------------------------Ideal------------------------------------------
3454 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3455 // When a store immediately follows a relevant allocation/initialization,
3456 // try to capture it into the initialization, or hoist it above.
3457 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3458 Node* p = MemNode::Ideal_common(phase, can_reshape);
3459 if (p) return (p == NodeSentinel) ? nullptr : p;
3460
3461 Node* mem = in(MemNode::Memory);
3462 Node* address = in(MemNode::Address);
3463 Node* value = in(MemNode::ValueIn);
3464 // Back-to-back stores to same address? Fold em up. Generally
3465 // unsafe if I have intervening uses... Also disallowed for StoreCM
3466 // since they must follow each StoreP operation. Redundant StoreCMs
3467 // are eliminated just before matching in final_graph_reshape.
3468 {
3469 Node* st = mem;
3470 // If Store 'st' has more than one use, we cannot fold 'st' away.
3471 // For example, 'st' might be the final state at a conditional
3472 // return. Or, 'st' might be used by some node which is live at
3473 // the same time 'st' is live, which might be unschedulable. So,
3474 // require exactly ONE user until such time as we clone 'mem' for
3475 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3476 // true).
3477 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
3478 // Looking at a dead closed cycle of memory?
3479 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3480 assert(Opcode() == st->Opcode() ||
3481 st->Opcode() == Op_StoreVector ||
3482 Opcode() == Op_StoreVector ||
3483 st->Opcode() == Op_StoreVectorScatter ||
3484 Opcode() == Op_StoreVectorScatter ||
3485 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3486 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3487 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3488 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3489 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3490
3491 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3492 st->as_Store()->memory_size() <= this->memory_size()) {
3493 Node* use = st->raw_out(0);
3494 if (phase->is_IterGVN()) {
3495 phase->is_IterGVN()->rehash_node_delayed(use);
3496 }
3497 // It's OK to do this in the parser, since DU info is always accurate,
3498 // and the parser always refers to nodes via SafePointNode maps.
3499 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3500 return this;
3501 }
3502 st = st->in(MemNode::Memory);
3503 }
3504 }
3505
3506
3507 // Capture an unaliased, unconditional, simple store into an initializer.
3594 const StoreVectorNode* store_vector = as_StoreVector();
3595 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3596 const Node* store_indices = store_vector->indices();
3597 const Node* mem_indices = mem_vector->indices();
3598 const Node* store_mask = store_vector->mask();
3599 const Node* mem_mask = mem_vector->mask();
3600 // Ensure types, indices, and masks match
3601 if (store_vector->vect_type() == mem_vector->vect_type() &&
3602 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3603 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3604 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3605 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3606 result = mem;
3607 }
3608 }
3609 }
3610
3611 // Store of zero anywhere into a freshly-allocated object?
3612 // Then the store is useless.
3613 // (It must already have been captured by the InitializeNode.)
3614 if (result == this &&
3615 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3616 // a newly allocated object is already all-zeroes everywhere
3617 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3618 result = mem;
3619 }
3620
3621 if (result == this) {
3622 // the store may also apply to zero-bits in an earlier object
3623 Node* prev_mem = find_previous_store(phase);
3624 // Steps (a), (b): Walk past independent stores to find an exact match.
3625 if (prev_mem != nullptr) {
3626 Node* prev_val = can_see_stored_value(prev_mem, phase);
3627 if (prev_val != nullptr && prev_val == val) {
3628 // prev_val and val might differ by a cast; it would be good
3629 // to keep the more informative of the two.
3630 result = mem;
3631 }
3632 }
3633 }
3634 }
3635
3636 PhaseIterGVN* igvn = phase->is_IterGVN();
3637 if (result != this && igvn != nullptr) {
3638 MemBarNode* trailing = trailing_membar();
3639 if (trailing != nullptr) {
3640 #ifdef ASSERT
3641 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
3786 Node* StoreCMNode::Identity(PhaseGVN* phase) {
3787 // No need to card mark when storing a null ptr
3788 Node* my_store = in(MemNode::OopStore);
3789 if (my_store->is_Store()) {
3790 const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
3791 if( t1 == TypePtr::NULL_PTR ) {
3792 return in(MemNode::Memory);
3793 }
3794 }
3795 return this;
3796 }
3797
3798 //=============================================================================
3799 //------------------------------Ideal---------------------------------------
3800 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
3801 Node* progress = StoreNode::Ideal(phase, can_reshape);
3802 if (progress != nullptr) return progress;
3803
3804 Node* my_store = in(MemNode::OopStore);
3805 if (my_store->is_MergeMem()) {
3806 Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
3807 set_req_X(MemNode::OopStore, mem, phase);
3808 return this;
3809 }
3810
3811 return nullptr;
3812 }
3813
3814 //------------------------------Value-----------------------------------------
3815 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
3816 // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
3817 // If extra input is TOP ==> the result is TOP
3818 const Type* t = phase->type(in(MemNode::OopStore));
3819 if (t == Type::TOP) {
3820 return Type::TOP;
3821 }
3822 return StoreNode::Value(phase);
3823 }
3824
3825
3826 //=============================================================================
3827 //----------------------------------SCMemProjNode------------------------------
3828 const Type* SCMemProjNode::Value(PhaseGVN* phase) const
3959 // Clearing a short array is faster with stores
3960 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3961 // Already know this is a large node, do not try to ideal it
3962 if (_is_large) return nullptr;
3963
3964 const int unit = BytesPerLong;
3965 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3966 if (!t) return nullptr;
3967 if (!t->is_con()) return nullptr;
3968 intptr_t raw_count = t->get_con();
3969 intptr_t size = raw_count;
3970 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3971 // Clearing nothing uses the Identity call.
3972 // Negative clears are possible on dead ClearArrays
3973 // (see jck test stmt114.stmt11402.val).
3974 if (size <= 0 || size % unit != 0) return nullptr;
3975 intptr_t count = size / unit;
3976 // Length too long; communicate this to matchers and assemblers.
3977 // Assemblers are responsible to produce fast hardware clears for it.
3978 if (size > InitArrayShortSize) {
3979 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3980 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3981 return nullptr;
3982 }
3983 if (!IdealizeClearArrayNode) return nullptr;
3984 Node *mem = in(1);
3985 if( phase->type(mem)==Type::TOP ) return nullptr;
3986 Node *adr = in(3);
3987 const Type* at = phase->type(adr);
3988 if( at==Type::TOP ) return nullptr;
3989 const TypePtr* atp = at->isa_ptr();
3990 // adjust atp to be the correct array element address type
3991 if (atp == nullptr) atp = TypePtr::BOTTOM;
3992 else atp = atp->add_offset(Type::OffsetBot);
3993 // Get base for derived pointer purposes
3994 if( adr->Opcode() != Op_AddP ) Unimplemented();
3995 Node *base = adr->in(1);
3996
3997 Node *zero = phase->makecon(TypeLong::ZERO);
3998 Node *off = phase->MakeConX(BytesPerLong);
3999 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4000 count--;
4001 while( count-- ) {
4002 mem = phase->transform(mem);
4003 adr = phase->transform(new AddPNode(base,adr,off));
4004 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4005 }
4006 return mem;
4007 }
4008
4009 //----------------------------step_through----------------------------------
4010 // Return allocation input memory edge if it is different instance
4011 // or itself if it is the one we are looking for.
4012 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4013 Node* n = *np;
4014 assert(n->is_ClearArray(), "sanity");
4015 intptr_t offset;
4016 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4017 // This method is called only before Allocate nodes are expanded
4018 // during macro nodes expansion. Before that ClearArray nodes are
4019 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4020 // Allocate nodes are expanded) which follows allocations.
4021 assert(alloc != nullptr, "should have allocation");
4022 if (alloc->_idx == instance_id) {
4023 // Can not bypass initialization of the instance we are looking for.
4024 return false;
4025 }
4026 // Otherwise skip it.
4027 InitializeNode* init = alloc->initialization();
4028 if (init != nullptr)
4029 *np = init->in(TypeFunc::Memory);
4030 else
4031 *np = alloc->in(TypeFunc::Memory);
4032 return true;
4033 }
4034
4035 //----------------------------clear_memory-------------------------------------
4036 // Generate code to initialize object storage to zero.
4037 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4038 intptr_t start_offset,
4039 Node* end_offset,
4040 PhaseGVN* phase) {
4041 intptr_t offset = start_offset;
4042
4043 int unit = BytesPerLong;
4044 if ((offset % unit) != 0) {
4045 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
4046 adr = phase->transform(adr);
4047 const TypePtr* atp = TypeRawPtr::BOTTOM;
4048 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4049 mem = phase->transform(mem);
4050 offset += BytesPerInt;
4051 }
4052 assert((offset % unit) == 0, "");
4053
4054 // Initialize the remaining stuff, if any, with a ClearArray.
4055 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
4056 }
4057
4058 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4059 Node* start_offset,
4060 Node* end_offset,
4061 PhaseGVN* phase) {
4062 if (start_offset == end_offset) {
4063 // nothing to do
4064 return mem;
4065 }
4066
4067 int unit = BytesPerLong;
4068 Node* zbase = start_offset;
4069 Node* zend = end_offset;
4070
4071 // Scale to the unit required by the CPU:
4072 if (!Matcher::init_array_count_is_in_bytes) {
4073 Node* shift = phase->intcon(exact_log2(unit));
4074 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4075 zend = phase->transform(new URShiftXNode(zend, shift) );
4076 }
4077
4078 // Bulk clear double-words
4079 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4080 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
4081 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4082 return phase->transform(mem);
4083 }
4084
4085 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4086 intptr_t start_offset,
4087 intptr_t end_offset,
4088 PhaseGVN* phase) {
4089 if (start_offset == end_offset) {
4090 // nothing to do
4091 return mem;
4092 }
4093
4094 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4095 intptr_t done_offset = end_offset;
4096 if ((done_offset % BytesPerLong) != 0) {
4097 done_offset -= BytesPerInt;
4098 }
4099 if (done_offset > start_offset) {
4100 mem = clear_memory(ctl, mem, dest,
4101 start_offset, phase->MakeConX(done_offset), phase);
4102 }
4103 if (done_offset < end_offset) { // emit the final 32-bit store
4104 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
4105 adr = phase->transform(adr);
4106 const TypePtr* atp = TypeRawPtr::BOTTOM;
4107 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4108 mem = phase->transform(mem);
4109 done_offset += BytesPerInt;
4110 }
4111 assert(done_offset == end_offset, "");
4112 return mem;
4113 }
4114
4115 //=============================================================================
4116 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4117 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4118 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4119 #ifdef ASSERT
4120 , _pair_idx(0)
4121 #endif
4122 {
4123 init_class_id(Class_MemBar);
4124 Node* top = C->top();
4125 init_req(TypeFunc::I_O,top);
4126 init_req(TypeFunc::FramePtr,top);
4127 init_req(TypeFunc::ReturnAdr,top);
4233 PhaseIterGVN* igvn = phase->is_IterGVN();
4234 remove(igvn);
4235 // Must return either the original node (now dead) or a new node
4236 // (Do not return a top here, since that would break the uniqueness of top.)
4237 return new ConINode(TypeInt::ZERO);
4238 }
4239 }
4240 return progress ? this : nullptr;
4241 }
4242
4243 //------------------------------Value------------------------------------------
4244 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4245 if( !in(0) ) return Type::TOP;
4246 if( phase->type(in(0)) == Type::TOP )
4247 return Type::TOP;
4248 return TypeTuple::MEMBAR;
4249 }
4250
4251 //------------------------------match------------------------------------------
4252 // Construct projections for memory.
4253 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4254 switch (proj->_con) {
4255 case TypeFunc::Control:
4256 case TypeFunc::Memory:
4257 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4258 }
4259 ShouldNotReachHere();
4260 return nullptr;
4261 }
4262
4263 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4264 trailing->_kind = TrailingStore;
4265 leading->_kind = LeadingStore;
4266 #ifdef ASSERT
4267 trailing->_pair_idx = leading->_idx;
4268 leading->_pair_idx = leading->_idx;
4269 #endif
4270 }
4271
4272 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4273 trailing->_kind = TrailingLoadStore;
4520 return (req() > RawStores);
4521 }
4522
4523 void InitializeNode::set_complete(PhaseGVN* phase) {
4524 assert(!is_complete(), "caller responsibility");
4525 _is_complete = Complete;
4526
4527 // After this node is complete, it contains a bunch of
4528 // raw-memory initializations. There is no need for
4529 // it to have anything to do with non-raw memory effects.
4530 // Therefore, tell all non-raw users to re-optimize themselves,
4531 // after skipping the memory effects of this initialization.
4532 PhaseIterGVN* igvn = phase->is_IterGVN();
4533 if (igvn) igvn->add_users_to_worklist(this);
4534 }
4535
4536 // convenience function
4537 // return false if the init contains any stores already
4538 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4539 InitializeNode* init = initialization();
4540 if (init == nullptr || init->is_complete()) return false;
4541 init->remove_extra_zeroes();
4542 // for now, if this allocation has already collected any inits, bail:
4543 if (init->is_non_zero()) return false;
4544 init->set_complete(phase);
4545 return true;
4546 }
4547
4548 void InitializeNode::remove_extra_zeroes() {
4549 if (req() == RawStores) return;
4550 Node* zmem = zero_memory();
4551 uint fill = RawStores;
4552 for (uint i = fill; i < req(); i++) {
4553 Node* n = in(i);
4554 if (n->is_top() || n == zmem) continue; // skip
4555 if (fill < i) set_req(fill, n); // compact
4556 ++fill;
4557 }
4558 // delete any empty spaces created:
4559 while (fill < req()) {
4560 del_req(fill);
4699 // store node that we'd like to capture. We need to check
4700 // the uses of the MergeMemNode.
4701 mems.push(n);
4702 }
4703 } else if (n->is_Mem()) {
4704 Node* other_adr = n->in(MemNode::Address);
4705 if (other_adr == adr) {
4706 failed = true;
4707 break;
4708 } else {
4709 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4710 if (other_t_adr != nullptr) {
4711 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4712 if (other_alias_idx == alias_idx) {
4713 // A load from the same memory slice as the store right
4714 // after the InitializeNode. We check the control of the
4715 // object/array that is loaded from. If it's the same as
4716 // the store control then we cannot capture the store.
4717 assert(!n->is_Store(), "2 stores to same slice on same control?");
4718 Node* base = other_adr;
4719 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4720 base = base->in(AddPNode::Base);
4721 if (base != nullptr) {
4722 base = base->uncast();
4723 if (base->is_Proj() && base->in(0) == alloc) {
4724 failed = true;
4725 break;
4726 }
4727 }
4728 }
4729 }
4730 }
4731 } else {
4732 failed = true;
4733 break;
4734 }
4735 }
4736 }
4737 }
4738 if (failed) {
5285 // z's_done 12 16 16 16 12 16 12
5286 // z's_needed 12 16 16 16 16 16 16
5287 // zsize 0 0 0 0 4 0 4
5288 if (next_full_store < 0) {
5289 // Conservative tack: Zero to end of current word.
5290 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5291 } else {
5292 // Zero to beginning of next fully initialized word.
5293 // Or, don't zero at all, if we are already in that word.
5294 assert(next_full_store >= zeroes_needed, "must go forward");
5295 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5296 zeroes_needed = next_full_store;
5297 }
5298 }
5299
5300 if (zeroes_needed > zeroes_done) {
5301 intptr_t zsize = zeroes_needed - zeroes_done;
5302 // Do some incremental zeroing on rawmem, in parallel with inits.
5303 zeroes_done = align_down(zeroes_done, BytesPerInt);
5304 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5305 zeroes_done, zeroes_needed,
5306 phase);
5307 zeroes_done = zeroes_needed;
5308 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5309 do_zeroing = false; // leave the hole, next time
5310 }
5311 }
5312
5313 // Collect the store and move on:
5314 phase->replace_input_of(st, MemNode::Memory, inits);
5315 inits = st; // put it on the linearized chain
5316 set_req(i, zmem); // unhook from previous position
5317
5318 if (zeroes_done == st_off)
5319 zeroes_done = next_init_off;
5320
5321 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5322
5323 #ifdef ASSERT
5324 // Various order invariants. Weaker than stores_are_sane because
5344 remove_extra_zeroes(); // clear out all the zmems left over
5345 add_req(inits);
5346
5347 if (!(UseTLAB && ZeroTLAB)) {
5348 // If anything remains to be zeroed, zero it all now.
5349 zeroes_done = align_down(zeroes_done, BytesPerInt);
5350 // if it is the last unused 4 bytes of an instance, forget about it
5351 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5352 if (zeroes_done + BytesPerLong >= size_limit) {
5353 AllocateNode* alloc = allocation();
5354 assert(alloc != nullptr, "must be present");
5355 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5356 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5357 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5358 if (zeroes_done == k->layout_helper())
5359 zeroes_done = size_limit;
5360 }
5361 }
5362 if (zeroes_done < size_limit) {
5363 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5364 zeroes_done, size_in_bytes, phase);
5365 }
5366 }
5367
5368 set_complete(phase);
5369 return rawmem;
5370 }
5371
5372
5373 #ifdef ASSERT
5374 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5375 if (is_complete())
5376 return true; // stores could be anything at this point
5377 assert(allocation() != nullptr, "must be present");
5378 intptr_t last_off = allocation()->minimum_header_size();
5379 for (uint i = InitializeNode::RawStores; i < req(); i++) {
5380 Node* st = in(i);
5381 intptr_t st_off = get_store_offset(st, phase);
5382 if (st_off < 0) continue; // ignore dead garbage
5383 if (last_off > st_off) {
|
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "ci/ciFlatArrayKlass.hpp"
28 #include "classfile/javaClasses.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "compiler/compileLog.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/c2/barrierSetC2.hpp"
33 #include "gc/shared/tlab_globals.hpp"
34 #include "memory/allocation.inline.hpp"
35 #include "memory/resourceArea.hpp"
36 #include "oops/objArrayKlass.hpp"
37 #include "opto/addnode.hpp"
38 #include "opto/arraycopynode.hpp"
39 #include "opto/cfgnode.hpp"
40 #include "opto/regalloc.hpp"
41 #include "opto/compile.hpp"
42 #include "opto/connode.hpp"
43 #include "opto/convertnode.hpp"
44 #include "opto/inlinetypenode.hpp"
45 #include "opto/loopnode.hpp"
46 #include "opto/machnode.hpp"
47 #include "opto/matcher.hpp"
48 #include "opto/memnode.hpp"
49 #include "opto/mulnode.hpp"
50 #include "opto/narrowptrnode.hpp"
51 #include "opto/phaseX.hpp"
52 #include "opto/regmask.hpp"
53 #include "opto/rootnode.hpp"
54 #include "opto/vectornode.hpp"
55 #include "utilities/align.hpp"
56 #include "utilities/copy.hpp"
57 #include "utilities/macros.hpp"
58 #include "utilities/powerOfTwo.hpp"
59 #include "utilities/vmError.hpp"
60
61 // Portions of code courtesy of Clifford Click
62
63 // Optimization - Graph Style
64
218 bool is_instance = t_oop->is_known_instance_field();
219 PhaseIterGVN *igvn = phase->is_IterGVN();
220 if (is_instance && igvn != nullptr && result->is_Phi()) {
221 PhiNode *mphi = result->as_Phi();
222 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
223 const TypePtr *t = mphi->adr_type();
224 bool do_split = false;
225 // In the following cases, Load memory input can be further optimized based on
226 // its precise address type
227 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
228 do_split = true;
229 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
230 const TypeOopPtr* mem_t =
231 t->is_oopptr()->cast_to_exactness(true)
232 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
233 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
234 if (t_oop->isa_aryptr()) {
235 mem_t = mem_t->is_aryptr()
236 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
237 ->cast_to_size(t_oop->is_aryptr()->size())
238 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
239 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
240 ->with_offset(t_oop->is_aryptr()->offset())
241 ->is_aryptr();
242 }
243 do_split = mem_t == t_oop;
244 }
245 if (do_split) {
246 // clone the Phi with our address type
247 result = mphi->split_out_instance(t_adr, igvn);
248 } else {
249 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
250 }
251 }
252 return result;
253 }
254
255 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
256 uint alias_idx = phase->C->get_alias_index(tp);
257 Node *mem = mmem;
258 #ifdef ASSERT
259 {
260 // Check that current type is consistent with the alias index used during graph construction
261 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
262 bool consistent = adr_check == nullptr || adr_check->empty() ||
263 phase->C->must_alias(adr_check, alias_idx );
264 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
265 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
266 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
267 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
268 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
269 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
270 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
271 // don't assert if it is dead code.
272 consistent = true;
273 }
274 if( !consistent ) {
275 st->print("alias_idx==%d, adr_check==", alias_idx);
276 if( adr_check == nullptr ) {
277 st->print("null");
278 } else {
279 adr_check->dump();
280 }
281 st->cr();
282 print_alias_types();
283 assert(consistent, "adr_check must match alias idx");
284 }
285 }
286 #endif
999 Node* ld = gvn.transform(load);
1000 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1001 }
1002
1003 return load;
1004 }
1005
1006 //------------------------------hash-------------------------------------------
1007 uint LoadNode::hash() const {
1008 // unroll addition of interesting fields
1009 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1010 }
1011
1012 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1013 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1014 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1015 bool is_stable_ary = FoldStableValues &&
1016 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1017 tp->isa_aryptr()->is_stable();
1018
1019 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1020 }
1021
1022 return false;
1023 }
1024
1025 LoadNode* LoadNode::pin_array_access_node() const {
1026 const TypePtr* adr_type = this->adr_type();
1027 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1028 return clone_pinned();
1029 }
1030 return nullptr;
1031 }
1032
1033 // Is the value loaded previously stored by an arraycopy? If so return
1034 // a load node that reads from the source array so we may be able to
1035 // optimize out the ArrayCopy node later.
1036 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1037 Node* ld_adr = in(MemNode::Address);
1038 intptr_t ld_off = 0;
1039 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1056 assert(ld_alloc != nullptr, "need an alloc");
1057 assert(addp->is_AddP(), "address must be addp");
1058 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1059 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1060 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1061 addp->set_req(AddPNode::Base, src);
1062 addp->set_req(AddPNode::Address, src);
1063 } else {
1064 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1065 ac->as_ArrayCopy()->is_copyof_validated() ||
1066 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1067 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1068 addp->set_req(AddPNode::Base, src);
1069 addp->set_req(AddPNode::Address, src);
1070
1071 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1072 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1073 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1074
1075 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1076 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1077
1078 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1079 #ifdef _LP64
1080 diff = phase->transform(new ConvI2LNode(diff));
1081 #endif
1082 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1083
1084 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1085 addp->set_req(AddPNode::Offset, offset);
1086 }
1087 addp = phase->transform(addp);
1088 #ifdef ASSERT
1089 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1090 ld->_adr_type = adr_type;
1091 #endif
1092 ld->set_req(MemNode::Address, addp);
1093 ld->set_req(0, ctl);
1094 ld->set_req(MemNode::Memory, mem);
1095 return ld;
1096 }
1175 // Same base, same offset.
1176 // Possible improvement for arrays: check index value instead of absolute offset.
1177
1178 // At this point we have proven something like this setup:
1179 // B = << base >>
1180 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1181 // S = StoreQ(AddP( B , #Off), V)
1182 // (Actually, we haven't yet proven the Q's are the same.)
1183 // In other words, we are loading from a casted version of
1184 // the same pointer-and-offset that we stored to.
1185 // Casted version may carry a dependency and it is respected.
1186 // Thus, we are able to replace L by V.
1187 }
1188 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1189 if (store_Opcode() != st->Opcode()) {
1190 return nullptr;
1191 }
1192 // LoadVector/StoreVector needs additional check to ensure the types match.
1193 if (st->is_StoreVector()) {
1194 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1195 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1196 if (in_vt != out_vt) {
1197 return nullptr;
1198 }
1199 }
1200 return st->in(MemNode::ValueIn);
1201 }
1202
1203 // A load from a freshly-created object always returns zero.
1204 // (This can happen after LoadNode::Ideal resets the load's memory input
1205 // to find_captured_store, which returned InitializeNode::zero_memory.)
1206 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1207 (st->in(0) == ld_alloc) &&
1208 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1209 // return a zero value for the load's basic type
1210 // (This is one of the few places where a generic PhaseTransform
1211 // can create new nodes. Think of it as lazily manifesting
1212 // virtually pre-existing constants.)
1213 Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1214 if (default_value != nullptr) {
1215 return default_value;
1216 }
1217 assert(ld_alloc->in(AllocateNode::RawDefaultValue) == nullptr, "default value may not be null");
1218 if (memory_type() != T_VOID) {
1219 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1220 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1221 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1222 // by the ArrayCopyNode.
1223 return phase->zerocon(memory_type());
1224 }
1225 } else {
1226 // TODO: materialize all-zero vector constant
1227 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1228 }
1229 }
1230
1231 // A load from an initialization barrier can match a captured store.
1232 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1233 InitializeNode* init = st->in(0)->as_Initialize();
1234 AllocateNode* alloc = init->allocation();
1235 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1236 // examine a captured store value
1237 st = init->find_captured_store(ld_off, memory_size(), phase);
1265 //----------------------is_instance_field_load_with_local_phi------------------
1266 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1267 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1268 in(Address)->is_AddP() ) {
1269 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1270 // Only instances and boxed values.
1271 if( t_oop != nullptr &&
1272 (t_oop->is_ptr_to_boxed_value() ||
1273 t_oop->is_known_instance_field()) &&
1274 t_oop->offset() != Type::OffsetBot &&
1275 t_oop->offset() != Type::OffsetTop) {
1276 return true;
1277 }
1278 }
1279 return false;
1280 }
1281
1282 //------------------------------Identity---------------------------------------
1283 // Loads are identity if previous store is to same address
1284 Node* LoadNode::Identity(PhaseGVN* phase) {
1285 // Loading from an InlineType? The InlineType has the values of
1286 // all fields as input. Look for the field with matching offset.
1287 Node* addr = in(Address);
1288 intptr_t offset;
1289 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1290 if (base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1291 Node* value = base->as_InlineType()->field_value_by_offset((int)offset, true);
1292 if (value != nullptr) {
1293 if (Opcode() == Op_LoadN) {
1294 // Encode oop value if we are loading a narrow oop
1295 assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1296 value = phase->transform(new EncodePNode(value, bottom_type()));
1297 }
1298 return value;
1299 }
1300 }
1301
1302 // If the previous store-maker is the right kind of Store, and the store is
1303 // to the same address, then we are equal to the value stored.
1304 Node* mem = in(Memory);
1305 Node* value = can_see_stored_value(mem, phase);
1306 if( value ) {
1307 // byte, short & char stores truncate naturally.
1308 // A load has to load the truncated value which requires
1309 // some sort of masking operation and that requires an
1310 // Ideal call instead of an Identity call.
1311 if (memory_size() < BytesPerInt) {
1312 // If the input to the store does not fit with the load's result type,
1313 // it must be truncated via an Ideal call.
1314 if (!phase->type(value)->higher_equal(phase->type(this)))
1315 return this;
1316 }
1317 // (This works even when value is a Con, but LoadNode::Value
1318 // usually runs first, producing the singleton type of the Con.)
1319 if (!has_pinned_control_dependency() || value->is_Con()) {
1320 return value;
1321 } else {
2066 }
2067 }
2068
2069 // Don't do this for integer types. There is only potential profit if
2070 // the element type t is lower than _type; that is, for int types, if _type is
2071 // more restrictive than t. This only happens here if one is short and the other
2072 // char (both 16 bits), and in those cases we've made an intentional decision
2073 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2074 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2075 //
2076 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2077 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2078 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2079 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2080 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2081 // In fact, that could have been the original type of p1, and p1 could have
2082 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2083 // expression (LShiftL quux 3) independently optimized to the constant 8.
2084 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2085 && (_type->isa_vect() == nullptr)
2086 && !ary->is_flat()
2087 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2088 // t might actually be lower than _type, if _type is a unique
2089 // concrete subclass of abstract class t.
2090 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2091 const Type* jt = t->join_speculative(_type);
2092 // In any case, do not allow the join, per se, to empty out the type.
2093 if (jt->empty() && !t->empty()) {
2094 // This can happen if a interface-typed array narrows to a class type.
2095 jt = _type;
2096 }
2097 #ifdef ASSERT
2098 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2099 // The pointers in the autobox arrays are always non-null
2100 Node* base = adr->in(AddPNode::Base);
2101 if ((base != nullptr) && base->is_DecodeN()) {
2102 // Get LoadN node which loads IntegerCache.cache field
2103 base = base->in(1);
2104 }
2105 if ((base != nullptr) && base->is_Con()) {
2106 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2107 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2108 // It could be narrow oop
2109 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2110 }
2111 }
2112 }
2113 #endif
2114 return jt;
2115 }
2116 }
2117 } else if (tp->base() == Type::InstPtr) {
2118 assert( off != Type::OffsetBot ||
2119 // arrays can be cast to Objects
2120 !tp->isa_instptr() ||
2121 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2122 // Default value load
2123 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2124 // unsafe field access may not have a constant offset
2125 C->has_unsafe_access(),
2126 "Field accesses must be precise" );
2127 // For oop loads, we expect the _type to be precise.
2128
2129 const TypeInstPtr* tinst = tp->is_instptr();
2130 BasicType bt = memory_type();
2131
2132 // Optimize loads from constant fields.
2133 ciObject* const_oop = tinst->const_oop();
2134 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2135 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2136 if (con_type != nullptr) {
2137 return con_type;
2138 }
2139 }
2140 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2141 assert(off != Type::OffsetBot ||
2142 !tp->isa_instklassptr() ||
2143 // arrays can be cast to Objects
2144 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2145 // also allow array-loading from the primary supertype
2146 // array during subtype checks
2147 Opcode() == Op_LoadKlass,
2148 "Field accesses must be precise");
2149 // For klass/static loads, we expect the _type to be precise
2150 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2151 /* With mirrors being an indirect in the Klass*
2152 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2153 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2154 *
2155 * So check the type and klass of the node before the LoadP.
2250 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2251 Node* value = can_see_stored_value(mem,phase);
2252 if (value != nullptr && value->is_Con()) {
2253 assert(value->bottom_type()->higher_equal(_type),"sanity");
2254 return value->bottom_type();
2255 }
2256 }
2257
2258 bool is_vect = (_type->isa_vect() != nullptr);
2259 if (is_instance && !is_vect) {
2260 // If we have an instance type and our memory input is the
2261 // programs's initial memory state, there is no matching store,
2262 // so just return a zero of the appropriate type -
2263 // except if it is vectorized - then we have no zero constant.
2264 Node *mem = in(MemNode::Memory);
2265 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2266 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2267 return Type::get_zero_type(_type->basic_type());
2268 }
2269 }
2270 Node* alloc = is_new_object_mark_load();
2271 if (alloc != nullptr) {
2272 if (EnableValhalla) {
2273 // The mark word may contain property bits (inline, flat, null-free)
2274 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2275 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2276 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2277 return TypeX::make(tkls->exact_klass()->prototype_header().value());
2278 }
2279 } else {
2280 return TypeX::make(markWord::prototype().value());
2281 }
2282 }
2283
2284 return _type;
2285 }
2286
2287 //------------------------------match_edge-------------------------------------
2288 // Do we Match on this edge index or not? Match only the address.
2289 uint LoadNode::match_edge(uint idx) const {
2290 return idx == MemNode::Address;
2291 }
2292
2293 //--------------------------LoadBNode::Ideal--------------------------------------
2294 //
2295 // If the previous store is to the same address as this load,
2296 // and the value stored was larger than a byte, replace this load
2297 // with the value stored truncated to a byte. If no truncation is
2298 // needed, the replacement is done in LoadNode::Identity().
2299 //
2300 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2301 Node* mem = in(MemNode::Memory);
2412 return LoadNode::Ideal(phase, can_reshape);
2413 }
2414
2415 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2416 Node* mem = in(MemNode::Memory);
2417 Node* value = can_see_stored_value(mem,phase);
2418 if (value != nullptr && value->is_Con() &&
2419 !value->bottom_type()->higher_equal(_type)) {
2420 // If the input to the store does not fit with the load's result type,
2421 // it must be truncated. We can't delay until Ideal call since
2422 // a singleton Value is needed for split_thru_phi optimization.
2423 int con = value->get_int();
2424 return TypeInt::make((con << 16) >> 16);
2425 }
2426 return LoadNode::Value(phase);
2427 }
2428
2429 //=============================================================================
2430 //----------------------------LoadKlassNode::make------------------------------
2431 // Polymorphic factory method:
2432 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2433 const TypeKlassPtr* tk) {
2434 // sanity check the alias category against the created node type
2435 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2436 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2437 #ifdef _LP64
2438 if (adr_type->is_ptr_to_narrowklass()) {
2439 assert(UseCompressedClassPointers, "no compressed klasses");
2440 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2441 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2442 }
2443 #endif
2444 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2445 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2446 }
2447
2448 //------------------------------Value------------------------------------------
2449 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2450 return klass_value_common(phase);
2451 }
2452
2453 // In most cases, LoadKlassNode does not have the control input set. If the control
2460 // Either input is TOP ==> the result is TOP
2461 const Type *t1 = phase->type( in(MemNode::Memory) );
2462 if (t1 == Type::TOP) return Type::TOP;
2463 Node *adr = in(MemNode::Address);
2464 const Type *t2 = phase->type( adr );
2465 if (t2 == Type::TOP) return Type::TOP;
2466 const TypePtr *tp = t2->is_ptr();
2467 if (TypePtr::above_centerline(tp->ptr()) ||
2468 tp->ptr() == TypePtr::Null) return Type::TOP;
2469
2470 // Return a more precise klass, if possible
2471 const TypeInstPtr *tinst = tp->isa_instptr();
2472 if (tinst != nullptr) {
2473 ciInstanceKlass* ik = tinst->instance_klass();
2474 int offset = tinst->offset();
2475 if (ik == phase->C->env()->Class_klass()
2476 && (offset == java_lang_Class::klass_offset() ||
2477 offset == java_lang_Class::array_klass_offset())) {
2478 // We are loading a special hidden field from a Class mirror object,
2479 // the field which points to the VM's Klass metaobject.
2480 bool is_null_free_array = false;
2481 ciType* t = tinst->java_mirror_type(&is_null_free_array);
2482 // java_mirror_type returns non-null for compile-time Class constants.
2483 if (t != nullptr) {
2484 // constant oop => constant klass
2485 if (offset == java_lang_Class::array_klass_offset()) {
2486 if (t->is_void()) {
2487 // We cannot create a void array. Since void is a primitive type return null
2488 // klass. Users of this result need to do a null check on the returned klass.
2489 return TypePtr::NULL_PTR;
2490 }
2491 const TypeKlassPtr* tklass = TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2492 if (is_null_free_array) {
2493 tklass = tklass->is_aryklassptr()->cast_to_null_free();
2494 }
2495 return tklass;
2496 }
2497 if (!t->is_klass()) {
2498 // a primitive Class (e.g., int.class) has null for a klass field
2499 return TypePtr::NULL_PTR;
2500 }
2501 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2502 const TypeKlassPtr* tklass = TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2503 if (is_null_free_array) {
2504 tklass = tklass->is_aryklassptr()->cast_to_null_free();
2505 }
2506 return tklass;
2507 }
2508 // non-constant mirror, so we can't tell what's going on
2509 }
2510 if (!tinst->is_loaded())
2511 return _type; // Bail out if not loaded
2512 if (offset == oopDesc::klass_offset_in_bytes()) {
2513 return tinst->as_klass_type(true);
2514 }
2515 }
2516
2517 // Check for loading klass from an array
2518 const TypeAryPtr* tary = tp->isa_aryptr();
2519 if (tary != nullptr &&
2520 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2521 return tary->as_klass_type(true);
2522 }
2523
2524 // Check for loading klass from an array klass
2525 const TypeKlassPtr *tkls = tp->isa_klassptr();
2526 if (tkls != nullptr && !StressReflectiveCode) {
2527 if (!tkls->is_loaded())
2528 return _type; // Bail out if not loaded
2529 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2530 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2531 // // Always returning precise element type is incorrect,
2532 // // e.g., element type could be object and array may contain strings
2533 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2534
2535 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2536 // according to the element type's subclassing.
2537 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2538 }
3498 merged_store->dump("\n", false, &ss);
3499 tty->print("%s", ss.as_string());
3500 }
3501 #endif
3502
3503 //------------------------------Ideal------------------------------------------
3504 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3505 // When a store immediately follows a relevant allocation/initialization,
3506 // try to capture it into the initialization, or hoist it above.
3507 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3508 Node* p = MemNode::Ideal_common(phase, can_reshape);
3509 if (p) return (p == NodeSentinel) ? nullptr : p;
3510
3511 Node* mem = in(MemNode::Memory);
3512 Node* address = in(MemNode::Address);
3513 Node* value = in(MemNode::ValueIn);
3514 // Back-to-back stores to same address? Fold em up. Generally
3515 // unsafe if I have intervening uses... Also disallowed for StoreCM
3516 // since they must follow each StoreP operation. Redundant StoreCMs
3517 // are eliminated just before matching in final_graph_reshape.
3518 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3519 Node* st = mem;
3520 // If Store 'st' has more than one use, we cannot fold 'st' away.
3521 // For example, 'st' might be the final state at a conditional
3522 // return. Or, 'st' might be used by some node which is live at
3523 // the same time 'st' is live, which might be unschedulable. So,
3524 // require exactly ONE user until such time as we clone 'mem' for
3525 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3526 // true).
3527 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
3528 // Looking at a dead closed cycle of memory?
3529 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3530 assert(Opcode() == st->Opcode() ||
3531 st->Opcode() == Op_StoreVector ||
3532 Opcode() == Op_StoreVector ||
3533 st->Opcode() == Op_StoreVectorScatter ||
3534 Opcode() == Op_StoreVectorScatter ||
3535 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3536 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3537 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3538 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3539 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3540 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3541
3542 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3543 st->as_Store()->memory_size() <= this->memory_size()) {
3544 Node* use = st->raw_out(0);
3545 if (phase->is_IterGVN()) {
3546 phase->is_IterGVN()->rehash_node_delayed(use);
3547 }
3548 // It's OK to do this in the parser, since DU info is always accurate,
3549 // and the parser always refers to nodes via SafePointNode maps.
3550 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3551 return this;
3552 }
3553 st = st->in(MemNode::Memory);
3554 }
3555 }
3556
3557
3558 // Capture an unaliased, unconditional, simple store into an initializer.
3645 const StoreVectorNode* store_vector = as_StoreVector();
3646 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3647 const Node* store_indices = store_vector->indices();
3648 const Node* mem_indices = mem_vector->indices();
3649 const Node* store_mask = store_vector->mask();
3650 const Node* mem_mask = mem_vector->mask();
3651 // Ensure types, indices, and masks match
3652 if (store_vector->vect_type() == mem_vector->vect_type() &&
3653 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3654 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3655 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3656 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3657 result = mem;
3658 }
3659 }
3660 }
3661
3662 // Store of zero anywhere into a freshly-allocated object?
3663 // Then the store is useless.
3664 // (It must already have been captured by the InitializeNode.)
3665 if (result == this && ReduceFieldZeroing) {
3666 // a newly allocated object is already all-zeroes everywhere
3667 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3668 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
3669 result = mem;
3670 }
3671
3672 if (result == this && phase->type(val)->is_zero_type()) {
3673 // the store may also apply to zero-bits in an earlier object
3674 Node* prev_mem = find_previous_store(phase);
3675 // Steps (a), (b): Walk past independent stores to find an exact match.
3676 if (prev_mem != nullptr) {
3677 Node* prev_val = can_see_stored_value(prev_mem, phase);
3678 if (prev_val != nullptr && prev_val == val) {
3679 // prev_val and val might differ by a cast; it would be good
3680 // to keep the more informative of the two.
3681 result = mem;
3682 }
3683 }
3684 }
3685 }
3686
3687 PhaseIterGVN* igvn = phase->is_IterGVN();
3688 if (result != this && igvn != nullptr) {
3689 MemBarNode* trailing = trailing_membar();
3690 if (trailing != nullptr) {
3691 #ifdef ASSERT
3692 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
3837 Node* StoreCMNode::Identity(PhaseGVN* phase) {
3838 // No need to card mark when storing a null ptr
3839 Node* my_store = in(MemNode::OopStore);
3840 if (my_store->is_Store()) {
3841 const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
3842 if( t1 == TypePtr::NULL_PTR ) {
3843 return in(MemNode::Memory);
3844 }
3845 }
3846 return this;
3847 }
3848
3849 //=============================================================================
3850 //------------------------------Ideal---------------------------------------
3851 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
3852 Node* progress = StoreNode::Ideal(phase, can_reshape);
3853 if (progress != nullptr) return progress;
3854
3855 Node* my_store = in(MemNode::OopStore);
3856 if (my_store->is_MergeMem()) {
3857 if (oop_alias_idx() != phase->C->get_alias_index(TypeAryPtr::INLINES) ||
3858 phase->C->flat_accesses_share_alias()) {
3859 // The alias that was recorded is no longer accurate enough.
3860 Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
3861 set_req_X(MemNode::OopStore, mem, phase);
3862 return this;
3863 }
3864 }
3865
3866 return nullptr;
3867 }
3868
3869 //------------------------------Value-----------------------------------------
3870 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
3871 // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
3872 // If extra input is TOP ==> the result is TOP
3873 const Type* t = phase->type(in(MemNode::OopStore));
3874 if (t == Type::TOP) {
3875 return Type::TOP;
3876 }
3877 return StoreNode::Value(phase);
3878 }
3879
3880
3881 //=============================================================================
3882 //----------------------------------SCMemProjNode------------------------------
3883 const Type* SCMemProjNode::Value(PhaseGVN* phase) const
4014 // Clearing a short array is faster with stores
4015 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4016 // Already know this is a large node, do not try to ideal it
4017 if (_is_large) return nullptr;
4018
4019 const int unit = BytesPerLong;
4020 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4021 if (!t) return nullptr;
4022 if (!t->is_con()) return nullptr;
4023 intptr_t raw_count = t->get_con();
4024 intptr_t size = raw_count;
4025 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4026 // Clearing nothing uses the Identity call.
4027 // Negative clears are possible on dead ClearArrays
4028 // (see jck test stmt114.stmt11402.val).
4029 if (size <= 0 || size % unit != 0) return nullptr;
4030 intptr_t count = size / unit;
4031 // Length too long; communicate this to matchers and assemblers.
4032 // Assemblers are responsible to produce fast hardware clears for it.
4033 if (size > InitArrayShortSize) {
4034 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4035 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4036 return nullptr;
4037 }
4038 if (!IdealizeClearArrayNode) return nullptr;
4039 Node *mem = in(1);
4040 if( phase->type(mem)==Type::TOP ) return nullptr;
4041 Node *adr = in(3);
4042 const Type* at = phase->type(adr);
4043 if( at==Type::TOP ) return nullptr;
4044 const TypePtr* atp = at->isa_ptr();
4045 // adjust atp to be the correct array element address type
4046 if (atp == nullptr) atp = TypePtr::BOTTOM;
4047 else atp = atp->add_offset(Type::OffsetBot);
4048 // Get base for derived pointer purposes
4049 if( adr->Opcode() != Op_AddP ) Unimplemented();
4050 Node *base = adr->in(1);
4051
4052 Node *val = in(4);
4053 Node *off = phase->MakeConX(BytesPerLong);
4054 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4055 count--;
4056 while( count-- ) {
4057 mem = phase->transform(mem);
4058 adr = phase->transform(new AddPNode(base,adr,off));
4059 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4060 }
4061 return mem;
4062 }
4063
4064 //----------------------------step_through----------------------------------
4065 // Return allocation input memory edge if it is different instance
4066 // or itself if it is the one we are looking for.
4067 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4068 Node* n = *np;
4069 assert(n->is_ClearArray(), "sanity");
4070 intptr_t offset;
4071 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4072 // This method is called only before Allocate nodes are expanded
4073 // during macro nodes expansion. Before that ClearArray nodes are
4074 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4075 // Allocate nodes are expanded) which follows allocations.
4076 assert(alloc != nullptr, "should have allocation");
4077 if (alloc->_idx == instance_id) {
4078 // Can not bypass initialization of the instance we are looking for.
4079 return false;
4080 }
4081 // Otherwise skip it.
4082 InitializeNode* init = alloc->initialization();
4083 if (init != nullptr)
4084 *np = init->in(TypeFunc::Memory);
4085 else
4086 *np = alloc->in(TypeFunc::Memory);
4087 return true;
4088 }
4089
4090 //----------------------------clear_memory-------------------------------------
4091 // Generate code to initialize object storage to zero.
4092 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4093 Node* val,
4094 Node* raw_val,
4095 intptr_t start_offset,
4096 Node* end_offset,
4097 PhaseGVN* phase) {
4098 intptr_t offset = start_offset;
4099
4100 int unit = BytesPerLong;
4101 if ((offset % unit) != 0) {
4102 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
4103 adr = phase->transform(adr);
4104 const TypePtr* atp = TypeRawPtr::BOTTOM;
4105 if (val != nullptr) {
4106 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4107 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4108 } else {
4109 assert(raw_val == nullptr, "val may not be null");
4110 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4111 }
4112 mem = phase->transform(mem);
4113 offset += BytesPerInt;
4114 }
4115 assert((offset % unit) == 0, "");
4116
4117 // Initialize the remaining stuff, if any, with a ClearArray.
4118 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
4119 }
4120
4121 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4122 Node* raw_val,
4123 Node* start_offset,
4124 Node* end_offset,
4125 PhaseGVN* phase) {
4126 if (start_offset == end_offset) {
4127 // nothing to do
4128 return mem;
4129 }
4130
4131 int unit = BytesPerLong;
4132 Node* zbase = start_offset;
4133 Node* zend = end_offset;
4134
4135 // Scale to the unit required by the CPU:
4136 if (!Matcher::init_array_count_is_in_bytes) {
4137 Node* shift = phase->intcon(exact_log2(unit));
4138 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4139 zend = phase->transform(new URShiftXNode(zend, shift) );
4140 }
4141
4142 // Bulk clear double-words
4143 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4144 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
4145 if (raw_val == nullptr) {
4146 raw_val = phase->MakeConX(0);
4147 }
4148 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4149 return phase->transform(mem);
4150 }
4151
4152 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4153 Node* val,
4154 Node* raw_val,
4155 intptr_t start_offset,
4156 intptr_t end_offset,
4157 PhaseGVN* phase) {
4158 if (start_offset == end_offset) {
4159 // nothing to do
4160 return mem;
4161 }
4162
4163 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4164 intptr_t done_offset = end_offset;
4165 if ((done_offset % BytesPerLong) != 0) {
4166 done_offset -= BytesPerInt;
4167 }
4168 if (done_offset > start_offset) {
4169 mem = clear_memory(ctl, mem, dest, val, raw_val,
4170 start_offset, phase->MakeConX(done_offset), phase);
4171 }
4172 if (done_offset < end_offset) { // emit the final 32-bit store
4173 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
4174 adr = phase->transform(adr);
4175 const TypePtr* atp = TypeRawPtr::BOTTOM;
4176 if (val != nullptr) {
4177 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4178 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4179 } else {
4180 assert(raw_val == nullptr, "val may not be null");
4181 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4182 }
4183 mem = phase->transform(mem);
4184 done_offset += BytesPerInt;
4185 }
4186 assert(done_offset == end_offset, "");
4187 return mem;
4188 }
4189
4190 //=============================================================================
4191 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4192 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4193 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4194 #ifdef ASSERT
4195 , _pair_idx(0)
4196 #endif
4197 {
4198 init_class_id(Class_MemBar);
4199 Node* top = C->top();
4200 init_req(TypeFunc::I_O,top);
4201 init_req(TypeFunc::FramePtr,top);
4202 init_req(TypeFunc::ReturnAdr,top);
4308 PhaseIterGVN* igvn = phase->is_IterGVN();
4309 remove(igvn);
4310 // Must return either the original node (now dead) or a new node
4311 // (Do not return a top here, since that would break the uniqueness of top.)
4312 return new ConINode(TypeInt::ZERO);
4313 }
4314 }
4315 return progress ? this : nullptr;
4316 }
4317
4318 //------------------------------Value------------------------------------------
4319 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4320 if( !in(0) ) return Type::TOP;
4321 if( phase->type(in(0)) == Type::TOP )
4322 return Type::TOP;
4323 return TypeTuple::MEMBAR;
4324 }
4325
4326 //------------------------------match------------------------------------------
4327 // Construct projections for memory.
4328 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4329 switch (proj->_con) {
4330 case TypeFunc::Control:
4331 case TypeFunc::Memory:
4332 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4333 }
4334 ShouldNotReachHere();
4335 return nullptr;
4336 }
4337
4338 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4339 trailing->_kind = TrailingStore;
4340 leading->_kind = LeadingStore;
4341 #ifdef ASSERT
4342 trailing->_pair_idx = leading->_idx;
4343 leading->_pair_idx = leading->_idx;
4344 #endif
4345 }
4346
4347 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4348 trailing->_kind = TrailingLoadStore;
4595 return (req() > RawStores);
4596 }
4597
4598 void InitializeNode::set_complete(PhaseGVN* phase) {
4599 assert(!is_complete(), "caller responsibility");
4600 _is_complete = Complete;
4601
4602 // After this node is complete, it contains a bunch of
4603 // raw-memory initializations. There is no need for
4604 // it to have anything to do with non-raw memory effects.
4605 // Therefore, tell all non-raw users to re-optimize themselves,
4606 // after skipping the memory effects of this initialization.
4607 PhaseIterGVN* igvn = phase->is_IterGVN();
4608 if (igvn) igvn->add_users_to_worklist(this);
4609 }
4610
4611 // convenience function
4612 // return false if the init contains any stores already
4613 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4614 InitializeNode* init = initialization();
4615 if (init == nullptr || init->is_complete()) {
4616 return false;
4617 }
4618 init->remove_extra_zeroes();
4619 // for now, if this allocation has already collected any inits, bail:
4620 if (init->is_non_zero()) return false;
4621 init->set_complete(phase);
4622 return true;
4623 }
4624
4625 void InitializeNode::remove_extra_zeroes() {
4626 if (req() == RawStores) return;
4627 Node* zmem = zero_memory();
4628 uint fill = RawStores;
4629 for (uint i = fill; i < req(); i++) {
4630 Node* n = in(i);
4631 if (n->is_top() || n == zmem) continue; // skip
4632 if (fill < i) set_req(fill, n); // compact
4633 ++fill;
4634 }
4635 // delete any empty spaces created:
4636 while (fill < req()) {
4637 del_req(fill);
4776 // store node that we'd like to capture. We need to check
4777 // the uses of the MergeMemNode.
4778 mems.push(n);
4779 }
4780 } else if (n->is_Mem()) {
4781 Node* other_adr = n->in(MemNode::Address);
4782 if (other_adr == adr) {
4783 failed = true;
4784 break;
4785 } else {
4786 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4787 if (other_t_adr != nullptr) {
4788 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4789 if (other_alias_idx == alias_idx) {
4790 // A load from the same memory slice as the store right
4791 // after the InitializeNode. We check the control of the
4792 // object/array that is loaded from. If it's the same as
4793 // the store control then we cannot capture the store.
4794 assert(!n->is_Store(), "2 stores to same slice on same control?");
4795 Node* base = other_adr;
4796 if (base->is_Phi()) {
4797 // In rare case, base may be a PhiNode and it may read
4798 // the same memory slice between InitializeNode and store.
4799 failed = true;
4800 break;
4801 }
4802 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4803 base = base->in(AddPNode::Base);
4804 if (base != nullptr) {
4805 base = base->uncast();
4806 if (base->is_Proj() && base->in(0) == alloc) {
4807 failed = true;
4808 break;
4809 }
4810 }
4811 }
4812 }
4813 }
4814 } else {
4815 failed = true;
4816 break;
4817 }
4818 }
4819 }
4820 }
4821 if (failed) {
5368 // z's_done 12 16 16 16 12 16 12
5369 // z's_needed 12 16 16 16 16 16 16
5370 // zsize 0 0 0 0 4 0 4
5371 if (next_full_store < 0) {
5372 // Conservative tack: Zero to end of current word.
5373 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5374 } else {
5375 // Zero to beginning of next fully initialized word.
5376 // Or, don't zero at all, if we are already in that word.
5377 assert(next_full_store >= zeroes_needed, "must go forward");
5378 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5379 zeroes_needed = next_full_store;
5380 }
5381 }
5382
5383 if (zeroes_needed > zeroes_done) {
5384 intptr_t zsize = zeroes_needed - zeroes_done;
5385 // Do some incremental zeroing on rawmem, in parallel with inits.
5386 zeroes_done = align_down(zeroes_done, BytesPerInt);
5387 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5388 allocation()->in(AllocateNode::DefaultValue),
5389 allocation()->in(AllocateNode::RawDefaultValue),
5390 zeroes_done, zeroes_needed,
5391 phase);
5392 zeroes_done = zeroes_needed;
5393 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5394 do_zeroing = false; // leave the hole, next time
5395 }
5396 }
5397
5398 // Collect the store and move on:
5399 phase->replace_input_of(st, MemNode::Memory, inits);
5400 inits = st; // put it on the linearized chain
5401 set_req(i, zmem); // unhook from previous position
5402
5403 if (zeroes_done == st_off)
5404 zeroes_done = next_init_off;
5405
5406 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5407
5408 #ifdef ASSERT
5409 // Various order invariants. Weaker than stores_are_sane because
5429 remove_extra_zeroes(); // clear out all the zmems left over
5430 add_req(inits);
5431
5432 if (!(UseTLAB && ZeroTLAB)) {
5433 // If anything remains to be zeroed, zero it all now.
5434 zeroes_done = align_down(zeroes_done, BytesPerInt);
5435 // if it is the last unused 4 bytes of an instance, forget about it
5436 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5437 if (zeroes_done + BytesPerLong >= size_limit) {
5438 AllocateNode* alloc = allocation();
5439 assert(alloc != nullptr, "must be present");
5440 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5441 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5442 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5443 if (zeroes_done == k->layout_helper())
5444 zeroes_done = size_limit;
5445 }
5446 }
5447 if (zeroes_done < size_limit) {
5448 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5449 allocation()->in(AllocateNode::DefaultValue),
5450 allocation()->in(AllocateNode::RawDefaultValue),
5451 zeroes_done, size_in_bytes, phase);
5452 }
5453 }
5454
5455 set_complete(phase);
5456 return rawmem;
5457 }
5458
5459
5460 #ifdef ASSERT
5461 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5462 if (is_complete())
5463 return true; // stores could be anything at this point
5464 assert(allocation() != nullptr, "must be present");
5465 intptr_t last_off = allocation()->minimum_header_size();
5466 for (uint i = InitializeNode::RawStores; i < req(); i++) {
5467 Node* st = in(i);
5468 intptr_t st_off = get_store_offset(st, phase);
5469 if (st_off < 0) continue; // ignore dead garbage
5470 if (last_off > st_off) {
|