7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "compiler/compileLog.hpp"
29 #include "gc/shared/barrierSet.hpp"
30 #include "gc/shared/c2/barrierSetC2.hpp"
31 #include "gc/shared/tlab_globals.hpp"
32 #include "memory/allocation.inline.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "oops/objArrayKlass.hpp"
35 #include "opto/addnode.hpp"
36 #include "opto/arraycopynode.hpp"
37 #include "opto/cfgnode.hpp"
38 #include "opto/regalloc.hpp"
39 #include "opto/compile.hpp"
40 #include "opto/connode.hpp"
41 #include "opto/convertnode.hpp"
42 #include "opto/loopnode.hpp"
43 #include "opto/machnode.hpp"
44 #include "opto/matcher.hpp"
45 #include "opto/memnode.hpp"
46 #include "opto/mempointer.hpp"
47 #include "opto/mulnode.hpp"
48 #include "opto/narrowptrnode.hpp"
49 #include "opto/phaseX.hpp"
50 #include "opto/regmask.hpp"
51 #include "opto/rootnode.hpp"
52 #include "opto/traceMergeStoresTag.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/macros.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #include "utilities/vmError.hpp"
59
60 // Portions of code courtesy of Clifford Click
61
217 bool is_instance = t_oop->is_known_instance_field();
218 PhaseIterGVN *igvn = phase->is_IterGVN();
219 if (is_instance && igvn != nullptr && result->is_Phi()) {
220 PhiNode *mphi = result->as_Phi();
221 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
222 const TypePtr *t = mphi->adr_type();
223 bool do_split = false;
224 // In the following cases, Load memory input can be further optimized based on
225 // its precise address type
226 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
227 do_split = true;
228 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
229 const TypeOopPtr* mem_t =
230 t->is_oopptr()->cast_to_exactness(true)
231 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
232 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
233 if (t_oop->isa_aryptr()) {
234 mem_t = mem_t->is_aryptr()
235 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
236 ->cast_to_size(t_oop->is_aryptr()->size())
237 ->with_offset(t_oop->is_aryptr()->offset())
238 ->is_aryptr();
239 }
240 do_split = mem_t == t_oop;
241 }
242 if (do_split) {
243 // clone the Phi with our address type
244 result = mphi->split_out_instance(t_adr, igvn);
245 } else {
246 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
247 }
248 }
249 return result;
250 }
251
252 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
253 uint alias_idx = phase->C->get_alias_index(tp);
254 Node *mem = mmem;
255 #ifdef ASSERT
256 {
257 // Check that current type is consistent with the alias index used during graph construction
258 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
259 bool consistent = adr_check == nullptr || adr_check->empty() ||
260 phase->C->must_alias(adr_check, alias_idx );
261 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
262 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
263 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
264 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
265 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
266 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
267 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
268 // don't assert if it is dead code.
269 consistent = true;
270 }
271 if( !consistent ) {
272 st->print("alias_idx==%d, adr_check==", alias_idx);
273 if( adr_check == nullptr ) {
274 st->print("null");
275 } else {
276 adr_check->dump();
277 }
278 st->cr();
279 print_alias_types();
280 assert(consistent, "adr_check must match alias idx");
281 }
282 }
283 #endif
996 Node* ld = gvn.transform(load);
997 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
998 }
999
1000 return load;
1001 }
1002
1003 //------------------------------hash-------------------------------------------
1004 uint LoadNode::hash() const {
1005 // unroll addition of interesting fields
1006 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1007 }
1008
1009 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1010 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1011 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1012 bool is_stable_ary = FoldStableValues &&
1013 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1014 tp->isa_aryptr()->is_stable();
1015
1016 return (eliminate_boxing && non_volatile) || is_stable_ary;
1017 }
1018
1019 return false;
1020 }
1021
1022 LoadNode* LoadNode::pin_array_access_node() const {
1023 const TypePtr* adr_type = this->adr_type();
1024 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1025 return clone_pinned();
1026 }
1027 return nullptr;
1028 }
1029
1030 // Is the value loaded previously stored by an arraycopy? If so return
1031 // a load node that reads from the source array so we may be able to
1032 // optimize out the ArrayCopy node later.
1033 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1034 Node* ld_adr = in(MemNode::Address);
1035 intptr_t ld_off = 0;
1036 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1053 assert(ld_alloc != nullptr, "need an alloc");
1054 assert(addp->is_AddP(), "address must be addp");
1055 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1056 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1057 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1058 addp->set_req(AddPNode::Base, src);
1059 addp->set_req(AddPNode::Address, src);
1060 } else {
1061 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1062 ac->as_ArrayCopy()->is_copyof_validated() ||
1063 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1064 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1065 addp->set_req(AddPNode::Base, src);
1066 addp->set_req(AddPNode::Address, src);
1067
1068 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1069 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1070 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1071
1072 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1073 uint shift = exact_log2(type2aelembytes(ary_elem));
1074
1075 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1076 #ifdef _LP64
1077 diff = phase->transform(new ConvI2LNode(diff));
1078 #endif
1079 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1080
1081 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1082 addp->set_req(AddPNode::Offset, offset);
1083 }
1084 addp = phase->transform(addp);
1085 #ifdef ASSERT
1086 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1087 ld->_adr_type = adr_type;
1088 #endif
1089 ld->set_req(MemNode::Address, addp);
1090 ld->set_req(0, ctl);
1091 ld->set_req(MemNode::Memory, mem);
1092 return ld;
1093 }
1172 // Same base, same offset.
1173 // Possible improvement for arrays: check index value instead of absolute offset.
1174
1175 // At this point we have proven something like this setup:
1176 // B = << base >>
1177 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1178 // S = StoreQ(AddP( B , #Off), V)
1179 // (Actually, we haven't yet proven the Q's are the same.)
1180 // In other words, we are loading from a casted version of
1181 // the same pointer-and-offset that we stored to.
1182 // Casted version may carry a dependency and it is respected.
1183 // Thus, we are able to replace L by V.
1184 }
1185 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1186 if (store_Opcode() != st->Opcode()) {
1187 return nullptr;
1188 }
1189 // LoadVector/StoreVector needs additional check to ensure the types match.
1190 if (st->is_StoreVector()) {
1191 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1192 const TypeVect* out_vt = as_LoadVector()->vect_type();
1193 if (in_vt != out_vt) {
1194 return nullptr;
1195 }
1196 }
1197 return st->in(MemNode::ValueIn);
1198 }
1199
1200 // A load from a freshly-created object always returns zero.
1201 // (This can happen after LoadNode::Ideal resets the load's memory input
1202 // to find_captured_store, which returned InitializeNode::zero_memory.)
1203 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1204 (st->in(0) == ld_alloc) &&
1205 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1206 // return a zero value for the load's basic type
1207 // (This is one of the few places where a generic PhaseTransform
1208 // can create new nodes. Think of it as lazily manifesting
1209 // virtually pre-existing constants.)
1210 if (memory_type() != T_VOID) {
1211 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1212 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1213 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1214 // by the ArrayCopyNode.
1215 return phase->zerocon(memory_type());
1216 }
1217 } else {
1218 // TODO: materialize all-zero vector constant
1219 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1220 }
1221 }
1222
1223 // A load from an initialization barrier can match a captured store.
1224 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1225 InitializeNode* init = st->in(0)->as_Initialize();
1226 AllocateNode* alloc = init->allocation();
1227 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1228 // examine a captured store value
1229 st = init->find_captured_store(ld_off, memory_size(), phase);
1257 //----------------------is_instance_field_load_with_local_phi------------------
1258 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1259 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1260 in(Address)->is_AddP() ) {
1261 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1262 // Only instances and boxed values.
1263 if( t_oop != nullptr &&
1264 (t_oop->is_ptr_to_boxed_value() ||
1265 t_oop->is_known_instance_field()) &&
1266 t_oop->offset() != Type::OffsetBot &&
1267 t_oop->offset() != Type::OffsetTop) {
1268 return true;
1269 }
1270 }
1271 return false;
1272 }
1273
1274 //------------------------------Identity---------------------------------------
1275 // Loads are identity if previous store is to same address
1276 Node* LoadNode::Identity(PhaseGVN* phase) {
1277 // If the previous store-maker is the right kind of Store, and the store is
1278 // to the same address, then we are equal to the value stored.
1279 Node* mem = in(Memory);
1280 Node* value = can_see_stored_value(mem, phase);
1281 if( value ) {
1282 // byte, short & char stores truncate naturally.
1283 // A load has to load the truncated value which requires
1284 // some sort of masking operation and that requires an
1285 // Ideal call instead of an Identity call.
1286 if (memory_size() < BytesPerInt) {
1287 // If the input to the store does not fit with the load's result type,
1288 // it must be truncated via an Ideal call.
1289 if (!phase->type(value)->higher_equal(phase->type(this)))
1290 return this;
1291 }
1292 // (This works even when value is a Con, but LoadNode::Value
1293 // usually runs first, producing the singleton type of the Con.)
1294 if (!has_pinned_control_dependency() || value->is_Con()) {
1295 return value;
1296 } else {
2041 }
2042 }
2043
2044 // Don't do this for integer types. There is only potential profit if
2045 // the element type t is lower than _type; that is, for int types, if _type is
2046 // more restrictive than t. This only happens here if one is short and the other
2047 // char (both 16 bits), and in those cases we've made an intentional decision
2048 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2049 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2050 //
2051 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2052 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2053 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2054 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2055 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2056 // In fact, that could have been the original type of p1, and p1 could have
2057 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2058 // expression (LShiftL quux 3) independently optimized to the constant 8.
2059 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2060 && (_type->isa_vect() == nullptr)
2061 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2062 // t might actually be lower than _type, if _type is a unique
2063 // concrete subclass of abstract class t.
2064 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2065 const Type* jt = t->join_speculative(_type);
2066 // In any case, do not allow the join, per se, to empty out the type.
2067 if (jt->empty() && !t->empty()) {
2068 // This can happen if a interface-typed array narrows to a class type.
2069 jt = _type;
2070 }
2071 #ifdef ASSERT
2072 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2073 // The pointers in the autobox arrays are always non-null
2074 Node* base = adr->in(AddPNode::Base);
2075 if ((base != nullptr) && base->is_DecodeN()) {
2076 // Get LoadN node which loads IntegerCache.cache field
2077 base = base->in(1);
2078 }
2079 if ((base != nullptr) && base->is_Con()) {
2080 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2081 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2082 // It could be narrow oop
2083 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2084 }
2085 }
2086 }
2087 #endif
2088 return jt;
2089 }
2090 }
2091 } else if (tp->base() == Type::InstPtr) {
2092 assert( off != Type::OffsetBot ||
2093 // arrays can be cast to Objects
2094 !tp->isa_instptr() ||
2095 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2096 // unsafe field access may not have a constant offset
2097 C->has_unsafe_access(),
2098 "Field accesses must be precise" );
2099 // For oop loads, we expect the _type to be precise.
2100
2101 // Optimize loads from constant fields.
2102 const TypeInstPtr* tinst = tp->is_instptr();
2103 ciObject* const_oop = tinst->const_oop();
2104 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2105 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
2106 if (con_type != nullptr) {
2107 return con_type;
2108 }
2109 }
2110 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2111 assert(off != Type::OffsetBot ||
2112 !tp->isa_instklassptr() ||
2113 // arrays can be cast to Objects
2114 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2115 // also allow array-loading from the primary supertype
2116 // array during subtype checks
2117 Opcode() == Op_LoadKlass,
2118 "Field accesses must be precise");
2119 // For klass/static loads, we expect the _type to be precise
2120 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2121 /* With mirrors being an indirect in the Klass*
2122 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2123 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2124 *
2125 * So check the type and klass of the node before the LoadP.
2220 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2221 Node* value = can_see_stored_value(mem,phase);
2222 if (value != nullptr && value->is_Con()) {
2223 assert(value->bottom_type()->higher_equal(_type),"sanity");
2224 return value->bottom_type();
2225 }
2226 }
2227
2228 bool is_vect = (_type->isa_vect() != nullptr);
2229 if (is_instance && !is_vect) {
2230 // If we have an instance type and our memory input is the
2231 // programs's initial memory state, there is no matching store,
2232 // so just return a zero of the appropriate type -
2233 // except if it is vectorized - then we have no zero constant.
2234 Node *mem = in(MemNode::Memory);
2235 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2236 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2237 return Type::get_zero_type(_type->basic_type());
2238 }
2239 }
2240
2241 Node* alloc = is_new_object_mark_load();
2242 if (alloc != nullptr) {
2243 return TypeX::make(markWord::prototype().value());
2244 }
2245
2246 return _type;
2247 }
2248
2249 //------------------------------match_edge-------------------------------------
2250 // Do we Match on this edge index or not? Match only the address.
2251 uint LoadNode::match_edge(uint idx) const {
2252 return idx == MemNode::Address;
2253 }
2254
2255 //--------------------------LoadBNode::Ideal--------------------------------------
2256 //
2257 // If the previous store is to the same address as this load,
2258 // and the value stored was larger than a byte, replace this load
2259 // with the value stored truncated to a byte. If no truncation is
2260 // needed, the replacement is done in LoadNode::Identity().
2261 //
2262 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2263 Node* mem = in(MemNode::Memory);
2374 return LoadNode::Ideal(phase, can_reshape);
2375 }
2376
2377 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2378 Node* mem = in(MemNode::Memory);
2379 Node* value = can_see_stored_value(mem,phase);
2380 if (value != nullptr && value->is_Con() &&
2381 !value->bottom_type()->higher_equal(_type)) {
2382 // If the input to the store does not fit with the load's result type,
2383 // it must be truncated. We can't delay until Ideal call since
2384 // a singleton Value is needed for split_thru_phi optimization.
2385 int con = value->get_int();
2386 return TypeInt::make((con << 16) >> 16);
2387 }
2388 return LoadNode::Value(phase);
2389 }
2390
2391 //=============================================================================
2392 //----------------------------LoadKlassNode::make------------------------------
2393 // Polymorphic factory method:
2394 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2395 // sanity check the alias category against the created node type
2396 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2397 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2398 #ifdef _LP64
2399 if (adr_type->is_ptr_to_narrowklass()) {
2400 assert(UseCompressedClassPointers, "no compressed klasses");
2401 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2402 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2403 }
2404 #endif
2405 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2406 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2407 }
2408
2409 //------------------------------Value------------------------------------------
2410 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2411 return klass_value_common(phase);
2412 }
2413
2414 // In most cases, LoadKlassNode does not have the control input set. If the control
2421 // Either input is TOP ==> the result is TOP
2422 const Type *t1 = phase->type( in(MemNode::Memory) );
2423 if (t1 == Type::TOP) return Type::TOP;
2424 Node *adr = in(MemNode::Address);
2425 const Type *t2 = phase->type( adr );
2426 if (t2 == Type::TOP) return Type::TOP;
2427 const TypePtr *tp = t2->is_ptr();
2428 if (TypePtr::above_centerline(tp->ptr()) ||
2429 tp->ptr() == TypePtr::Null) return Type::TOP;
2430
2431 // Return a more precise klass, if possible
2432 const TypeInstPtr *tinst = tp->isa_instptr();
2433 if (tinst != nullptr) {
2434 ciInstanceKlass* ik = tinst->instance_klass();
2435 int offset = tinst->offset();
2436 if (ik == phase->C->env()->Class_klass()
2437 && (offset == java_lang_Class::klass_offset() ||
2438 offset == java_lang_Class::array_klass_offset())) {
2439 // We are loading a special hidden field from a Class mirror object,
2440 // the field which points to the VM's Klass metaobject.
2441 ciType* t = tinst->java_mirror_type();
2442 // java_mirror_type returns non-null for compile-time Class constants.
2443 if (t != nullptr) {
2444 // constant oop => constant klass
2445 if (offset == java_lang_Class::array_klass_offset()) {
2446 if (t->is_void()) {
2447 // We cannot create a void array. Since void is a primitive type return null
2448 // klass. Users of this result need to do a null check on the returned klass.
2449 return TypePtr::NULL_PTR;
2450 }
2451 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2452 }
2453 if (!t->is_klass()) {
2454 // a primitive Class (e.g., int.class) has null for a klass field
2455 return TypePtr::NULL_PTR;
2456 }
2457 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2458 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2459 }
2460 // non-constant mirror, so we can't tell what's going on
2461 }
2462 if (!tinst->is_loaded())
2463 return _type; // Bail out if not loaded
2464 if (offset == oopDesc::klass_offset_in_bytes()) {
2465 return tinst->as_klass_type(true);
2466 }
2467 }
2468
2469 // Check for loading klass from an array
2470 const TypeAryPtr *tary = tp->isa_aryptr();
2471 if (tary != nullptr &&
2472 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2473 return tary->as_klass_type(true);
2474 }
2475
2476 // Check for loading klass from an array klass
2477 const TypeKlassPtr *tkls = tp->isa_klassptr();
2478 if (tkls != nullptr && !StressReflectiveCode) {
2479 if (!tkls->is_loaded())
2480 return _type; // Bail out if not loaded
2481 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2482 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2483 // // Always returning precise element type is incorrect,
2484 // // e.g., element type could be object and array may contain strings
2485 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2486
2487 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2488 // according to the element type's subclassing.
2489 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2490 }
3286 }
3287 ss.print_cr("[TraceMergeStores]: with");
3288 merged_input_value->dump("\n", false, &ss);
3289 merged_store->dump("\n", false, &ss);
3290 tty->print("%s", ss.as_string());
3291 }
3292 #endif
3293
3294 //------------------------------Ideal------------------------------------------
3295 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3296 // When a store immediately follows a relevant allocation/initialization,
3297 // try to capture it into the initialization, or hoist it above.
3298 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3299 Node* p = MemNode::Ideal_common(phase, can_reshape);
3300 if (p) return (p == NodeSentinel) ? nullptr : p;
3301
3302 Node* mem = in(MemNode::Memory);
3303 Node* address = in(MemNode::Address);
3304 Node* value = in(MemNode::ValueIn);
3305 // Back-to-back stores to same address? Fold em up. Generally
3306 // unsafe if I have intervening uses.
3307 {
3308 Node* st = mem;
3309 // If Store 'st' has more than one use, we cannot fold 'st' away.
3310 // For example, 'st' might be the final state at a conditional
3311 // return. Or, 'st' might be used by some node which is live at
3312 // the same time 'st' is live, which might be unschedulable. So,
3313 // require exactly ONE user until such time as we clone 'mem' for
3314 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3315 // true).
3316 while (st->is_Store() && st->outcnt() == 1) {
3317 // Looking at a dead closed cycle of memory?
3318 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3319 assert(Opcode() == st->Opcode() ||
3320 st->Opcode() == Op_StoreVector ||
3321 Opcode() == Op_StoreVector ||
3322 st->Opcode() == Op_StoreVectorScatter ||
3323 Opcode() == Op_StoreVectorScatter ||
3324 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3325 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3326 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3327 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3328 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3329
3330 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3331 st->as_Store()->memory_size() <= this->memory_size()) {
3332 Node* use = st->raw_out(0);
3333 if (phase->is_IterGVN()) {
3334 phase->is_IterGVN()->rehash_node_delayed(use);
3335 }
3336 // It's OK to do this in the parser, since DU info is always accurate,
3337 // and the parser always refers to nodes via SafePointNode maps.
3338 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3339 return this;
3340 }
3341 st = st->in(MemNode::Memory);
3342 }
3343 }
3344
3345
3346 // Capture an unaliased, unconditional, simple store into an initializer.
3433 const StoreVectorNode* store_vector = as_StoreVector();
3434 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3435 const Node* store_indices = store_vector->indices();
3436 const Node* mem_indices = mem_vector->indices();
3437 const Node* store_mask = store_vector->mask();
3438 const Node* mem_mask = mem_vector->mask();
3439 // Ensure types, indices, and masks match
3440 if (store_vector->vect_type() == mem_vector->vect_type() &&
3441 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3442 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3443 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3444 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3445 result = mem;
3446 }
3447 }
3448 }
3449
3450 // Store of zero anywhere into a freshly-allocated object?
3451 // Then the store is useless.
3452 // (It must already have been captured by the InitializeNode.)
3453 if (result == this &&
3454 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3455 // a newly allocated object is already all-zeroes everywhere
3456 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3457 result = mem;
3458 }
3459
3460 if (result == this) {
3461 // the store may also apply to zero-bits in an earlier object
3462 Node* prev_mem = find_previous_store(phase);
3463 // Steps (a), (b): Walk past independent stores to find an exact match.
3464 if (prev_mem != nullptr) {
3465 Node* prev_val = can_see_stored_value(prev_mem, phase);
3466 if (prev_val != nullptr && prev_val == val) {
3467 // prev_val and val might differ by a cast; it would be good
3468 // to keep the more informative of the two.
3469 result = mem;
3470 }
3471 }
3472 }
3473 }
3474
3475 PhaseIterGVN* igvn = phase->is_IterGVN();
3476 if (result != this && igvn != nullptr) {
3477 MemBarNode* trailing = trailing_membar();
3478 if (trailing != nullptr) {
3479 #ifdef ASSERT
3480 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
3756 // Clearing a short array is faster with stores
3757 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3758 // Already know this is a large node, do not try to ideal it
3759 if (_is_large) return nullptr;
3760
3761 const int unit = BytesPerLong;
3762 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3763 if (!t) return nullptr;
3764 if (!t->is_con()) return nullptr;
3765 intptr_t raw_count = t->get_con();
3766 intptr_t size = raw_count;
3767 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3768 // Clearing nothing uses the Identity call.
3769 // Negative clears are possible on dead ClearArrays
3770 // (see jck test stmt114.stmt11402.val).
3771 if (size <= 0 || size % unit != 0) return nullptr;
3772 intptr_t count = size / unit;
3773 // Length too long; communicate this to matchers and assemblers.
3774 // Assemblers are responsible to produce fast hardware clears for it.
3775 if (size > InitArrayShortSize) {
3776 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3777 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3778 return nullptr;
3779 }
3780 if (!IdealizeClearArrayNode) return nullptr;
3781 Node *mem = in(1);
3782 if( phase->type(mem)==Type::TOP ) return nullptr;
3783 Node *adr = in(3);
3784 const Type* at = phase->type(adr);
3785 if( at==Type::TOP ) return nullptr;
3786 const TypePtr* atp = at->isa_ptr();
3787 // adjust atp to be the correct array element address type
3788 if (atp == nullptr) atp = TypePtr::BOTTOM;
3789 else atp = atp->add_offset(Type::OffsetBot);
3790 // Get base for derived pointer purposes
3791 if( adr->Opcode() != Op_AddP ) Unimplemented();
3792 Node *base = adr->in(1);
3793
3794 Node *zero = phase->makecon(TypeLong::ZERO);
3795 Node *off = phase->MakeConX(BytesPerLong);
3796 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3797 count--;
3798 while( count-- ) {
3799 mem = phase->transform(mem);
3800 adr = phase->transform(new AddPNode(base,adr,off));
3801 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3802 }
3803 return mem;
3804 }
3805
3806 //----------------------------step_through----------------------------------
3807 // Return allocation input memory edge if it is different instance
3808 // or itself if it is the one we are looking for.
3809 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3810 Node* n = *np;
3811 assert(n->is_ClearArray(), "sanity");
3812 intptr_t offset;
3813 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3814 // This method is called only before Allocate nodes are expanded
3815 // during macro nodes expansion. Before that ClearArray nodes are
3816 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3817 // Allocate nodes are expanded) which follows allocations.
3818 assert(alloc != nullptr, "should have allocation");
3819 if (alloc->_idx == instance_id) {
3820 // Can not bypass initialization of the instance we are looking for.
3821 return false;
3822 }
3823 // Otherwise skip it.
3824 InitializeNode* init = alloc->initialization();
3825 if (init != nullptr)
3826 *np = init->in(TypeFunc::Memory);
3827 else
3828 *np = alloc->in(TypeFunc::Memory);
3829 return true;
3830 }
3831
3832 //----------------------------clear_memory-------------------------------------
3833 // Generate code to initialize object storage to zero.
3834 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3835 intptr_t start_offset,
3836 Node* end_offset,
3837 PhaseGVN* phase) {
3838 intptr_t offset = start_offset;
3839
3840 int unit = BytesPerLong;
3841 if ((offset % unit) != 0) {
3842 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3843 adr = phase->transform(adr);
3844 const TypePtr* atp = TypeRawPtr::BOTTOM;
3845 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3846 mem = phase->transform(mem);
3847 offset += BytesPerInt;
3848 }
3849 assert((offset % unit) == 0, "");
3850
3851 // Initialize the remaining stuff, if any, with a ClearArray.
3852 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
3853 }
3854
3855 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3856 Node* start_offset,
3857 Node* end_offset,
3858 PhaseGVN* phase) {
3859 if (start_offset == end_offset) {
3860 // nothing to do
3861 return mem;
3862 }
3863
3864 int unit = BytesPerLong;
3865 Node* zbase = start_offset;
3866 Node* zend = end_offset;
3867
3868 // Scale to the unit required by the CPU:
3869 if (!Matcher::init_array_count_is_in_bytes) {
3870 Node* shift = phase->intcon(exact_log2(unit));
3871 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3872 zend = phase->transform(new URShiftXNode(zend, shift) );
3873 }
3874
3875 // Bulk clear double-words
3876 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3877 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3878 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
3879 return phase->transform(mem);
3880 }
3881
3882 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3883 intptr_t start_offset,
3884 intptr_t end_offset,
3885 PhaseGVN* phase) {
3886 if (start_offset == end_offset) {
3887 // nothing to do
3888 return mem;
3889 }
3890
3891 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3892 intptr_t done_offset = end_offset;
3893 if ((done_offset % BytesPerLong) != 0) {
3894 done_offset -= BytesPerInt;
3895 }
3896 if (done_offset > start_offset) {
3897 mem = clear_memory(ctl, mem, dest,
3898 start_offset, phase->MakeConX(done_offset), phase);
3899 }
3900 if (done_offset < end_offset) { // emit the final 32-bit store
3901 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3902 adr = phase->transform(adr);
3903 const TypePtr* atp = TypeRawPtr::BOTTOM;
3904 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3905 mem = phase->transform(mem);
3906 done_offset += BytesPerInt;
3907 }
3908 assert(done_offset == end_offset, "");
3909 return mem;
3910 }
3911
3912 //=============================================================================
3913 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3914 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3915 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3916 #ifdef ASSERT
3917 , _pair_idx(0)
3918 #endif
3919 {
3920 init_class_id(Class_MemBar);
3921 Node* top = C->top();
3922 init_req(TypeFunc::I_O,top);
3923 init_req(TypeFunc::FramePtr,top);
3924 init_req(TypeFunc::ReturnAdr,top);
4030 PhaseIterGVN* igvn = phase->is_IterGVN();
4031 remove(igvn);
4032 // Must return either the original node (now dead) or a new node
4033 // (Do not return a top here, since that would break the uniqueness of top.)
4034 return new ConINode(TypeInt::ZERO);
4035 }
4036 }
4037 return progress ? this : nullptr;
4038 }
4039
4040 //------------------------------Value------------------------------------------
4041 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4042 if( !in(0) ) return Type::TOP;
4043 if( phase->type(in(0)) == Type::TOP )
4044 return Type::TOP;
4045 return TypeTuple::MEMBAR;
4046 }
4047
4048 //------------------------------match------------------------------------------
4049 // Construct projections for memory.
4050 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4051 switch (proj->_con) {
4052 case TypeFunc::Control:
4053 case TypeFunc::Memory:
4054 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4055 }
4056 ShouldNotReachHere();
4057 return nullptr;
4058 }
4059
4060 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4061 trailing->_kind = TrailingStore;
4062 leading->_kind = LeadingStore;
4063 #ifdef ASSERT
4064 trailing->_pair_idx = leading->_idx;
4065 leading->_pair_idx = leading->_idx;
4066 #endif
4067 }
4068
4069 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4070 trailing->_kind = TrailingLoadStore;
4317 return (req() > RawStores);
4318 }
4319
4320 void InitializeNode::set_complete(PhaseGVN* phase) {
4321 assert(!is_complete(), "caller responsibility");
4322 _is_complete = Complete;
4323
4324 // After this node is complete, it contains a bunch of
4325 // raw-memory initializations. There is no need for
4326 // it to have anything to do with non-raw memory effects.
4327 // Therefore, tell all non-raw users to re-optimize themselves,
4328 // after skipping the memory effects of this initialization.
4329 PhaseIterGVN* igvn = phase->is_IterGVN();
4330 if (igvn) igvn->add_users_to_worklist(this);
4331 }
4332
4333 // convenience function
4334 // return false if the init contains any stores already
4335 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4336 InitializeNode* init = initialization();
4337 if (init == nullptr || init->is_complete()) return false;
4338 init->remove_extra_zeroes();
4339 // for now, if this allocation has already collected any inits, bail:
4340 if (init->is_non_zero()) return false;
4341 init->set_complete(phase);
4342 return true;
4343 }
4344
4345 void InitializeNode::remove_extra_zeroes() {
4346 if (req() == RawStores) return;
4347 Node* zmem = zero_memory();
4348 uint fill = RawStores;
4349 for (uint i = fill; i < req(); i++) {
4350 Node* n = in(i);
4351 if (n->is_top() || n == zmem) continue; // skip
4352 if (fill < i) set_req(fill, n); // compact
4353 ++fill;
4354 }
4355 // delete any empty spaces created:
4356 while (fill < req()) {
4357 del_req(fill);
4501 // store node that we'd like to capture. We need to check
4502 // the uses of the MergeMemNode.
4503 mems.push(n);
4504 }
4505 } else if (n->is_Mem()) {
4506 Node* other_adr = n->in(MemNode::Address);
4507 if (other_adr == adr) {
4508 failed = true;
4509 break;
4510 } else {
4511 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4512 if (other_t_adr != nullptr) {
4513 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4514 if (other_alias_idx == alias_idx) {
4515 // A load from the same memory slice as the store right
4516 // after the InitializeNode. We check the control of the
4517 // object/array that is loaded from. If it's the same as
4518 // the store control then we cannot capture the store.
4519 assert(!n->is_Store(), "2 stores to same slice on same control?");
4520 Node* base = other_adr;
4521 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4522 base = base->in(AddPNode::Base);
4523 if (base != nullptr) {
4524 base = base->uncast();
4525 if (base->is_Proj() && base->in(0) == alloc) {
4526 failed = true;
4527 break;
4528 }
4529 }
4530 }
4531 }
4532 }
4533 } else {
4534 failed = true;
4535 break;
4536 }
4537 }
4538 }
4539 }
4540 if (failed) {
5087 // z's_done 12 16 16 16 12 16 12
5088 // z's_needed 12 16 16 16 16 16 16
5089 // zsize 0 0 0 0 4 0 4
5090 if (next_full_store < 0) {
5091 // Conservative tack: Zero to end of current word.
5092 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5093 } else {
5094 // Zero to beginning of next fully initialized word.
5095 // Or, don't zero at all, if we are already in that word.
5096 assert(next_full_store >= zeroes_needed, "must go forward");
5097 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5098 zeroes_needed = next_full_store;
5099 }
5100 }
5101
5102 if (zeroes_needed > zeroes_done) {
5103 intptr_t zsize = zeroes_needed - zeroes_done;
5104 // Do some incremental zeroing on rawmem, in parallel with inits.
5105 zeroes_done = align_down(zeroes_done, BytesPerInt);
5106 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5107 zeroes_done, zeroes_needed,
5108 phase);
5109 zeroes_done = zeroes_needed;
5110 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5111 do_zeroing = false; // leave the hole, next time
5112 }
5113 }
5114
5115 // Collect the store and move on:
5116 phase->replace_input_of(st, MemNode::Memory, inits);
5117 inits = st; // put it on the linearized chain
5118 set_req(i, zmem); // unhook from previous position
5119
5120 if (zeroes_done == st_off)
5121 zeroes_done = next_init_off;
5122
5123 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5124
5125 #ifdef ASSERT
5126 // Various order invariants. Weaker than stores_are_sane because
5146 remove_extra_zeroes(); // clear out all the zmems left over
5147 add_req(inits);
5148
5149 if (!(UseTLAB && ZeroTLAB)) {
5150 // If anything remains to be zeroed, zero it all now.
5151 zeroes_done = align_down(zeroes_done, BytesPerInt);
5152 // if it is the last unused 4 bytes of an instance, forget about it
5153 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5154 if (zeroes_done + BytesPerLong >= size_limit) {
5155 AllocateNode* alloc = allocation();
5156 assert(alloc != nullptr, "must be present");
5157 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5158 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5159 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5160 if (zeroes_done == k->layout_helper())
5161 zeroes_done = size_limit;
5162 }
5163 }
5164 if (zeroes_done < size_limit) {
5165 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5166 zeroes_done, size_in_bytes, phase);
5167 }
5168 }
5169
5170 set_complete(phase);
5171 return rawmem;
5172 }
5173
5174
5175 #ifdef ASSERT
5176 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5177 if (is_complete())
5178 return true; // stores could be anything at this point
5179 assert(allocation() != nullptr, "must be present");
5180 intptr_t last_off = allocation()->minimum_header_size();
5181 for (uint i = InitializeNode::RawStores; i < req(); i++) {
5182 Node* st = in(i);
5183 intptr_t st_off = get_store_offset(st, phase);
5184 if (st_off < 0) continue; // ignore dead garbage
5185 if (last_off > st_off) {
|
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "ci/ciFlatArrayKlass.hpp"
28 #include "classfile/javaClasses.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "compiler/compileLog.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/c2/barrierSetC2.hpp"
33 #include "gc/shared/tlab_globals.hpp"
34 #include "memory/allocation.inline.hpp"
35 #include "memory/resourceArea.hpp"
36 #include "oops/objArrayKlass.hpp"
37 #include "opto/addnode.hpp"
38 #include "opto/arraycopynode.hpp"
39 #include "opto/cfgnode.hpp"
40 #include "opto/regalloc.hpp"
41 #include "opto/compile.hpp"
42 #include "opto/connode.hpp"
43 #include "opto/convertnode.hpp"
44 #include "opto/inlinetypenode.hpp"
45 #include "opto/loopnode.hpp"
46 #include "opto/machnode.hpp"
47 #include "opto/matcher.hpp"
48 #include "opto/memnode.hpp"
49 #include "opto/mempointer.hpp"
50 #include "opto/mulnode.hpp"
51 #include "opto/narrowptrnode.hpp"
52 #include "opto/phaseX.hpp"
53 #include "opto/regmask.hpp"
54 #include "opto/rootnode.hpp"
55 #include "opto/traceMergeStoresTag.hpp"
56 #include "opto/vectornode.hpp"
57 #include "utilities/align.hpp"
58 #include "utilities/copy.hpp"
59 #include "utilities/macros.hpp"
60 #include "utilities/powerOfTwo.hpp"
61 #include "utilities/vmError.hpp"
62
63 // Portions of code courtesy of Clifford Click
64
220 bool is_instance = t_oop->is_known_instance_field();
221 PhaseIterGVN *igvn = phase->is_IterGVN();
222 if (is_instance && igvn != nullptr && result->is_Phi()) {
223 PhiNode *mphi = result->as_Phi();
224 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
225 const TypePtr *t = mphi->adr_type();
226 bool do_split = false;
227 // In the following cases, Load memory input can be further optimized based on
228 // its precise address type
229 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
230 do_split = true;
231 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
232 const TypeOopPtr* mem_t =
233 t->is_oopptr()->cast_to_exactness(true)
234 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
235 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
236 if (t_oop->isa_aryptr()) {
237 mem_t = mem_t->is_aryptr()
238 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
239 ->cast_to_size(t_oop->is_aryptr()->size())
240 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
241 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
242 ->with_offset(t_oop->is_aryptr()->offset())
243 ->is_aryptr();
244 }
245 do_split = mem_t == t_oop;
246 }
247 if (do_split) {
248 // clone the Phi with our address type
249 result = mphi->split_out_instance(t_adr, igvn);
250 } else {
251 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
252 }
253 }
254 return result;
255 }
256
257 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
258 uint alias_idx = phase->C->get_alias_index(tp);
259 Node *mem = mmem;
260 #ifdef ASSERT
261 {
262 // Check that current type is consistent with the alias index used during graph construction
263 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
264 bool consistent = adr_check == nullptr || adr_check->empty() ||
265 phase->C->must_alias(adr_check, alias_idx );
266 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
267 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
268 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
269 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
270 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
271 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
272 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
273 // don't assert if it is dead code.
274 consistent = true;
275 }
276 if( !consistent ) {
277 st->print("alias_idx==%d, adr_check==", alias_idx);
278 if( adr_check == nullptr ) {
279 st->print("null");
280 } else {
281 adr_check->dump();
282 }
283 st->cr();
284 print_alias_types();
285 assert(consistent, "adr_check must match alias idx");
286 }
287 }
288 #endif
1001 Node* ld = gvn.transform(load);
1002 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1003 }
1004
1005 return load;
1006 }
1007
1008 //------------------------------hash-------------------------------------------
1009 uint LoadNode::hash() const {
1010 // unroll addition of interesting fields
1011 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1012 }
1013
1014 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1015 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1016 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1017 bool is_stable_ary = FoldStableValues &&
1018 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1019 tp->isa_aryptr()->is_stable();
1020
1021 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1022 }
1023
1024 return false;
1025 }
1026
1027 LoadNode* LoadNode::pin_array_access_node() const {
1028 const TypePtr* adr_type = this->adr_type();
1029 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1030 return clone_pinned();
1031 }
1032 return nullptr;
1033 }
1034
1035 // Is the value loaded previously stored by an arraycopy? If so return
1036 // a load node that reads from the source array so we may be able to
1037 // optimize out the ArrayCopy node later.
1038 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1039 Node* ld_adr = in(MemNode::Address);
1040 intptr_t ld_off = 0;
1041 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1058 assert(ld_alloc != nullptr, "need an alloc");
1059 assert(addp->is_AddP(), "address must be addp");
1060 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1061 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1062 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1063 addp->set_req(AddPNode::Base, src);
1064 addp->set_req(AddPNode::Address, src);
1065 } else {
1066 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1067 ac->as_ArrayCopy()->is_copyof_validated() ||
1068 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1069 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1070 addp->set_req(AddPNode::Base, src);
1071 addp->set_req(AddPNode::Address, src);
1072
1073 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1074 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1075 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1076
1077 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1078 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1079
1080 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1081 #ifdef _LP64
1082 diff = phase->transform(new ConvI2LNode(diff));
1083 #endif
1084 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1085
1086 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1087 addp->set_req(AddPNode::Offset, offset);
1088 }
1089 addp = phase->transform(addp);
1090 #ifdef ASSERT
1091 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1092 ld->_adr_type = adr_type;
1093 #endif
1094 ld->set_req(MemNode::Address, addp);
1095 ld->set_req(0, ctl);
1096 ld->set_req(MemNode::Memory, mem);
1097 return ld;
1098 }
1177 // Same base, same offset.
1178 // Possible improvement for arrays: check index value instead of absolute offset.
1179
1180 // At this point we have proven something like this setup:
1181 // B = << base >>
1182 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1183 // S = StoreQ(AddP( B , #Off), V)
1184 // (Actually, we haven't yet proven the Q's are the same.)
1185 // In other words, we are loading from a casted version of
1186 // the same pointer-and-offset that we stored to.
1187 // Casted version may carry a dependency and it is respected.
1188 // Thus, we are able to replace L by V.
1189 }
1190 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1191 if (store_Opcode() != st->Opcode()) {
1192 return nullptr;
1193 }
1194 // LoadVector/StoreVector needs additional check to ensure the types match.
1195 if (st->is_StoreVector()) {
1196 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1197 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1198 if (in_vt != out_vt) {
1199 return nullptr;
1200 }
1201 }
1202 return st->in(MemNode::ValueIn);
1203 }
1204
1205 // A load from a freshly-created object always returns zero.
1206 // (This can happen after LoadNode::Ideal resets the load's memory input
1207 // to find_captured_store, which returned InitializeNode::zero_memory.)
1208 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1209 (st->in(0) == ld_alloc) &&
1210 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1211 // return a zero value for the load's basic type
1212 // (This is one of the few places where a generic PhaseTransform
1213 // can create new nodes. Think of it as lazily manifesting
1214 // virtually pre-existing constants.)
1215 Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1216 if (default_value != nullptr) {
1217 return default_value;
1218 }
1219 assert(ld_alloc->in(AllocateNode::RawDefaultValue) == nullptr, "default value may not be null");
1220 if (memory_type() != T_VOID) {
1221 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1222 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1223 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1224 // by the ArrayCopyNode.
1225 return phase->zerocon(memory_type());
1226 }
1227 } else {
1228 // TODO: materialize all-zero vector constant
1229 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1230 }
1231 }
1232
1233 // A load from an initialization barrier can match a captured store.
1234 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1235 InitializeNode* init = st->in(0)->as_Initialize();
1236 AllocateNode* alloc = init->allocation();
1237 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1238 // examine a captured store value
1239 st = init->find_captured_store(ld_off, memory_size(), phase);
1267 //----------------------is_instance_field_load_with_local_phi------------------
1268 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1269 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1270 in(Address)->is_AddP() ) {
1271 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1272 // Only instances and boxed values.
1273 if( t_oop != nullptr &&
1274 (t_oop->is_ptr_to_boxed_value() ||
1275 t_oop->is_known_instance_field()) &&
1276 t_oop->offset() != Type::OffsetBot &&
1277 t_oop->offset() != Type::OffsetTop) {
1278 return true;
1279 }
1280 }
1281 return false;
1282 }
1283
1284 //------------------------------Identity---------------------------------------
1285 // Loads are identity if previous store is to same address
1286 Node* LoadNode::Identity(PhaseGVN* phase) {
1287 // Loading from an InlineType? The InlineType has the values of
1288 // all fields as input. Look for the field with matching offset.
1289 Node* addr = in(Address);
1290 intptr_t offset;
1291 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1292 if (base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1293 Node* value = base->as_InlineType()->field_value_by_offset((int)offset, true);
1294 if (value != nullptr) {
1295 if (Opcode() == Op_LoadN) {
1296 // Encode oop value if we are loading a narrow oop
1297 assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1298 value = phase->transform(new EncodePNode(value, bottom_type()));
1299 }
1300 return value;
1301 }
1302 }
1303
1304 // If the previous store-maker is the right kind of Store, and the store is
1305 // to the same address, then we are equal to the value stored.
1306 Node* mem = in(Memory);
1307 Node* value = can_see_stored_value(mem, phase);
1308 if( value ) {
1309 // byte, short & char stores truncate naturally.
1310 // A load has to load the truncated value which requires
1311 // some sort of masking operation and that requires an
1312 // Ideal call instead of an Identity call.
1313 if (memory_size() < BytesPerInt) {
1314 // If the input to the store does not fit with the load's result type,
1315 // it must be truncated via an Ideal call.
1316 if (!phase->type(value)->higher_equal(phase->type(this)))
1317 return this;
1318 }
1319 // (This works even when value is a Con, but LoadNode::Value
1320 // usually runs first, producing the singleton type of the Con.)
1321 if (!has_pinned_control_dependency() || value->is_Con()) {
1322 return value;
1323 } else {
2068 }
2069 }
2070
2071 // Don't do this for integer types. There is only potential profit if
2072 // the element type t is lower than _type; that is, for int types, if _type is
2073 // more restrictive than t. This only happens here if one is short and the other
2074 // char (both 16 bits), and in those cases we've made an intentional decision
2075 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2076 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2077 //
2078 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2079 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2080 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2081 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2082 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2083 // In fact, that could have been the original type of p1, and p1 could have
2084 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2085 // expression (LShiftL quux 3) independently optimized to the constant 8.
2086 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2087 && (_type->isa_vect() == nullptr)
2088 && !ary->is_flat()
2089 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2090 // t might actually be lower than _type, if _type is a unique
2091 // concrete subclass of abstract class t.
2092 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2093 const Type* jt = t->join_speculative(_type);
2094 // In any case, do not allow the join, per se, to empty out the type.
2095 if (jt->empty() && !t->empty()) {
2096 // This can happen if a interface-typed array narrows to a class type.
2097 jt = _type;
2098 }
2099 #ifdef ASSERT
2100 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2101 // The pointers in the autobox arrays are always non-null
2102 Node* base = adr->in(AddPNode::Base);
2103 if ((base != nullptr) && base->is_DecodeN()) {
2104 // Get LoadN node which loads IntegerCache.cache field
2105 base = base->in(1);
2106 }
2107 if ((base != nullptr) && base->is_Con()) {
2108 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2109 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2110 // It could be narrow oop
2111 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2112 }
2113 }
2114 }
2115 #endif
2116 return jt;
2117 }
2118 }
2119 } else if (tp->base() == Type::InstPtr) {
2120 assert( off != Type::OffsetBot ||
2121 // arrays can be cast to Objects
2122 !tp->isa_instptr() ||
2123 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2124 // Default value load
2125 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2126 // unsafe field access may not have a constant offset
2127 C->has_unsafe_access(),
2128 "Field accesses must be precise" );
2129 // For oop loads, we expect the _type to be precise.
2130
2131 const TypeInstPtr* tinst = tp->is_instptr();
2132 BasicType bt = memory_type();
2133
2134 // Optimize loads from constant fields.
2135 ciObject* const_oop = tinst->const_oop();
2136 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2137 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2138 if (con_type != nullptr) {
2139 return con_type;
2140 }
2141 }
2142 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2143 assert(off != Type::OffsetBot ||
2144 !tp->isa_instklassptr() ||
2145 // arrays can be cast to Objects
2146 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2147 // also allow array-loading from the primary supertype
2148 // array during subtype checks
2149 Opcode() == Op_LoadKlass,
2150 "Field accesses must be precise");
2151 // For klass/static loads, we expect the _type to be precise
2152 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2153 /* With mirrors being an indirect in the Klass*
2154 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2155 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2156 *
2157 * So check the type and klass of the node before the LoadP.
2252 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2253 Node* value = can_see_stored_value(mem,phase);
2254 if (value != nullptr && value->is_Con()) {
2255 assert(value->bottom_type()->higher_equal(_type),"sanity");
2256 return value->bottom_type();
2257 }
2258 }
2259
2260 bool is_vect = (_type->isa_vect() != nullptr);
2261 if (is_instance && !is_vect) {
2262 // If we have an instance type and our memory input is the
2263 // programs's initial memory state, there is no matching store,
2264 // so just return a zero of the appropriate type -
2265 // except if it is vectorized - then we have no zero constant.
2266 Node *mem = in(MemNode::Memory);
2267 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2268 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2269 return Type::get_zero_type(_type->basic_type());
2270 }
2271 }
2272 Node* alloc = is_new_object_mark_load();
2273 if (alloc != nullptr) {
2274 if (EnableValhalla) {
2275 // The mark word may contain property bits (inline, flat, null-free)
2276 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2277 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2278 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2279 return TypeX::make(tkls->exact_klass()->prototype_header().value());
2280 }
2281 } else {
2282 return TypeX::make(markWord::prototype().value());
2283 }
2284 }
2285
2286 return _type;
2287 }
2288
2289 //------------------------------match_edge-------------------------------------
2290 // Do we Match on this edge index or not? Match only the address.
2291 uint LoadNode::match_edge(uint idx) const {
2292 return idx == MemNode::Address;
2293 }
2294
2295 //--------------------------LoadBNode::Ideal--------------------------------------
2296 //
2297 // If the previous store is to the same address as this load,
2298 // and the value stored was larger than a byte, replace this load
2299 // with the value stored truncated to a byte. If no truncation is
2300 // needed, the replacement is done in LoadNode::Identity().
2301 //
2302 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2303 Node* mem = in(MemNode::Memory);
2414 return LoadNode::Ideal(phase, can_reshape);
2415 }
2416
2417 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2418 Node* mem = in(MemNode::Memory);
2419 Node* value = can_see_stored_value(mem,phase);
2420 if (value != nullptr && value->is_Con() &&
2421 !value->bottom_type()->higher_equal(_type)) {
2422 // If the input to the store does not fit with the load's result type,
2423 // it must be truncated. We can't delay until Ideal call since
2424 // a singleton Value is needed for split_thru_phi optimization.
2425 int con = value->get_int();
2426 return TypeInt::make((con << 16) >> 16);
2427 }
2428 return LoadNode::Value(phase);
2429 }
2430
2431 //=============================================================================
2432 //----------------------------LoadKlassNode::make------------------------------
2433 // Polymorphic factory method:
2434 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2435 const TypeKlassPtr* tk) {
2436 // sanity check the alias category against the created node type
2437 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2438 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2439 #ifdef _LP64
2440 if (adr_type->is_ptr_to_narrowklass()) {
2441 assert(UseCompressedClassPointers, "no compressed klasses");
2442 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2443 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2444 }
2445 #endif
2446 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2447 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2448 }
2449
2450 //------------------------------Value------------------------------------------
2451 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2452 return klass_value_common(phase);
2453 }
2454
2455 // In most cases, LoadKlassNode does not have the control input set. If the control
2462 // Either input is TOP ==> the result is TOP
2463 const Type *t1 = phase->type( in(MemNode::Memory) );
2464 if (t1 == Type::TOP) return Type::TOP;
2465 Node *adr = in(MemNode::Address);
2466 const Type *t2 = phase->type( adr );
2467 if (t2 == Type::TOP) return Type::TOP;
2468 const TypePtr *tp = t2->is_ptr();
2469 if (TypePtr::above_centerline(tp->ptr()) ||
2470 tp->ptr() == TypePtr::Null) return Type::TOP;
2471
2472 // Return a more precise klass, if possible
2473 const TypeInstPtr *tinst = tp->isa_instptr();
2474 if (tinst != nullptr) {
2475 ciInstanceKlass* ik = tinst->instance_klass();
2476 int offset = tinst->offset();
2477 if (ik == phase->C->env()->Class_klass()
2478 && (offset == java_lang_Class::klass_offset() ||
2479 offset == java_lang_Class::array_klass_offset())) {
2480 // We are loading a special hidden field from a Class mirror object,
2481 // the field which points to the VM's Klass metaobject.
2482 bool is_null_free_array = false;
2483 ciType* t = tinst->java_mirror_type(&is_null_free_array);
2484 // java_mirror_type returns non-null for compile-time Class constants.
2485 if (t != nullptr) {
2486 // constant oop => constant klass
2487 if (offset == java_lang_Class::array_klass_offset()) {
2488 if (t->is_void()) {
2489 // We cannot create a void array. Since void is a primitive type return null
2490 // klass. Users of this result need to do a null check on the returned klass.
2491 return TypePtr::NULL_PTR;
2492 }
2493 const TypeKlassPtr* tklass = TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2494 if (is_null_free_array) {
2495 tklass = tklass->is_aryklassptr()->cast_to_null_free();
2496 }
2497 return tklass;
2498 }
2499 if (!t->is_klass()) {
2500 // a primitive Class (e.g., int.class) has null for a klass field
2501 return TypePtr::NULL_PTR;
2502 }
2503 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2504 const TypeKlassPtr* tklass = TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2505 if (is_null_free_array) {
2506 tklass = tklass->is_aryklassptr()->cast_to_null_free();
2507 }
2508 return tklass;
2509 }
2510 // non-constant mirror, so we can't tell what's going on
2511 }
2512 if (!tinst->is_loaded())
2513 return _type; // Bail out if not loaded
2514 if (offset == oopDesc::klass_offset_in_bytes()) {
2515 return tinst->as_klass_type(true);
2516 }
2517 }
2518
2519 // Check for loading klass from an array
2520 const TypeAryPtr* tary = tp->isa_aryptr();
2521 if (tary != nullptr &&
2522 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2523 return tary->as_klass_type(true);
2524 }
2525
2526 // Check for loading klass from an array klass
2527 const TypeKlassPtr *tkls = tp->isa_klassptr();
2528 if (tkls != nullptr && !StressReflectiveCode) {
2529 if (!tkls->is_loaded())
2530 return _type; // Bail out if not loaded
2531 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2532 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2533 // // Always returning precise element type is incorrect,
2534 // // e.g., element type could be object and array may contain strings
2535 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2536
2537 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2538 // according to the element type's subclassing.
2539 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2540 }
3336 }
3337 ss.print_cr("[TraceMergeStores]: with");
3338 merged_input_value->dump("\n", false, &ss);
3339 merged_store->dump("\n", false, &ss);
3340 tty->print("%s", ss.as_string());
3341 }
3342 #endif
3343
3344 //------------------------------Ideal------------------------------------------
3345 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3346 // When a store immediately follows a relevant allocation/initialization,
3347 // try to capture it into the initialization, or hoist it above.
3348 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3349 Node* p = MemNode::Ideal_common(phase, can_reshape);
3350 if (p) return (p == NodeSentinel) ? nullptr : p;
3351
3352 Node* mem = in(MemNode::Memory);
3353 Node* address = in(MemNode::Address);
3354 Node* value = in(MemNode::ValueIn);
3355 // Back-to-back stores to same address? Fold em up. Generally
3356 // unsafe if I have intervening uses...
3357 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3358 Node* st = mem;
3359 // If Store 'st' has more than one use, we cannot fold 'st' away.
3360 // For example, 'st' might be the final state at a conditional
3361 // return. Or, 'st' might be used by some node which is live at
3362 // the same time 'st' is live, which might be unschedulable. So,
3363 // require exactly ONE user until such time as we clone 'mem' for
3364 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3365 // true).
3366 while (st->is_Store() && st->outcnt() == 1) {
3367 // Looking at a dead closed cycle of memory?
3368 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3369 assert(Opcode() == st->Opcode() ||
3370 st->Opcode() == Op_StoreVector ||
3371 Opcode() == Op_StoreVector ||
3372 st->Opcode() == Op_StoreVectorScatter ||
3373 Opcode() == Op_StoreVectorScatter ||
3374 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3375 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3376 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3377 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3378 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3379 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3380
3381 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3382 st->as_Store()->memory_size() <= this->memory_size()) {
3383 Node* use = st->raw_out(0);
3384 if (phase->is_IterGVN()) {
3385 phase->is_IterGVN()->rehash_node_delayed(use);
3386 }
3387 // It's OK to do this in the parser, since DU info is always accurate,
3388 // and the parser always refers to nodes via SafePointNode maps.
3389 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3390 return this;
3391 }
3392 st = st->in(MemNode::Memory);
3393 }
3394 }
3395
3396
3397 // Capture an unaliased, unconditional, simple store into an initializer.
3484 const StoreVectorNode* store_vector = as_StoreVector();
3485 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3486 const Node* store_indices = store_vector->indices();
3487 const Node* mem_indices = mem_vector->indices();
3488 const Node* store_mask = store_vector->mask();
3489 const Node* mem_mask = mem_vector->mask();
3490 // Ensure types, indices, and masks match
3491 if (store_vector->vect_type() == mem_vector->vect_type() &&
3492 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3493 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3494 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3495 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3496 result = mem;
3497 }
3498 }
3499 }
3500
3501 // Store of zero anywhere into a freshly-allocated object?
3502 // Then the store is useless.
3503 // (It must already have been captured by the InitializeNode.)
3504 if (result == this && ReduceFieldZeroing) {
3505 // a newly allocated object is already all-zeroes everywhere
3506 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3507 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
3508 result = mem;
3509 }
3510
3511 if (result == this && phase->type(val)->is_zero_type()) {
3512 // the store may also apply to zero-bits in an earlier object
3513 Node* prev_mem = find_previous_store(phase);
3514 // Steps (a), (b): Walk past independent stores to find an exact match.
3515 if (prev_mem != nullptr) {
3516 Node* prev_val = can_see_stored_value(prev_mem, phase);
3517 if (prev_val != nullptr && prev_val == val) {
3518 // prev_val and val might differ by a cast; it would be good
3519 // to keep the more informative of the two.
3520 result = mem;
3521 }
3522 }
3523 }
3524 }
3525
3526 PhaseIterGVN* igvn = phase->is_IterGVN();
3527 if (result != this && igvn != nullptr) {
3528 MemBarNode* trailing = trailing_membar();
3529 if (trailing != nullptr) {
3530 #ifdef ASSERT
3531 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
3807 // Clearing a short array is faster with stores
3808 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3809 // Already know this is a large node, do not try to ideal it
3810 if (_is_large) return nullptr;
3811
3812 const int unit = BytesPerLong;
3813 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3814 if (!t) return nullptr;
3815 if (!t->is_con()) return nullptr;
3816 intptr_t raw_count = t->get_con();
3817 intptr_t size = raw_count;
3818 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3819 // Clearing nothing uses the Identity call.
3820 // Negative clears are possible on dead ClearArrays
3821 // (see jck test stmt114.stmt11402.val).
3822 if (size <= 0 || size % unit != 0) return nullptr;
3823 intptr_t count = size / unit;
3824 // Length too long; communicate this to matchers and assemblers.
3825 // Assemblers are responsible to produce fast hardware clears for it.
3826 if (size > InitArrayShortSize) {
3827 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3828 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3829 return nullptr;
3830 }
3831 if (!IdealizeClearArrayNode) return nullptr;
3832 Node *mem = in(1);
3833 if( phase->type(mem)==Type::TOP ) return nullptr;
3834 Node *adr = in(3);
3835 const Type* at = phase->type(adr);
3836 if( at==Type::TOP ) return nullptr;
3837 const TypePtr* atp = at->isa_ptr();
3838 // adjust atp to be the correct array element address type
3839 if (atp == nullptr) atp = TypePtr::BOTTOM;
3840 else atp = atp->add_offset(Type::OffsetBot);
3841 // Get base for derived pointer purposes
3842 if( adr->Opcode() != Op_AddP ) Unimplemented();
3843 Node *base = adr->in(1);
3844
3845 Node *val = in(4);
3846 Node *off = phase->MakeConX(BytesPerLong);
3847 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3848 count--;
3849 while( count-- ) {
3850 mem = phase->transform(mem);
3851 adr = phase->transform(new AddPNode(base,adr,off));
3852 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3853 }
3854 return mem;
3855 }
3856
3857 //----------------------------step_through----------------------------------
3858 // Return allocation input memory edge if it is different instance
3859 // or itself if it is the one we are looking for.
3860 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3861 Node* n = *np;
3862 assert(n->is_ClearArray(), "sanity");
3863 intptr_t offset;
3864 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3865 // This method is called only before Allocate nodes are expanded
3866 // during macro nodes expansion. Before that ClearArray nodes are
3867 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3868 // Allocate nodes are expanded) which follows allocations.
3869 assert(alloc != nullptr, "should have allocation");
3870 if (alloc->_idx == instance_id) {
3871 // Can not bypass initialization of the instance we are looking for.
3872 return false;
3873 }
3874 // Otherwise skip it.
3875 InitializeNode* init = alloc->initialization();
3876 if (init != nullptr)
3877 *np = init->in(TypeFunc::Memory);
3878 else
3879 *np = alloc->in(TypeFunc::Memory);
3880 return true;
3881 }
3882
3883 //----------------------------clear_memory-------------------------------------
3884 // Generate code to initialize object storage to zero.
3885 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3886 Node* val,
3887 Node* raw_val,
3888 intptr_t start_offset,
3889 Node* end_offset,
3890 PhaseGVN* phase) {
3891 intptr_t offset = start_offset;
3892
3893 int unit = BytesPerLong;
3894 if ((offset % unit) != 0) {
3895 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3896 adr = phase->transform(adr);
3897 const TypePtr* atp = TypeRawPtr::BOTTOM;
3898 if (val != nullptr) {
3899 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3900 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3901 } else {
3902 assert(raw_val == nullptr, "val may not be null");
3903 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3904 }
3905 mem = phase->transform(mem);
3906 offset += BytesPerInt;
3907 }
3908 assert((offset % unit) == 0, "");
3909
3910 // Initialize the remaining stuff, if any, with a ClearArray.
3911 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3912 }
3913
3914 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3915 Node* raw_val,
3916 Node* start_offset,
3917 Node* end_offset,
3918 PhaseGVN* phase) {
3919 if (start_offset == end_offset) {
3920 // nothing to do
3921 return mem;
3922 }
3923
3924 int unit = BytesPerLong;
3925 Node* zbase = start_offset;
3926 Node* zend = end_offset;
3927
3928 // Scale to the unit required by the CPU:
3929 if (!Matcher::init_array_count_is_in_bytes) {
3930 Node* shift = phase->intcon(exact_log2(unit));
3931 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3932 zend = phase->transform(new URShiftXNode(zend, shift) );
3933 }
3934
3935 // Bulk clear double-words
3936 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3937 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3938 if (raw_val == nullptr) {
3939 raw_val = phase->MakeConX(0);
3940 }
3941 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3942 return phase->transform(mem);
3943 }
3944
3945 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3946 Node* val,
3947 Node* raw_val,
3948 intptr_t start_offset,
3949 intptr_t end_offset,
3950 PhaseGVN* phase) {
3951 if (start_offset == end_offset) {
3952 // nothing to do
3953 return mem;
3954 }
3955
3956 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3957 intptr_t done_offset = end_offset;
3958 if ((done_offset % BytesPerLong) != 0) {
3959 done_offset -= BytesPerInt;
3960 }
3961 if (done_offset > start_offset) {
3962 mem = clear_memory(ctl, mem, dest, val, raw_val,
3963 start_offset, phase->MakeConX(done_offset), phase);
3964 }
3965 if (done_offset < end_offset) { // emit the final 32-bit store
3966 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3967 adr = phase->transform(adr);
3968 const TypePtr* atp = TypeRawPtr::BOTTOM;
3969 if (val != nullptr) {
3970 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3971 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3972 } else {
3973 assert(raw_val == nullptr, "val may not be null");
3974 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3975 }
3976 mem = phase->transform(mem);
3977 done_offset += BytesPerInt;
3978 }
3979 assert(done_offset == end_offset, "");
3980 return mem;
3981 }
3982
3983 //=============================================================================
3984 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3985 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3986 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3987 #ifdef ASSERT
3988 , _pair_idx(0)
3989 #endif
3990 {
3991 init_class_id(Class_MemBar);
3992 Node* top = C->top();
3993 init_req(TypeFunc::I_O,top);
3994 init_req(TypeFunc::FramePtr,top);
3995 init_req(TypeFunc::ReturnAdr,top);
4101 PhaseIterGVN* igvn = phase->is_IterGVN();
4102 remove(igvn);
4103 // Must return either the original node (now dead) or a new node
4104 // (Do not return a top here, since that would break the uniqueness of top.)
4105 return new ConINode(TypeInt::ZERO);
4106 }
4107 }
4108 return progress ? this : nullptr;
4109 }
4110
4111 //------------------------------Value------------------------------------------
4112 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4113 if( !in(0) ) return Type::TOP;
4114 if( phase->type(in(0)) == Type::TOP )
4115 return Type::TOP;
4116 return TypeTuple::MEMBAR;
4117 }
4118
4119 //------------------------------match------------------------------------------
4120 // Construct projections for memory.
4121 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4122 switch (proj->_con) {
4123 case TypeFunc::Control:
4124 case TypeFunc::Memory:
4125 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4126 }
4127 ShouldNotReachHere();
4128 return nullptr;
4129 }
4130
4131 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4132 trailing->_kind = TrailingStore;
4133 leading->_kind = LeadingStore;
4134 #ifdef ASSERT
4135 trailing->_pair_idx = leading->_idx;
4136 leading->_pair_idx = leading->_idx;
4137 #endif
4138 }
4139
4140 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4141 trailing->_kind = TrailingLoadStore;
4388 return (req() > RawStores);
4389 }
4390
4391 void InitializeNode::set_complete(PhaseGVN* phase) {
4392 assert(!is_complete(), "caller responsibility");
4393 _is_complete = Complete;
4394
4395 // After this node is complete, it contains a bunch of
4396 // raw-memory initializations. There is no need for
4397 // it to have anything to do with non-raw memory effects.
4398 // Therefore, tell all non-raw users to re-optimize themselves,
4399 // after skipping the memory effects of this initialization.
4400 PhaseIterGVN* igvn = phase->is_IterGVN();
4401 if (igvn) igvn->add_users_to_worklist(this);
4402 }
4403
4404 // convenience function
4405 // return false if the init contains any stores already
4406 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4407 InitializeNode* init = initialization();
4408 if (init == nullptr || init->is_complete()) {
4409 return false;
4410 }
4411 init->remove_extra_zeroes();
4412 // for now, if this allocation has already collected any inits, bail:
4413 if (init->is_non_zero()) return false;
4414 init->set_complete(phase);
4415 return true;
4416 }
4417
4418 void InitializeNode::remove_extra_zeroes() {
4419 if (req() == RawStores) return;
4420 Node* zmem = zero_memory();
4421 uint fill = RawStores;
4422 for (uint i = fill; i < req(); i++) {
4423 Node* n = in(i);
4424 if (n->is_top() || n == zmem) continue; // skip
4425 if (fill < i) set_req(fill, n); // compact
4426 ++fill;
4427 }
4428 // delete any empty spaces created:
4429 while (fill < req()) {
4430 del_req(fill);
4574 // store node that we'd like to capture. We need to check
4575 // the uses of the MergeMemNode.
4576 mems.push(n);
4577 }
4578 } else if (n->is_Mem()) {
4579 Node* other_adr = n->in(MemNode::Address);
4580 if (other_adr == adr) {
4581 failed = true;
4582 break;
4583 } else {
4584 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4585 if (other_t_adr != nullptr) {
4586 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4587 if (other_alias_idx == alias_idx) {
4588 // A load from the same memory slice as the store right
4589 // after the InitializeNode. We check the control of the
4590 // object/array that is loaded from. If it's the same as
4591 // the store control then we cannot capture the store.
4592 assert(!n->is_Store(), "2 stores to same slice on same control?");
4593 Node* base = other_adr;
4594 if (base->is_Phi()) {
4595 // In rare case, base may be a PhiNode and it may read
4596 // the same memory slice between InitializeNode and store.
4597 failed = true;
4598 break;
4599 }
4600 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4601 base = base->in(AddPNode::Base);
4602 if (base != nullptr) {
4603 base = base->uncast();
4604 if (base->is_Proj() && base->in(0) == alloc) {
4605 failed = true;
4606 break;
4607 }
4608 }
4609 }
4610 }
4611 }
4612 } else {
4613 failed = true;
4614 break;
4615 }
4616 }
4617 }
4618 }
4619 if (failed) {
5166 // z's_done 12 16 16 16 12 16 12
5167 // z's_needed 12 16 16 16 16 16 16
5168 // zsize 0 0 0 0 4 0 4
5169 if (next_full_store < 0) {
5170 // Conservative tack: Zero to end of current word.
5171 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5172 } else {
5173 // Zero to beginning of next fully initialized word.
5174 // Or, don't zero at all, if we are already in that word.
5175 assert(next_full_store >= zeroes_needed, "must go forward");
5176 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5177 zeroes_needed = next_full_store;
5178 }
5179 }
5180
5181 if (zeroes_needed > zeroes_done) {
5182 intptr_t zsize = zeroes_needed - zeroes_done;
5183 // Do some incremental zeroing on rawmem, in parallel with inits.
5184 zeroes_done = align_down(zeroes_done, BytesPerInt);
5185 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5186 allocation()->in(AllocateNode::DefaultValue),
5187 allocation()->in(AllocateNode::RawDefaultValue),
5188 zeroes_done, zeroes_needed,
5189 phase);
5190 zeroes_done = zeroes_needed;
5191 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5192 do_zeroing = false; // leave the hole, next time
5193 }
5194 }
5195
5196 // Collect the store and move on:
5197 phase->replace_input_of(st, MemNode::Memory, inits);
5198 inits = st; // put it on the linearized chain
5199 set_req(i, zmem); // unhook from previous position
5200
5201 if (zeroes_done == st_off)
5202 zeroes_done = next_init_off;
5203
5204 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5205
5206 #ifdef ASSERT
5207 // Various order invariants. Weaker than stores_are_sane because
5227 remove_extra_zeroes(); // clear out all the zmems left over
5228 add_req(inits);
5229
5230 if (!(UseTLAB && ZeroTLAB)) {
5231 // If anything remains to be zeroed, zero it all now.
5232 zeroes_done = align_down(zeroes_done, BytesPerInt);
5233 // if it is the last unused 4 bytes of an instance, forget about it
5234 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5235 if (zeroes_done + BytesPerLong >= size_limit) {
5236 AllocateNode* alloc = allocation();
5237 assert(alloc != nullptr, "must be present");
5238 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5239 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5240 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5241 if (zeroes_done == k->layout_helper())
5242 zeroes_done = size_limit;
5243 }
5244 }
5245 if (zeroes_done < size_limit) {
5246 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5247 allocation()->in(AllocateNode::DefaultValue),
5248 allocation()->in(AllocateNode::RawDefaultValue),
5249 zeroes_done, size_in_bytes, phase);
5250 }
5251 }
5252
5253 set_complete(phase);
5254 return rawmem;
5255 }
5256
5257
5258 #ifdef ASSERT
5259 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5260 if (is_complete())
5261 return true; // stores could be anything at this point
5262 assert(allocation() != nullptr, "must be present");
5263 intptr_t last_off = allocation()->minimum_header_size();
5264 for (uint i = InitializeNode::RawStores; i < req(); i++) {
5265 Node* st = in(i);
5266 intptr_t st_off = get_store_offset(st, phase);
5267 if (st_off < 0) continue; // ignore dead garbage
5268 if (last_off > st_off) {
|