6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/regalloc.hpp"
38 #include "opto/compile.hpp"
39 #include "opto/connode.hpp"
40 #include "opto/convertnode.hpp"
41 #include "opto/loopnode.hpp"
42 #include "opto/machnode.hpp"
43 #include "opto/matcher.hpp"
44 #include "opto/memnode.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/phaseX.hpp"
48 #include "opto/regmask.hpp"
49 #include "opto/rootnode.hpp"
50 #include "opto/vectornode.hpp"
51 #include "utilities/align.hpp"
52 #include "utilities/copy.hpp"
53 #include "utilities/macros.hpp"
54 #include "utilities/powerOfTwo.hpp"
55 #include "utilities/vmError.hpp"
56
57 // Portions of code courtesy of Clifford Click
58
59 // Optimization - Graph Style
60
214 bool is_instance = t_oop->is_known_instance_field();
215 PhaseIterGVN *igvn = phase->is_IterGVN();
216 if (is_instance && igvn != nullptr && result->is_Phi()) {
217 PhiNode *mphi = result->as_Phi();
218 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
219 const TypePtr *t = mphi->adr_type();
220 bool do_split = false;
221 // In the following cases, Load memory input can be further optimized based on
222 // its precise address type
223 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
224 do_split = true;
225 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
226 const TypeOopPtr* mem_t =
227 t->is_oopptr()->cast_to_exactness(true)
228 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
229 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
230 if (t_oop->is_aryptr()) {
231 mem_t = mem_t->is_aryptr()
232 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
233 ->cast_to_size(t_oop->is_aryptr()->size())
234 ->with_offset(t_oop->is_aryptr()->offset())
235 ->is_aryptr();
236 }
237 do_split = mem_t == t_oop;
238 }
239 if (do_split) {
240 // clone the Phi with our address type
241 result = mphi->split_out_instance(t_adr, igvn);
242 } else {
243 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
244 }
245 }
246 return result;
247 }
248
249 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
250 uint alias_idx = phase->C->get_alias_index(tp);
251 Node *mem = mmem;
252 #ifdef ASSERT
253 {
254 // Check that current type is consistent with the alias index used during graph construction
255 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
256 bool consistent = adr_check == nullptr || adr_check->empty() ||
257 phase->C->must_alias(adr_check, alias_idx );
258 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
259 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
260 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
261 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
262 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
263 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
264 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
265 // don't assert if it is dead code.
266 consistent = true;
267 }
268 if( !consistent ) {
269 st->print("alias_idx==%d, adr_check==", alias_idx);
270 if( adr_check == nullptr ) {
271 st->print("null");
272 } else {
273 adr_check->dump();
274 }
275 st->cr();
276 print_alias_types();
277 assert(consistent, "adr_check must match alias idx");
278 }
279 }
280 #endif
960 Node* ld = gvn.transform(load);
961 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
962 }
963
964 return load;
965 }
966
967 //------------------------------hash-------------------------------------------
968 uint LoadNode::hash() const {
969 // unroll addition of interesting fields
970 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
971 }
972
973 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
974 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
975 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
976 bool is_stable_ary = FoldStableValues &&
977 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
978 tp->isa_aryptr()->is_stable();
979
980 return (eliminate_boxing && non_volatile) || is_stable_ary;
981 }
982
983 return false;
984 }
985
986 // Is the value loaded previously stored by an arraycopy? If so return
987 // a load node that reads from the source array so we may be able to
988 // optimize out the ArrayCopy node later.
989 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
990 Node* ld_adr = in(MemNode::Address);
991 intptr_t ld_off = 0;
992 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
993 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
994 if (ac != nullptr) {
995 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
996
997 Node* mem = ac->in(TypeFunc::Memory);
998 Node* ctl = ac->in(0);
999 Node* src = ac->in(ArrayCopyNode::Src);
1000
1008 assert(ld_alloc != nullptr, "need an alloc");
1009 assert(addp->is_AddP(), "address must be addp");
1010 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1011 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1012 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1013 addp->set_req(AddPNode::Base, src);
1014 addp->set_req(AddPNode::Address, src);
1015 } else {
1016 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1017 ac->as_ArrayCopy()->is_copyof_validated() ||
1018 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1019 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1020 addp->set_req(AddPNode::Base, src);
1021 addp->set_req(AddPNode::Address, src);
1022
1023 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1024 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1025 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1026
1027 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1028 uint shift = exact_log2(type2aelembytes(ary_elem));
1029
1030 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1031 #ifdef _LP64
1032 diff = phase->transform(new ConvI2LNode(diff));
1033 #endif
1034 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1035
1036 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1037 addp->set_req(AddPNode::Offset, offset);
1038 }
1039 addp = phase->transform(addp);
1040 #ifdef ASSERT
1041 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1042 ld->_adr_type = adr_type;
1043 #endif
1044 ld->set_req(MemNode::Address, addp);
1045 ld->set_req(0, ctl);
1046 ld->set_req(MemNode::Memory, mem);
1047 // load depends on the tests that validate the arraycopy
1048 ld->_control_dependency = UnknownControl;
1129 // Same base, same offset.
1130 // Possible improvement for arrays: check index value instead of absolute offset.
1131
1132 // At this point we have proven something like this setup:
1133 // B = << base >>
1134 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1135 // S = StoreQ(AddP( B , #Off), V)
1136 // (Actually, we haven't yet proven the Q's are the same.)
1137 // In other words, we are loading from a casted version of
1138 // the same pointer-and-offset that we stored to.
1139 // Casted version may carry a dependency and it is respected.
1140 // Thus, we are able to replace L by V.
1141 }
1142 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1143 if (store_Opcode() != st->Opcode()) {
1144 return nullptr;
1145 }
1146 // LoadVector/StoreVector needs additional check to ensure the types match.
1147 if (st->is_StoreVector()) {
1148 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1149 const TypeVect* out_vt = as_LoadVector()->vect_type();
1150 if (in_vt != out_vt) {
1151 return nullptr;
1152 }
1153 }
1154 return st->in(MemNode::ValueIn);
1155 }
1156
1157 // A load from a freshly-created object always returns zero.
1158 // (This can happen after LoadNode::Ideal resets the load's memory input
1159 // to find_captured_store, which returned InitializeNode::zero_memory.)
1160 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1161 (st->in(0) == ld_alloc) &&
1162 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1163 // return a zero value for the load's basic type
1164 // (This is one of the few places where a generic PhaseTransform
1165 // can create new nodes. Think of it as lazily manifesting
1166 // virtually pre-existing constants.)
1167 if (memory_type() != T_VOID) {
1168 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1169 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1170 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1171 // by the ArrayCopyNode.
1172 return phase->zerocon(memory_type());
1173 }
1174 } else {
1175 // TODO: materialize all-zero vector constant
1176 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1177 }
1178 }
1179
1180 // A load from an initialization barrier can match a captured store.
1181 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1182 InitializeNode* init = st->in(0)->as_Initialize();
1183 AllocateNode* alloc = init->allocation();
1184 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1185 // examine a captured store value
1186 st = init->find_captured_store(ld_off, memory_size(), phase);
1214 //----------------------is_instance_field_load_with_local_phi------------------
1215 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1216 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1217 in(Address)->is_AddP() ) {
1218 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1219 // Only instances and boxed values.
1220 if( t_oop != nullptr &&
1221 (t_oop->is_ptr_to_boxed_value() ||
1222 t_oop->is_known_instance_field()) &&
1223 t_oop->offset() != Type::OffsetBot &&
1224 t_oop->offset() != Type::OffsetTop) {
1225 return true;
1226 }
1227 }
1228 return false;
1229 }
1230
1231 //------------------------------Identity---------------------------------------
1232 // Loads are identity if previous store is to same address
1233 Node* LoadNode::Identity(PhaseGVN* phase) {
1234 // If the previous store-maker is the right kind of Store, and the store is
1235 // to the same address, then we are equal to the value stored.
1236 Node* mem = in(Memory);
1237 Node* value = can_see_stored_value(mem, phase);
1238 if( value ) {
1239 // byte, short & char stores truncate naturally.
1240 // A load has to load the truncated value which requires
1241 // some sort of masking operation and that requires an
1242 // Ideal call instead of an Identity call.
1243 if (memory_size() < BytesPerInt) {
1244 // If the input to the store does not fit with the load's result type,
1245 // it must be truncated via an Ideal call.
1246 if (!phase->type(value)->higher_equal(phase->type(this)))
1247 return this;
1248 }
1249 // (This works even when value is a Con, but LoadNode::Value
1250 // usually runs first, producing the singleton type of the Con.)
1251 if (!has_pinned_control_dependency() || value->is_Con()) {
1252 return value;
1253 } else {
1972 }
1973 }
1974
1975 // Don't do this for integer types. There is only potential profit if
1976 // the element type t is lower than _type; that is, for int types, if _type is
1977 // more restrictive than t. This only happens here if one is short and the other
1978 // char (both 16 bits), and in those cases we've made an intentional decision
1979 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1980 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1981 //
1982 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1983 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
1984 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1985 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
1986 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1987 // In fact, that could have been the original type of p1, and p1 could have
1988 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1989 // expression (LShiftL quux 3) independently optimized to the constant 8.
1990 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
1991 && (_type->isa_vect() == nullptr)
1992 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1993 // t might actually be lower than _type, if _type is a unique
1994 // concrete subclass of abstract class t.
1995 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
1996 const Type* jt = t->join_speculative(_type);
1997 // In any case, do not allow the join, per se, to empty out the type.
1998 if (jt->empty() && !t->empty()) {
1999 // This can happen if a interface-typed array narrows to a class type.
2000 jt = _type;
2001 }
2002 #ifdef ASSERT
2003 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2004 // The pointers in the autobox arrays are always non-null
2005 Node* base = adr->in(AddPNode::Base);
2006 if ((base != nullptr) && base->is_DecodeN()) {
2007 // Get LoadN node which loads IntegerCache.cache field
2008 base = base->in(1);
2009 }
2010 if ((base != nullptr) && base->is_Con()) {
2011 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2012 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2013 // It could be narrow oop
2014 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2015 }
2016 }
2017 }
2018 #endif
2019 return jt;
2020 }
2021 }
2022 } else if (tp->base() == Type::InstPtr) {
2023 assert( off != Type::OffsetBot ||
2024 // arrays can be cast to Objects
2025 !tp->isa_instptr() ||
2026 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2027 // unsafe field access may not have a constant offset
2028 C->has_unsafe_access(),
2029 "Field accesses must be precise" );
2030 // For oop loads, we expect the _type to be precise.
2031
2032 // Optimize loads from constant fields.
2033 const TypeInstPtr* tinst = tp->is_instptr();
2034 ciObject* const_oop = tinst->const_oop();
2035 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2036 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
2037 if (con_type != nullptr) {
2038 return con_type;
2039 }
2040 }
2041 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2042 assert(off != Type::OffsetBot ||
2043 !tp->isa_instklassptr() ||
2044 // arrays can be cast to Objects
2045 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2046 // also allow array-loading from the primary supertype
2047 // array during subtype checks
2048 Opcode() == Op_LoadKlass,
2049 "Field accesses must be precise");
2050 // For klass/static loads, we expect the _type to be precise
2051 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2052 /* With mirrors being an indirect in the Klass*
2053 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2054 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2055 *
2056 * So check the type and klass of the node before the LoadP.
2057 */
2058 Node* adr2 = adr->in(MemNode::Address);
2059 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2060 if (tkls != nullptr && !StressReflectiveCode) {
2061 if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2062 ciKlass* klass = tkls->exact_klass();
2063 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2064 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2065 return TypeInstPtr::make(klass->java_mirror());
2066 }
2067 }
2068 }
2069
2070 const TypeKlassPtr *tkls = tp->isa_klassptr();
2071 if (tkls != nullptr) {
2072 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2073 ciKlass* klass = tkls->exact_klass();
2074 // We are loading a field from a Klass metaobject whose identity
2075 // is known at compile time (the type is "exact" or "precise").
2076 // Check for fields we know are maintained as constants by the VM.
2077 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2078 // The field is Klass::_super_check_offset. Return its (constant) value.
2079 // (Folds up type checking code.)
2080 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2081 return TypeInt::make(klass->super_check_offset());
2082 }
2083 // Compute index into primary_supers array
2084 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2085 // Check for overflowing; use unsigned compare to handle the negative case.
2151 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2152 Node* value = can_see_stored_value(mem,phase);
2153 if (value != nullptr && value->is_Con()) {
2154 assert(value->bottom_type()->higher_equal(_type),"sanity");
2155 return value->bottom_type();
2156 }
2157 }
2158
2159 bool is_vect = (_type->isa_vect() != nullptr);
2160 if (is_instance && !is_vect) {
2161 // If we have an instance type and our memory input is the
2162 // programs's initial memory state, there is no matching store,
2163 // so just return a zero of the appropriate type -
2164 // except if it is vectorized - then we have no zero constant.
2165 Node *mem = in(MemNode::Memory);
2166 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2167 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2168 return Type::get_zero_type(_type->basic_type());
2169 }
2170 }
2171
2172 Node* alloc = is_new_object_mark_load();
2173 if (alloc != nullptr) {
2174 return TypeX::make(markWord::prototype().value());
2175 }
2176
2177 return _type;
2178 }
2179
2180 //------------------------------match_edge-------------------------------------
2181 // Do we Match on this edge index or not? Match only the address.
2182 uint LoadNode::match_edge(uint idx) const {
2183 return idx == MemNode::Address;
2184 }
2185
2186 //--------------------------LoadBNode::Ideal--------------------------------------
2187 //
2188 // If the previous store is to the same address as this load,
2189 // and the value stored was larger than a byte, replace this load
2190 // with the value stored truncated to a byte. If no truncation is
2191 // needed, the replacement is done in LoadNode::Identity().
2192 //
2193 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2194 Node* mem = in(MemNode::Memory);
2305 return LoadNode::Ideal(phase, can_reshape);
2306 }
2307
2308 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2309 Node* mem = in(MemNode::Memory);
2310 Node* value = can_see_stored_value(mem,phase);
2311 if (value != nullptr && value->is_Con() &&
2312 !value->bottom_type()->higher_equal(_type)) {
2313 // If the input to the store does not fit with the load's result type,
2314 // it must be truncated. We can't delay until Ideal call since
2315 // a singleton Value is needed for split_thru_phi optimization.
2316 int con = value->get_int();
2317 return TypeInt::make((con << 16) >> 16);
2318 }
2319 return LoadNode::Value(phase);
2320 }
2321
2322 //=============================================================================
2323 //----------------------------LoadKlassNode::make------------------------------
2324 // Polymorphic factory method:
2325 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2326 // sanity check the alias category against the created node type
2327 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2328 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2329 #ifdef _LP64
2330 if (adr_type->is_ptr_to_narrowklass()) {
2331 assert(UseCompressedClassPointers, "no compressed klasses");
2332 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2333 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2334 }
2335 #endif
2336 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2337 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2338 }
2339
2340 //------------------------------Value------------------------------------------
2341 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2342 return klass_value_common(phase);
2343 }
2344
2345 // In most cases, LoadKlassNode does not have the control input set. If the control
2381 }
2382 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2383 }
2384 if (!t->is_klass()) {
2385 // a primitive Class (e.g., int.class) has null for a klass field
2386 return TypePtr::NULL_PTR;
2387 }
2388 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2389 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2390 }
2391 // non-constant mirror, so we can't tell what's going on
2392 }
2393 if (!tinst->is_loaded())
2394 return _type; // Bail out if not loaded
2395 if (offset == oopDesc::klass_offset_in_bytes()) {
2396 return tinst->as_klass_type(true);
2397 }
2398 }
2399
2400 // Check for loading klass from an array
2401 const TypeAryPtr *tary = tp->isa_aryptr();
2402 if (tary != nullptr &&
2403 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2404 return tary->as_klass_type(true);
2405 }
2406
2407 // Check for loading klass from an array klass
2408 const TypeKlassPtr *tkls = tp->isa_klassptr();
2409 if (tkls != nullptr && !StressReflectiveCode) {
2410 if (!tkls->is_loaded())
2411 return _type; // Bail out if not loaded
2412 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2413 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2414 // // Always returning precise element type is incorrect,
2415 // // e.g., element type could be object and array may contain strings
2416 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2417
2418 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2419 // according to the element type's subclassing.
2420 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2421 }
2657
2658 // Since they are not commoned, do not hash them:
2659 return NO_HASH;
2660 }
2661
2662 //------------------------------Ideal------------------------------------------
2663 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2664 // When a store immediately follows a relevant allocation/initialization,
2665 // try to capture it into the initialization, or hoist it above.
2666 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2667 Node* p = MemNode::Ideal_common(phase, can_reshape);
2668 if (p) return (p == NodeSentinel) ? nullptr : p;
2669
2670 Node* mem = in(MemNode::Memory);
2671 Node* address = in(MemNode::Address);
2672 Node* value = in(MemNode::ValueIn);
2673 // Back-to-back stores to same address? Fold em up. Generally
2674 // unsafe if I have intervening uses... Also disallowed for StoreCM
2675 // since they must follow each StoreP operation. Redundant StoreCMs
2676 // are eliminated just before matching in final_graph_reshape.
2677 {
2678 Node* st = mem;
2679 // If Store 'st' has more than one use, we cannot fold 'st' away.
2680 // For example, 'st' might be the final state at a conditional
2681 // return. Or, 'st' might be used by some node which is live at
2682 // the same time 'st' is live, which might be unschedulable. So,
2683 // require exactly ONE user until such time as we clone 'mem' for
2684 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2685 // true).
2686 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2687 // Looking at a dead closed cycle of memory?
2688 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2689 assert(Opcode() == st->Opcode() ||
2690 st->Opcode() == Op_StoreVector ||
2691 Opcode() == Op_StoreVector ||
2692 st->Opcode() == Op_StoreVectorScatter ||
2693 Opcode() == Op_StoreVectorScatter ||
2694 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2695 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2696 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2697 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2698 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2699
2700 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2701 st->as_Store()->memory_size() <= this->memory_size()) {
2702 Node* use = st->raw_out(0);
2703 if (phase->is_IterGVN()) {
2704 phase->is_IterGVN()->rehash_node_delayed(use);
2705 }
2706 // It's OK to do this in the parser, since DU info is always accurate,
2707 // and the parser always refers to nodes via SafePointNode maps.
2708 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2709 return this;
2710 }
2711 st = st->in(MemNode::Memory);
2712 }
2713 }
2714
2715
2716 // Capture an unaliased, unconditional, simple store into an initializer.
2773 // Load then Store? Then the Store is useless
2774 if (val->is_Load() &&
2775 val->in(MemNode::Address)->eqv_uncast(adr) &&
2776 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2777 val->as_Load()->store_Opcode() == Opcode()) {
2778 result = mem;
2779 }
2780
2781 // Two stores in a row of the same value?
2782 if (result == this &&
2783 mem->is_Store() &&
2784 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2785 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2786 mem->Opcode() == Opcode()) {
2787 result = mem;
2788 }
2789
2790 // Store of zero anywhere into a freshly-allocated object?
2791 // Then the store is useless.
2792 // (It must already have been captured by the InitializeNode.)
2793 if (result == this &&
2794 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2795 // a newly allocated object is already all-zeroes everywhere
2796 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
2797 result = mem;
2798 }
2799
2800 if (result == this) {
2801 // the store may also apply to zero-bits in an earlier object
2802 Node* prev_mem = find_previous_store(phase);
2803 // Steps (a), (b): Walk past independent stores to find an exact match.
2804 if (prev_mem != nullptr) {
2805 Node* prev_val = can_see_stored_value(prev_mem, phase);
2806 if (prev_val != nullptr && prev_val == val) {
2807 // prev_val and val might differ by a cast; it would be good
2808 // to keep the more informative of the two.
2809 result = mem;
2810 }
2811 }
2812 }
2813 }
2814
2815 PhaseIterGVN* igvn = phase->is_IterGVN();
2816 if (result != this && igvn != nullptr) {
2817 MemBarNode* trailing = trailing_membar();
2818 if (trailing != nullptr) {
2819 #ifdef ASSERT
2820 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2965 Node* StoreCMNode::Identity(PhaseGVN* phase) {
2966 // No need to card mark when storing a null ptr
2967 Node* my_store = in(MemNode::OopStore);
2968 if (my_store->is_Store()) {
2969 const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
2970 if( t1 == TypePtr::NULL_PTR ) {
2971 return in(MemNode::Memory);
2972 }
2973 }
2974 return this;
2975 }
2976
2977 //=============================================================================
2978 //------------------------------Ideal---------------------------------------
2979 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
2980 Node* progress = StoreNode::Ideal(phase, can_reshape);
2981 if (progress != nullptr) return progress;
2982
2983 Node* my_store = in(MemNode::OopStore);
2984 if (my_store->is_MergeMem()) {
2985 Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
2986 set_req_X(MemNode::OopStore, mem, phase);
2987 return this;
2988 }
2989
2990 return nullptr;
2991 }
2992
2993 //------------------------------Value-----------------------------------------
2994 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
2995 // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
2996 // If extra input is TOP ==> the result is TOP
2997 const Type* t = phase->type(in(MemNode::OopStore));
2998 if (t == Type::TOP) {
2999 return Type::TOP;
3000 }
3001 return StoreNode::Value(phase);
3002 }
3003
3004
3005 //=============================================================================
3006 //----------------------------------SCMemProjNode------------------------------
3007 const Type* SCMemProjNode::Value(PhaseGVN* phase) const
3138 // Clearing a short array is faster with stores
3139 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3140 // Already know this is a large node, do not try to ideal it
3141 if (_is_large) return nullptr;
3142
3143 const int unit = BytesPerLong;
3144 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3145 if (!t) return nullptr;
3146 if (!t->is_con()) return nullptr;
3147 intptr_t raw_count = t->get_con();
3148 intptr_t size = raw_count;
3149 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3150 // Clearing nothing uses the Identity call.
3151 // Negative clears are possible on dead ClearArrays
3152 // (see jck test stmt114.stmt11402.val).
3153 if (size <= 0 || size % unit != 0) return nullptr;
3154 intptr_t count = size / unit;
3155 // Length too long; communicate this to matchers and assemblers.
3156 // Assemblers are responsible to produce fast hardware clears for it.
3157 if (size > InitArrayShortSize) {
3158 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3159 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3160 return nullptr;
3161 }
3162 if (!IdealizeClearArrayNode) return nullptr;
3163 Node *mem = in(1);
3164 if( phase->type(mem)==Type::TOP ) return nullptr;
3165 Node *adr = in(3);
3166 const Type* at = phase->type(adr);
3167 if( at==Type::TOP ) return nullptr;
3168 const TypePtr* atp = at->isa_ptr();
3169 // adjust atp to be the correct array element address type
3170 if (atp == nullptr) atp = TypePtr::BOTTOM;
3171 else atp = atp->add_offset(Type::OffsetBot);
3172 // Get base for derived pointer purposes
3173 if( adr->Opcode() != Op_AddP ) Unimplemented();
3174 Node *base = adr->in(1);
3175
3176 Node *zero = phase->makecon(TypeLong::ZERO);
3177 Node *off = phase->MakeConX(BytesPerLong);
3178 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3179 count--;
3180 while( count-- ) {
3181 mem = phase->transform(mem);
3182 adr = phase->transform(new AddPNode(base,adr,off));
3183 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3184 }
3185 return mem;
3186 }
3187
3188 //----------------------------step_through----------------------------------
3189 // Return allocation input memory edge if it is different instance
3190 // or itself if it is the one we are looking for.
3191 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3192 Node* n = *np;
3193 assert(n->is_ClearArray(), "sanity");
3194 intptr_t offset;
3195 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3196 // This method is called only before Allocate nodes are expanded
3197 // during macro nodes expansion. Before that ClearArray nodes are
3198 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3199 // Allocate nodes are expanded) which follows allocations.
3200 assert(alloc != nullptr, "should have allocation");
3201 if (alloc->_idx == instance_id) {
3202 // Can not bypass initialization of the instance we are looking for.
3203 return false;
3204 }
3205 // Otherwise skip it.
3206 InitializeNode* init = alloc->initialization();
3207 if (init != nullptr)
3208 *np = init->in(TypeFunc::Memory);
3209 else
3210 *np = alloc->in(TypeFunc::Memory);
3211 return true;
3212 }
3213
3214 //----------------------------clear_memory-------------------------------------
3215 // Generate code to initialize object storage to zero.
3216 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3217 intptr_t start_offset,
3218 Node* end_offset,
3219 PhaseGVN* phase) {
3220 intptr_t offset = start_offset;
3221
3222 int unit = BytesPerLong;
3223 if ((offset % unit) != 0) {
3224 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3225 adr = phase->transform(adr);
3226 const TypePtr* atp = TypeRawPtr::BOTTOM;
3227 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3228 mem = phase->transform(mem);
3229 offset += BytesPerInt;
3230 }
3231 assert((offset % unit) == 0, "");
3232
3233 // Initialize the remaining stuff, if any, with a ClearArray.
3234 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
3235 }
3236
3237 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3238 Node* start_offset,
3239 Node* end_offset,
3240 PhaseGVN* phase) {
3241 if (start_offset == end_offset) {
3242 // nothing to do
3243 return mem;
3244 }
3245
3246 int unit = BytesPerLong;
3247 Node* zbase = start_offset;
3248 Node* zend = end_offset;
3249
3250 // Scale to the unit required by the CPU:
3251 if (!Matcher::init_array_count_is_in_bytes) {
3252 Node* shift = phase->intcon(exact_log2(unit));
3253 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3254 zend = phase->transform(new URShiftXNode(zend, shift) );
3255 }
3256
3257 // Bulk clear double-words
3258 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3259 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3260 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
3261 return phase->transform(mem);
3262 }
3263
3264 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3265 intptr_t start_offset,
3266 intptr_t end_offset,
3267 PhaseGVN* phase) {
3268 if (start_offset == end_offset) {
3269 // nothing to do
3270 return mem;
3271 }
3272
3273 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3274 intptr_t done_offset = end_offset;
3275 if ((done_offset % BytesPerLong) != 0) {
3276 done_offset -= BytesPerInt;
3277 }
3278 if (done_offset > start_offset) {
3279 mem = clear_memory(ctl, mem, dest,
3280 start_offset, phase->MakeConX(done_offset), phase);
3281 }
3282 if (done_offset < end_offset) { // emit the final 32-bit store
3283 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3284 adr = phase->transform(adr);
3285 const TypePtr* atp = TypeRawPtr::BOTTOM;
3286 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3287 mem = phase->transform(mem);
3288 done_offset += BytesPerInt;
3289 }
3290 assert(done_offset == end_offset, "");
3291 return mem;
3292 }
3293
3294 //=============================================================================
3295 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3296 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3297 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3298 #ifdef ASSERT
3299 , _pair_idx(0)
3300 #endif
3301 {
3302 init_class_id(Class_MemBar);
3303 Node* top = C->top();
3304 init_req(TypeFunc::I_O,top);
3305 init_req(TypeFunc::FramePtr,top);
3306 init_req(TypeFunc::ReturnAdr,top);
3411 PhaseIterGVN* igvn = phase->is_IterGVN();
3412 remove(igvn);
3413 // Must return either the original node (now dead) or a new node
3414 // (Do not return a top here, since that would break the uniqueness of top.)
3415 return new ConINode(TypeInt::ZERO);
3416 }
3417 }
3418 return progress ? this : nullptr;
3419 }
3420
3421 //------------------------------Value------------------------------------------
3422 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3423 if( !in(0) ) return Type::TOP;
3424 if( phase->type(in(0)) == Type::TOP )
3425 return Type::TOP;
3426 return TypeTuple::MEMBAR;
3427 }
3428
3429 //------------------------------match------------------------------------------
3430 // Construct projections for memory.
3431 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3432 switch (proj->_con) {
3433 case TypeFunc::Control:
3434 case TypeFunc::Memory:
3435 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3436 }
3437 ShouldNotReachHere();
3438 return nullptr;
3439 }
3440
3441 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3442 trailing->_kind = TrailingStore;
3443 leading->_kind = LeadingStore;
3444 #ifdef ASSERT
3445 trailing->_pair_idx = leading->_idx;
3446 leading->_pair_idx = leading->_idx;
3447 #endif
3448 }
3449
3450 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3451 trailing->_kind = TrailingLoadStore;
3698 return (req() > RawStores);
3699 }
3700
3701 void InitializeNode::set_complete(PhaseGVN* phase) {
3702 assert(!is_complete(), "caller responsibility");
3703 _is_complete = Complete;
3704
3705 // After this node is complete, it contains a bunch of
3706 // raw-memory initializations. There is no need for
3707 // it to have anything to do with non-raw memory effects.
3708 // Therefore, tell all non-raw users to re-optimize themselves,
3709 // after skipping the memory effects of this initialization.
3710 PhaseIterGVN* igvn = phase->is_IterGVN();
3711 if (igvn) igvn->add_users_to_worklist(this);
3712 }
3713
3714 // convenience function
3715 // return false if the init contains any stores already
3716 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3717 InitializeNode* init = initialization();
3718 if (init == nullptr || init->is_complete()) return false;
3719 init->remove_extra_zeroes();
3720 // for now, if this allocation has already collected any inits, bail:
3721 if (init->is_non_zero()) return false;
3722 init->set_complete(phase);
3723 return true;
3724 }
3725
3726 void InitializeNode::remove_extra_zeroes() {
3727 if (req() == RawStores) return;
3728 Node* zmem = zero_memory();
3729 uint fill = RawStores;
3730 for (uint i = fill; i < req(); i++) {
3731 Node* n = in(i);
3732 if (n->is_top() || n == zmem) continue; // skip
3733 if (fill < i) set_req(fill, n); // compact
3734 ++fill;
3735 }
3736 // delete any empty spaces created:
3737 while (fill < req()) {
3738 del_req(fill);
3876 // store node that we'd like to capture. We need to check
3877 // the uses of the MergeMemNode.
3878 mems.push(n);
3879 }
3880 } else if (n->is_Mem()) {
3881 Node* other_adr = n->in(MemNode::Address);
3882 if (other_adr == adr) {
3883 failed = true;
3884 break;
3885 } else {
3886 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3887 if (other_t_adr != nullptr) {
3888 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3889 if (other_alias_idx == alias_idx) {
3890 // A load from the same memory slice as the store right
3891 // after the InitializeNode. We check the control of the
3892 // object/array that is loaded from. If it's the same as
3893 // the store control then we cannot capture the store.
3894 assert(!n->is_Store(), "2 stores to same slice on same control?");
3895 Node* base = other_adr;
3896 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3897 base = base->in(AddPNode::Base);
3898 if (base != nullptr) {
3899 base = base->uncast();
3900 if (base->is_Proj() && base->in(0) == alloc) {
3901 failed = true;
3902 break;
3903 }
3904 }
3905 }
3906 }
3907 }
3908 } else {
3909 failed = true;
3910 break;
3911 }
3912 }
3913 }
3914 }
3915 if (failed) {
4462 // z's_done 12 16 16 16 12 16 12
4463 // z's_needed 12 16 16 16 16 16 16
4464 // zsize 0 0 0 0 4 0 4
4465 if (next_full_store < 0) {
4466 // Conservative tack: Zero to end of current word.
4467 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4468 } else {
4469 // Zero to beginning of next fully initialized word.
4470 // Or, don't zero at all, if we are already in that word.
4471 assert(next_full_store >= zeroes_needed, "must go forward");
4472 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4473 zeroes_needed = next_full_store;
4474 }
4475 }
4476
4477 if (zeroes_needed > zeroes_done) {
4478 intptr_t zsize = zeroes_needed - zeroes_done;
4479 // Do some incremental zeroing on rawmem, in parallel with inits.
4480 zeroes_done = align_down(zeroes_done, BytesPerInt);
4481 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4482 zeroes_done, zeroes_needed,
4483 phase);
4484 zeroes_done = zeroes_needed;
4485 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4486 do_zeroing = false; // leave the hole, next time
4487 }
4488 }
4489
4490 // Collect the store and move on:
4491 phase->replace_input_of(st, MemNode::Memory, inits);
4492 inits = st; // put it on the linearized chain
4493 set_req(i, zmem); // unhook from previous position
4494
4495 if (zeroes_done == st_off)
4496 zeroes_done = next_init_off;
4497
4498 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4499
4500 #ifdef ASSERT
4501 // Various order invariants. Weaker than stores_are_sane because
4521 remove_extra_zeroes(); // clear out all the zmems left over
4522 add_req(inits);
4523
4524 if (!(UseTLAB && ZeroTLAB)) {
4525 // If anything remains to be zeroed, zero it all now.
4526 zeroes_done = align_down(zeroes_done, BytesPerInt);
4527 // if it is the last unused 4 bytes of an instance, forget about it
4528 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4529 if (zeroes_done + BytesPerLong >= size_limit) {
4530 AllocateNode* alloc = allocation();
4531 assert(alloc != nullptr, "must be present");
4532 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
4533 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4534 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4535 if (zeroes_done == k->layout_helper())
4536 zeroes_done = size_limit;
4537 }
4538 }
4539 if (zeroes_done < size_limit) {
4540 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4541 zeroes_done, size_in_bytes, phase);
4542 }
4543 }
4544
4545 set_complete(phase);
4546 return rawmem;
4547 }
4548
4549
4550 #ifdef ASSERT
4551 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
4552 if (is_complete())
4553 return true; // stores could be anything at this point
4554 assert(allocation() != nullptr, "must be present");
4555 intptr_t last_off = allocation()->minimum_header_size();
4556 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4557 Node* st = in(i);
4558 intptr_t st_off = get_store_offset(st, phase);
4559 if (st_off < 0) continue; // ignore dead garbage
4560 if (last_off > st_off) {
|
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "compiler/compileLog.hpp"
30 #include "gc/shared/barrierSet.hpp"
31 #include "gc/shared/c2/barrierSetC2.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "memory/allocation.inline.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "oops/objArrayKlass.hpp"
36 #include "opto/addnode.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/cfgnode.hpp"
39 #include "opto/regalloc.hpp"
40 #include "opto/compile.hpp"
41 #include "opto/connode.hpp"
42 #include "opto/convertnode.hpp"
43 #include "opto/inlinetypenode.hpp"
44 #include "opto/loopnode.hpp"
45 #include "opto/machnode.hpp"
46 #include "opto/matcher.hpp"
47 #include "opto/memnode.hpp"
48 #include "opto/mulnode.hpp"
49 #include "opto/narrowptrnode.hpp"
50 #include "opto/phaseX.hpp"
51 #include "opto/regmask.hpp"
52 #include "opto/rootnode.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/macros.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #include "utilities/vmError.hpp"
59
60 // Portions of code courtesy of Clifford Click
61
62 // Optimization - Graph Style
63
217 bool is_instance = t_oop->is_known_instance_field();
218 PhaseIterGVN *igvn = phase->is_IterGVN();
219 if (is_instance && igvn != nullptr && result->is_Phi()) {
220 PhiNode *mphi = result->as_Phi();
221 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
222 const TypePtr *t = mphi->adr_type();
223 bool do_split = false;
224 // In the following cases, Load memory input can be further optimized based on
225 // its precise address type
226 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
227 do_split = true;
228 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
229 const TypeOopPtr* mem_t =
230 t->is_oopptr()->cast_to_exactness(true)
231 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
232 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
233 if (t_oop->is_aryptr()) {
234 mem_t = mem_t->is_aryptr()
235 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
236 ->cast_to_size(t_oop->is_aryptr()->size())
237 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
238 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
239 ->with_offset(t_oop->is_aryptr()->offset())
240 ->is_aryptr();
241 }
242 do_split = mem_t == t_oop;
243 }
244 if (do_split) {
245 // clone the Phi with our address type
246 result = mphi->split_out_instance(t_adr, igvn);
247 } else {
248 // TODO 8325106
249 // assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
250 }
251 }
252 return result;
253 }
254
255 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
256 uint alias_idx = phase->C->get_alias_index(tp);
257 Node *mem = mmem;
258 #ifdef ASSERT
259 {
260 // Check that current type is consistent with the alias index used during graph construction
261 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
262 bool consistent = adr_check == nullptr || adr_check->empty() ||
263 phase->C->must_alias(adr_check, alias_idx );
264 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
265 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
266 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
267 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
268 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
269 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
270 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
271 // don't assert if it is dead code.
272 consistent = true;
273 }
274 if( !consistent ) {
275 st->print("alias_idx==%d, adr_check==", alias_idx);
276 if( adr_check == nullptr ) {
277 st->print("null");
278 } else {
279 adr_check->dump();
280 }
281 st->cr();
282 print_alias_types();
283 assert(consistent, "adr_check must match alias idx");
284 }
285 }
286 #endif
966 Node* ld = gvn.transform(load);
967 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
968 }
969
970 return load;
971 }
972
973 //------------------------------hash-------------------------------------------
974 uint LoadNode::hash() const {
975 // unroll addition of interesting fields
976 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
977 }
978
979 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
980 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
981 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
982 bool is_stable_ary = FoldStableValues &&
983 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
984 tp->isa_aryptr()->is_stable();
985
986 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
987 }
988
989 return false;
990 }
991
992 // Is the value loaded previously stored by an arraycopy? If so return
993 // a load node that reads from the source array so we may be able to
994 // optimize out the ArrayCopy node later.
995 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
996 Node* ld_adr = in(MemNode::Address);
997 intptr_t ld_off = 0;
998 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
999 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1000 if (ac != nullptr) {
1001 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1002
1003 Node* mem = ac->in(TypeFunc::Memory);
1004 Node* ctl = ac->in(0);
1005 Node* src = ac->in(ArrayCopyNode::Src);
1006
1014 assert(ld_alloc != nullptr, "need an alloc");
1015 assert(addp->is_AddP(), "address must be addp");
1016 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1017 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1018 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1019 addp->set_req(AddPNode::Base, src);
1020 addp->set_req(AddPNode::Address, src);
1021 } else {
1022 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1023 ac->as_ArrayCopy()->is_copyof_validated() ||
1024 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1025 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1026 addp->set_req(AddPNode::Base, src);
1027 addp->set_req(AddPNode::Address, src);
1028
1029 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1030 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1031 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1032
1033 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1034 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1035
1036 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1037 #ifdef _LP64
1038 diff = phase->transform(new ConvI2LNode(diff));
1039 #endif
1040 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1041
1042 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1043 addp->set_req(AddPNode::Offset, offset);
1044 }
1045 addp = phase->transform(addp);
1046 #ifdef ASSERT
1047 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1048 ld->_adr_type = adr_type;
1049 #endif
1050 ld->set_req(MemNode::Address, addp);
1051 ld->set_req(0, ctl);
1052 ld->set_req(MemNode::Memory, mem);
1053 // load depends on the tests that validate the arraycopy
1054 ld->_control_dependency = UnknownControl;
1135 // Same base, same offset.
1136 // Possible improvement for arrays: check index value instead of absolute offset.
1137
1138 // At this point we have proven something like this setup:
1139 // B = << base >>
1140 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1141 // S = StoreQ(AddP( B , #Off), V)
1142 // (Actually, we haven't yet proven the Q's are the same.)
1143 // In other words, we are loading from a casted version of
1144 // the same pointer-and-offset that we stored to.
1145 // Casted version may carry a dependency and it is respected.
1146 // Thus, we are able to replace L by V.
1147 }
1148 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1149 if (store_Opcode() != st->Opcode()) {
1150 return nullptr;
1151 }
1152 // LoadVector/StoreVector needs additional check to ensure the types match.
1153 if (st->is_StoreVector()) {
1154 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1155 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1156 if (in_vt != out_vt) {
1157 return nullptr;
1158 }
1159 }
1160 return st->in(MemNode::ValueIn);
1161 }
1162
1163 // A load from a freshly-created object always returns zero.
1164 // (This can happen after LoadNode::Ideal resets the load's memory input
1165 // to find_captured_store, which returned InitializeNode::zero_memory.)
1166 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1167 (st->in(0) == ld_alloc) &&
1168 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1169 // return a zero value for the load's basic type
1170 // (This is one of the few places where a generic PhaseTransform
1171 // can create new nodes. Think of it as lazily manifesting
1172 // virtually pre-existing constants.)
1173 Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1174 if (default_value != nullptr) {
1175 return default_value;
1176 }
1177 assert(ld_alloc->in(AllocateNode::RawDefaultValue) == nullptr, "default value may not be null");
1178 if (memory_type() != T_VOID) {
1179 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1180 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1181 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1182 // by the ArrayCopyNode.
1183 return phase->zerocon(memory_type());
1184 }
1185 } else {
1186 // TODO: materialize all-zero vector constant
1187 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1188 }
1189 }
1190
1191 // A load from an initialization barrier can match a captured store.
1192 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1193 InitializeNode* init = st->in(0)->as_Initialize();
1194 AllocateNode* alloc = init->allocation();
1195 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1196 // examine a captured store value
1197 st = init->find_captured_store(ld_off, memory_size(), phase);
1225 //----------------------is_instance_field_load_with_local_phi------------------
1226 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1227 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1228 in(Address)->is_AddP() ) {
1229 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1230 // Only instances and boxed values.
1231 if( t_oop != nullptr &&
1232 (t_oop->is_ptr_to_boxed_value() ||
1233 t_oop->is_known_instance_field()) &&
1234 t_oop->offset() != Type::OffsetBot &&
1235 t_oop->offset() != Type::OffsetTop) {
1236 return true;
1237 }
1238 }
1239 return false;
1240 }
1241
1242 //------------------------------Identity---------------------------------------
1243 // Loads are identity if previous store is to same address
1244 Node* LoadNode::Identity(PhaseGVN* phase) {
1245 // Loading from an InlineType? The InlineType has the values of
1246 // all fields as input. Look for the field with matching offset.
1247 Node* addr = in(Address);
1248 intptr_t offset;
1249 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1250 if (base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1251 Node* value = base->as_InlineType()->field_value_by_offset((int)offset, true);
1252 if (value != nullptr) {
1253 if (Opcode() == Op_LoadN) {
1254 // Encode oop value if we are loading a narrow oop
1255 assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1256 value = phase->transform(new EncodePNode(value, bottom_type()));
1257 }
1258 return value;
1259 }
1260 }
1261
1262 // If the previous store-maker is the right kind of Store, and the store is
1263 // to the same address, then we are equal to the value stored.
1264 Node* mem = in(Memory);
1265 Node* value = can_see_stored_value(mem, phase);
1266 if( value ) {
1267 // byte, short & char stores truncate naturally.
1268 // A load has to load the truncated value which requires
1269 // some sort of masking operation and that requires an
1270 // Ideal call instead of an Identity call.
1271 if (memory_size() < BytesPerInt) {
1272 // If the input to the store does not fit with the load's result type,
1273 // it must be truncated via an Ideal call.
1274 if (!phase->type(value)->higher_equal(phase->type(this)))
1275 return this;
1276 }
1277 // (This works even when value is a Con, but LoadNode::Value
1278 // usually runs first, producing the singleton type of the Con.)
1279 if (!has_pinned_control_dependency() || value->is_Con()) {
1280 return value;
1281 } else {
2000 }
2001 }
2002
2003 // Don't do this for integer types. There is only potential profit if
2004 // the element type t is lower than _type; that is, for int types, if _type is
2005 // more restrictive than t. This only happens here if one is short and the other
2006 // char (both 16 bits), and in those cases we've made an intentional decision
2007 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2008 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2009 //
2010 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2011 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2012 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2013 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2014 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2015 // In fact, that could have been the original type of p1, and p1 could have
2016 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2017 // expression (LShiftL quux 3) independently optimized to the constant 8.
2018 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2019 && (_type->isa_vect() == nullptr)
2020 && !ary->is_flat()
2021 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2022 // t might actually be lower than _type, if _type is a unique
2023 // concrete subclass of abstract class t.
2024 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2025 const Type* jt = t->join_speculative(_type);
2026 // In any case, do not allow the join, per se, to empty out the type.
2027 if (jt->empty() && !t->empty()) {
2028 // This can happen if a interface-typed array narrows to a class type.
2029 jt = _type;
2030 }
2031 #ifdef ASSERT
2032 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2033 // The pointers in the autobox arrays are always non-null
2034 Node* base = adr->in(AddPNode::Base);
2035 if ((base != nullptr) && base->is_DecodeN()) {
2036 // Get LoadN node which loads IntegerCache.cache field
2037 base = base->in(1);
2038 }
2039 if ((base != nullptr) && base->is_Con()) {
2040 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2041 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2042 // It could be narrow oop
2043 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2044 }
2045 }
2046 }
2047 #endif
2048 return jt;
2049 }
2050 }
2051 } else if (tp->base() == Type::InstPtr) {
2052 assert( off != Type::OffsetBot ||
2053 // arrays can be cast to Objects
2054 !tp->isa_instptr() ||
2055 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2056 // Default value load
2057 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2058 // unsafe field access may not have a constant offset
2059 C->has_unsafe_access(),
2060 "Field accesses must be precise" );
2061 // For oop loads, we expect the _type to be precise.
2062
2063 const TypeInstPtr* tinst = tp->is_instptr();
2064 BasicType bt = memory_type();
2065
2066 // Optimize loads from constant fields.
2067 ciObject* const_oop = tinst->const_oop();
2068 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2069 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2070 if (con_type != nullptr) {
2071 return con_type;
2072 }
2073 }
2074 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2075 assert(off != Type::OffsetBot ||
2076 !tp->isa_instklassptr() ||
2077 // arrays can be cast to Objects
2078 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2079 // also allow array-loading from the primary supertype
2080 // array during subtype checks
2081 Opcode() == Op_LoadKlass,
2082 "Field accesses must be precise");
2083 // For klass/static loads, we expect the _type to be precise
2084 } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
2085 if (adr->is_Load() && off == 0) {
2086 /* With mirrors being an indirect in the Klass*
2087 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2088 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2089 *
2090 * So check the type and klass of the node before the LoadP.
2091 */
2092 Node* adr2 = adr->in(MemNode::Address);
2093 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2094 if (tkls != nullptr) {
2095 if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2096 ciKlass* klass = tkls->exact_klass();
2097 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2098 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2099 return TypeInstPtr::make(klass->java_mirror());
2100 }
2101 }
2102 } else {
2103 // Check for a load of the default value offset from the InlineKlassFixedBlock:
2104 // LoadI(LoadP(inline_klass, adr_inlineklass_fixed_block_offset), default_value_offset_offset)
2105 // TODO 8325106 remove?
2106 intptr_t offset = 0;
2107 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2108 if (base != nullptr && base->is_Load() && offset == in_bytes(InlineKlass::default_value_offset_offset())) {
2109 const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
2110 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact() && tkls->exact_klass()->is_inlinetype() &&
2111 tkls->offset() == in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())) {
2112 assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
2113 assert(Opcode() == Op_LoadI, "must load an int from fixed block");
2114 return TypeInt::make(tkls->exact_klass()->as_inline_klass()->default_value_offset());
2115 }
2116 }
2117 }
2118 }
2119
2120 const TypeKlassPtr *tkls = tp->isa_klassptr();
2121 if (tkls != nullptr) {
2122 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2123 ciKlass* klass = tkls->exact_klass();
2124 // We are loading a field from a Klass metaobject whose identity
2125 // is known at compile time (the type is "exact" or "precise").
2126 // Check for fields we know are maintained as constants by the VM.
2127 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2128 // The field is Klass::_super_check_offset. Return its (constant) value.
2129 // (Folds up type checking code.)
2130 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2131 return TypeInt::make(klass->super_check_offset());
2132 }
2133 // Compute index into primary_supers array
2134 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2135 // Check for overflowing; use unsigned compare to handle the negative case.
2201 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2202 Node* value = can_see_stored_value(mem,phase);
2203 if (value != nullptr && value->is_Con()) {
2204 assert(value->bottom_type()->higher_equal(_type),"sanity");
2205 return value->bottom_type();
2206 }
2207 }
2208
2209 bool is_vect = (_type->isa_vect() != nullptr);
2210 if (is_instance && !is_vect) {
2211 // If we have an instance type and our memory input is the
2212 // programs's initial memory state, there is no matching store,
2213 // so just return a zero of the appropriate type -
2214 // except if it is vectorized - then we have no zero constant.
2215 Node *mem = in(MemNode::Memory);
2216 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2217 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2218 return Type::get_zero_type(_type->basic_type());
2219 }
2220 }
2221 Node* alloc = is_new_object_mark_load();
2222 if (alloc != nullptr) {
2223 if (EnableValhalla) {
2224 // The mark word may contain property bits (inline, flat, null-free)
2225 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2226 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2227 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2228 return TypeX::make(tkls->exact_klass()->prototype_header().value());
2229 }
2230 } else {
2231 return TypeX::make(markWord::prototype().value());
2232 }
2233 }
2234
2235 return _type;
2236 }
2237
2238 //------------------------------match_edge-------------------------------------
2239 // Do we Match on this edge index or not? Match only the address.
2240 uint LoadNode::match_edge(uint idx) const {
2241 return idx == MemNode::Address;
2242 }
2243
2244 //--------------------------LoadBNode::Ideal--------------------------------------
2245 //
2246 // If the previous store is to the same address as this load,
2247 // and the value stored was larger than a byte, replace this load
2248 // with the value stored truncated to a byte. If no truncation is
2249 // needed, the replacement is done in LoadNode::Identity().
2250 //
2251 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2252 Node* mem = in(MemNode::Memory);
2363 return LoadNode::Ideal(phase, can_reshape);
2364 }
2365
2366 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2367 Node* mem = in(MemNode::Memory);
2368 Node* value = can_see_stored_value(mem,phase);
2369 if (value != nullptr && value->is_Con() &&
2370 !value->bottom_type()->higher_equal(_type)) {
2371 // If the input to the store does not fit with the load's result type,
2372 // it must be truncated. We can't delay until Ideal call since
2373 // a singleton Value is needed for split_thru_phi optimization.
2374 int con = value->get_int();
2375 return TypeInt::make((con << 16) >> 16);
2376 }
2377 return LoadNode::Value(phase);
2378 }
2379
2380 //=============================================================================
2381 //----------------------------LoadKlassNode::make------------------------------
2382 // Polymorphic factory method:
2383 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2384 const TypeKlassPtr* tk) {
2385 // sanity check the alias category against the created node type
2386 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2387 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2388 #ifdef _LP64
2389 if (adr_type->is_ptr_to_narrowklass()) {
2390 assert(UseCompressedClassPointers, "no compressed klasses");
2391 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2392 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2393 }
2394 #endif
2395 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2396 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2397 }
2398
2399 //------------------------------Value------------------------------------------
2400 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2401 return klass_value_common(phase);
2402 }
2403
2404 // In most cases, LoadKlassNode does not have the control input set. If the control
2440 }
2441 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2442 }
2443 if (!t->is_klass()) {
2444 // a primitive Class (e.g., int.class) has null for a klass field
2445 return TypePtr::NULL_PTR;
2446 }
2447 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2448 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2449 }
2450 // non-constant mirror, so we can't tell what's going on
2451 }
2452 if (!tinst->is_loaded())
2453 return _type; // Bail out if not loaded
2454 if (offset == oopDesc::klass_offset_in_bytes()) {
2455 return tinst->as_klass_type(true);
2456 }
2457 }
2458
2459 // Check for loading klass from an array
2460 const TypeAryPtr* tary = tp->isa_aryptr();
2461 if (tary != nullptr &&
2462 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2463 return tary->as_klass_type(true);
2464 }
2465
2466 // Check for loading klass from an array klass
2467 const TypeKlassPtr *tkls = tp->isa_klassptr();
2468 if (tkls != nullptr && !StressReflectiveCode) {
2469 if (!tkls->is_loaded())
2470 return _type; // Bail out if not loaded
2471 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2472 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2473 // // Always returning precise element type is incorrect,
2474 // // e.g., element type could be object and array may contain strings
2475 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2476
2477 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2478 // according to the element type's subclassing.
2479 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2480 }
2716
2717 // Since they are not commoned, do not hash them:
2718 return NO_HASH;
2719 }
2720
2721 //------------------------------Ideal------------------------------------------
2722 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2723 // When a store immediately follows a relevant allocation/initialization,
2724 // try to capture it into the initialization, or hoist it above.
2725 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2726 Node* p = MemNode::Ideal_common(phase, can_reshape);
2727 if (p) return (p == NodeSentinel) ? nullptr : p;
2728
2729 Node* mem = in(MemNode::Memory);
2730 Node* address = in(MemNode::Address);
2731 Node* value = in(MemNode::ValueIn);
2732 // Back-to-back stores to same address? Fold em up. Generally
2733 // unsafe if I have intervening uses... Also disallowed for StoreCM
2734 // since they must follow each StoreP operation. Redundant StoreCMs
2735 // are eliminated just before matching in final_graph_reshape.
2736 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
2737 Node* st = mem;
2738 // If Store 'st' has more than one use, we cannot fold 'st' away.
2739 // For example, 'st' might be the final state at a conditional
2740 // return. Or, 'st' might be used by some node which is live at
2741 // the same time 'st' is live, which might be unschedulable. So,
2742 // require exactly ONE user until such time as we clone 'mem' for
2743 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2744 // true).
2745 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2746 // Looking at a dead closed cycle of memory?
2747 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2748 assert(Opcode() == st->Opcode() ||
2749 st->Opcode() == Op_StoreVector ||
2750 Opcode() == Op_StoreVector ||
2751 st->Opcode() == Op_StoreVectorScatter ||
2752 Opcode() == Op_StoreVectorScatter ||
2753 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2754 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2755 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2756 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
2757 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2758 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2759
2760 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2761 st->as_Store()->memory_size() <= this->memory_size()) {
2762 Node* use = st->raw_out(0);
2763 if (phase->is_IterGVN()) {
2764 phase->is_IterGVN()->rehash_node_delayed(use);
2765 }
2766 // It's OK to do this in the parser, since DU info is always accurate,
2767 // and the parser always refers to nodes via SafePointNode maps.
2768 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2769 return this;
2770 }
2771 st = st->in(MemNode::Memory);
2772 }
2773 }
2774
2775
2776 // Capture an unaliased, unconditional, simple store into an initializer.
2833 // Load then Store? Then the Store is useless
2834 if (val->is_Load() &&
2835 val->in(MemNode::Address)->eqv_uncast(adr) &&
2836 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2837 val->as_Load()->store_Opcode() == Opcode()) {
2838 result = mem;
2839 }
2840
2841 // Two stores in a row of the same value?
2842 if (result == this &&
2843 mem->is_Store() &&
2844 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2845 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2846 mem->Opcode() == Opcode()) {
2847 result = mem;
2848 }
2849
2850 // Store of zero anywhere into a freshly-allocated object?
2851 // Then the store is useless.
2852 // (It must already have been captured by the InitializeNode.)
2853 if (result == this && ReduceFieldZeroing) {
2854 // a newly allocated object is already all-zeroes everywhere
2855 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
2856 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
2857 result = mem;
2858 }
2859
2860 if (result == this && phase->type(val)->is_zero_type()) {
2861 // the store may also apply to zero-bits in an earlier object
2862 Node* prev_mem = find_previous_store(phase);
2863 // Steps (a), (b): Walk past independent stores to find an exact match.
2864 if (prev_mem != nullptr) {
2865 Node* prev_val = can_see_stored_value(prev_mem, phase);
2866 if (prev_val != nullptr && prev_val == val) {
2867 // prev_val and val might differ by a cast; it would be good
2868 // to keep the more informative of the two.
2869 result = mem;
2870 }
2871 }
2872 }
2873 }
2874
2875 PhaseIterGVN* igvn = phase->is_IterGVN();
2876 if (result != this && igvn != nullptr) {
2877 MemBarNode* trailing = trailing_membar();
2878 if (trailing != nullptr) {
2879 #ifdef ASSERT
2880 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
3025 Node* StoreCMNode::Identity(PhaseGVN* phase) {
3026 // No need to card mark when storing a null ptr
3027 Node* my_store = in(MemNode::OopStore);
3028 if (my_store->is_Store()) {
3029 const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
3030 if( t1 == TypePtr::NULL_PTR ) {
3031 return in(MemNode::Memory);
3032 }
3033 }
3034 return this;
3035 }
3036
3037 //=============================================================================
3038 //------------------------------Ideal---------------------------------------
3039 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
3040 Node* progress = StoreNode::Ideal(phase, can_reshape);
3041 if (progress != nullptr) return progress;
3042
3043 Node* my_store = in(MemNode::OopStore);
3044 if (my_store->is_MergeMem()) {
3045 if (oop_alias_idx() != phase->C->get_alias_index(TypeAryPtr::INLINES) ||
3046 phase->C->flat_accesses_share_alias()) {
3047 // The alias that was recorded is no longer accurate enough.
3048 Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
3049 set_req_X(MemNode::OopStore, mem, phase);
3050 return this;
3051 }
3052 }
3053
3054 return nullptr;
3055 }
3056
3057 //------------------------------Value-----------------------------------------
3058 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
3059 // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
3060 // If extra input is TOP ==> the result is TOP
3061 const Type* t = phase->type(in(MemNode::OopStore));
3062 if (t == Type::TOP) {
3063 return Type::TOP;
3064 }
3065 return StoreNode::Value(phase);
3066 }
3067
3068
3069 //=============================================================================
3070 //----------------------------------SCMemProjNode------------------------------
3071 const Type* SCMemProjNode::Value(PhaseGVN* phase) const
3202 // Clearing a short array is faster with stores
3203 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3204 // Already know this is a large node, do not try to ideal it
3205 if (_is_large) return nullptr;
3206
3207 const int unit = BytesPerLong;
3208 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3209 if (!t) return nullptr;
3210 if (!t->is_con()) return nullptr;
3211 intptr_t raw_count = t->get_con();
3212 intptr_t size = raw_count;
3213 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3214 // Clearing nothing uses the Identity call.
3215 // Negative clears are possible on dead ClearArrays
3216 // (see jck test stmt114.stmt11402.val).
3217 if (size <= 0 || size % unit != 0) return nullptr;
3218 intptr_t count = size / unit;
3219 // Length too long; communicate this to matchers and assemblers.
3220 // Assemblers are responsible to produce fast hardware clears for it.
3221 if (size > InitArrayShortSize) {
3222 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3223 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3224 return nullptr;
3225 }
3226 if (!IdealizeClearArrayNode) return nullptr;
3227 Node *mem = in(1);
3228 if( phase->type(mem)==Type::TOP ) return nullptr;
3229 Node *adr = in(3);
3230 const Type* at = phase->type(adr);
3231 if( at==Type::TOP ) return nullptr;
3232 const TypePtr* atp = at->isa_ptr();
3233 // adjust atp to be the correct array element address type
3234 if (atp == nullptr) atp = TypePtr::BOTTOM;
3235 else atp = atp->add_offset(Type::OffsetBot);
3236 // Get base for derived pointer purposes
3237 if( adr->Opcode() != Op_AddP ) Unimplemented();
3238 Node *base = adr->in(1);
3239
3240 Node *val = in(4);
3241 Node *off = phase->MakeConX(BytesPerLong);
3242 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3243 count--;
3244 while( count-- ) {
3245 mem = phase->transform(mem);
3246 adr = phase->transform(new AddPNode(base,adr,off));
3247 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3248 }
3249 return mem;
3250 }
3251
3252 //----------------------------step_through----------------------------------
3253 // Return allocation input memory edge if it is different instance
3254 // or itself if it is the one we are looking for.
3255 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3256 Node* n = *np;
3257 assert(n->is_ClearArray(), "sanity");
3258 intptr_t offset;
3259 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3260 // This method is called only before Allocate nodes are expanded
3261 // during macro nodes expansion. Before that ClearArray nodes are
3262 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3263 // Allocate nodes are expanded) which follows allocations.
3264 assert(alloc != nullptr, "should have allocation");
3265 if (alloc->_idx == instance_id) {
3266 // Can not bypass initialization of the instance we are looking for.
3267 return false;
3268 }
3269 // Otherwise skip it.
3270 InitializeNode* init = alloc->initialization();
3271 if (init != nullptr)
3272 *np = init->in(TypeFunc::Memory);
3273 else
3274 *np = alloc->in(TypeFunc::Memory);
3275 return true;
3276 }
3277
3278 //----------------------------clear_memory-------------------------------------
3279 // Generate code to initialize object storage to zero.
3280 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3281 Node* val,
3282 Node* raw_val,
3283 intptr_t start_offset,
3284 Node* end_offset,
3285 PhaseGVN* phase) {
3286 intptr_t offset = start_offset;
3287
3288 int unit = BytesPerLong;
3289 if ((offset % unit) != 0) {
3290 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3291 adr = phase->transform(adr);
3292 const TypePtr* atp = TypeRawPtr::BOTTOM;
3293 if (val != nullptr) {
3294 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3295 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3296 } else {
3297 assert(raw_val == nullptr, "val may not be null");
3298 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3299 }
3300 mem = phase->transform(mem);
3301 offset += BytesPerInt;
3302 }
3303 assert((offset % unit) == 0, "");
3304
3305 // Initialize the remaining stuff, if any, with a ClearArray.
3306 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3307 }
3308
3309 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3310 Node* raw_val,
3311 Node* start_offset,
3312 Node* end_offset,
3313 PhaseGVN* phase) {
3314 if (start_offset == end_offset) {
3315 // nothing to do
3316 return mem;
3317 }
3318
3319 int unit = BytesPerLong;
3320 Node* zbase = start_offset;
3321 Node* zend = end_offset;
3322
3323 // Scale to the unit required by the CPU:
3324 if (!Matcher::init_array_count_is_in_bytes) {
3325 Node* shift = phase->intcon(exact_log2(unit));
3326 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3327 zend = phase->transform(new URShiftXNode(zend, shift) );
3328 }
3329
3330 // Bulk clear double-words
3331 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3332 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3333 if (raw_val == nullptr) {
3334 raw_val = phase->MakeConX(0);
3335 }
3336 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3337 return phase->transform(mem);
3338 }
3339
3340 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3341 Node* val,
3342 Node* raw_val,
3343 intptr_t start_offset,
3344 intptr_t end_offset,
3345 PhaseGVN* phase) {
3346 if (start_offset == end_offset) {
3347 // nothing to do
3348 return mem;
3349 }
3350
3351 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3352 intptr_t done_offset = end_offset;
3353 if ((done_offset % BytesPerLong) != 0) {
3354 done_offset -= BytesPerInt;
3355 }
3356 if (done_offset > start_offset) {
3357 mem = clear_memory(ctl, mem, dest, val, raw_val,
3358 start_offset, phase->MakeConX(done_offset), phase);
3359 }
3360 if (done_offset < end_offset) { // emit the final 32-bit store
3361 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3362 adr = phase->transform(adr);
3363 const TypePtr* atp = TypeRawPtr::BOTTOM;
3364 if (val != nullptr) {
3365 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3366 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3367 } else {
3368 assert(raw_val == nullptr, "val may not be null");
3369 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3370 }
3371 mem = phase->transform(mem);
3372 done_offset += BytesPerInt;
3373 }
3374 assert(done_offset == end_offset, "");
3375 return mem;
3376 }
3377
3378 //=============================================================================
3379 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3380 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3381 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3382 #ifdef ASSERT
3383 , _pair_idx(0)
3384 #endif
3385 {
3386 init_class_id(Class_MemBar);
3387 Node* top = C->top();
3388 init_req(TypeFunc::I_O,top);
3389 init_req(TypeFunc::FramePtr,top);
3390 init_req(TypeFunc::ReturnAdr,top);
3495 PhaseIterGVN* igvn = phase->is_IterGVN();
3496 remove(igvn);
3497 // Must return either the original node (now dead) or a new node
3498 // (Do not return a top here, since that would break the uniqueness of top.)
3499 return new ConINode(TypeInt::ZERO);
3500 }
3501 }
3502 return progress ? this : nullptr;
3503 }
3504
3505 //------------------------------Value------------------------------------------
3506 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3507 if( !in(0) ) return Type::TOP;
3508 if( phase->type(in(0)) == Type::TOP )
3509 return Type::TOP;
3510 return TypeTuple::MEMBAR;
3511 }
3512
3513 //------------------------------match------------------------------------------
3514 // Construct projections for memory.
3515 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
3516 switch (proj->_con) {
3517 case TypeFunc::Control:
3518 case TypeFunc::Memory:
3519 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3520 }
3521 ShouldNotReachHere();
3522 return nullptr;
3523 }
3524
3525 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3526 trailing->_kind = TrailingStore;
3527 leading->_kind = LeadingStore;
3528 #ifdef ASSERT
3529 trailing->_pair_idx = leading->_idx;
3530 leading->_pair_idx = leading->_idx;
3531 #endif
3532 }
3533
3534 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3535 trailing->_kind = TrailingLoadStore;
3782 return (req() > RawStores);
3783 }
3784
3785 void InitializeNode::set_complete(PhaseGVN* phase) {
3786 assert(!is_complete(), "caller responsibility");
3787 _is_complete = Complete;
3788
3789 // After this node is complete, it contains a bunch of
3790 // raw-memory initializations. There is no need for
3791 // it to have anything to do with non-raw memory effects.
3792 // Therefore, tell all non-raw users to re-optimize themselves,
3793 // after skipping the memory effects of this initialization.
3794 PhaseIterGVN* igvn = phase->is_IterGVN();
3795 if (igvn) igvn->add_users_to_worklist(this);
3796 }
3797
3798 // convenience function
3799 // return false if the init contains any stores already
3800 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3801 InitializeNode* init = initialization();
3802 if (init == nullptr || init->is_complete()) {
3803 return false;
3804 }
3805 init->remove_extra_zeroes();
3806 // for now, if this allocation has already collected any inits, bail:
3807 if (init->is_non_zero()) return false;
3808 init->set_complete(phase);
3809 return true;
3810 }
3811
3812 void InitializeNode::remove_extra_zeroes() {
3813 if (req() == RawStores) return;
3814 Node* zmem = zero_memory();
3815 uint fill = RawStores;
3816 for (uint i = fill; i < req(); i++) {
3817 Node* n = in(i);
3818 if (n->is_top() || n == zmem) continue; // skip
3819 if (fill < i) set_req(fill, n); // compact
3820 ++fill;
3821 }
3822 // delete any empty spaces created:
3823 while (fill < req()) {
3824 del_req(fill);
3962 // store node that we'd like to capture. We need to check
3963 // the uses of the MergeMemNode.
3964 mems.push(n);
3965 }
3966 } else if (n->is_Mem()) {
3967 Node* other_adr = n->in(MemNode::Address);
3968 if (other_adr == adr) {
3969 failed = true;
3970 break;
3971 } else {
3972 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3973 if (other_t_adr != nullptr) {
3974 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3975 if (other_alias_idx == alias_idx) {
3976 // A load from the same memory slice as the store right
3977 // after the InitializeNode. We check the control of the
3978 // object/array that is loaded from. If it's the same as
3979 // the store control then we cannot capture the store.
3980 assert(!n->is_Store(), "2 stores to same slice on same control?");
3981 Node* base = other_adr;
3982 if (base->is_Phi()) {
3983 // In rare case, base may be a PhiNode and it may read
3984 // the same memory slice between InitializeNode and store.
3985 failed = true;
3986 break;
3987 }
3988 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3989 base = base->in(AddPNode::Base);
3990 if (base != nullptr) {
3991 base = base->uncast();
3992 if (base->is_Proj() && base->in(0) == alloc) {
3993 failed = true;
3994 break;
3995 }
3996 }
3997 }
3998 }
3999 }
4000 } else {
4001 failed = true;
4002 break;
4003 }
4004 }
4005 }
4006 }
4007 if (failed) {
4554 // z's_done 12 16 16 16 12 16 12
4555 // z's_needed 12 16 16 16 16 16 16
4556 // zsize 0 0 0 0 4 0 4
4557 if (next_full_store < 0) {
4558 // Conservative tack: Zero to end of current word.
4559 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4560 } else {
4561 // Zero to beginning of next fully initialized word.
4562 // Or, don't zero at all, if we are already in that word.
4563 assert(next_full_store >= zeroes_needed, "must go forward");
4564 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4565 zeroes_needed = next_full_store;
4566 }
4567 }
4568
4569 if (zeroes_needed > zeroes_done) {
4570 intptr_t zsize = zeroes_needed - zeroes_done;
4571 // Do some incremental zeroing on rawmem, in parallel with inits.
4572 zeroes_done = align_down(zeroes_done, BytesPerInt);
4573 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4574 allocation()->in(AllocateNode::DefaultValue),
4575 allocation()->in(AllocateNode::RawDefaultValue),
4576 zeroes_done, zeroes_needed,
4577 phase);
4578 zeroes_done = zeroes_needed;
4579 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4580 do_zeroing = false; // leave the hole, next time
4581 }
4582 }
4583
4584 // Collect the store and move on:
4585 phase->replace_input_of(st, MemNode::Memory, inits);
4586 inits = st; // put it on the linearized chain
4587 set_req(i, zmem); // unhook from previous position
4588
4589 if (zeroes_done == st_off)
4590 zeroes_done = next_init_off;
4591
4592 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4593
4594 #ifdef ASSERT
4595 // Various order invariants. Weaker than stores_are_sane because
4615 remove_extra_zeroes(); // clear out all the zmems left over
4616 add_req(inits);
4617
4618 if (!(UseTLAB && ZeroTLAB)) {
4619 // If anything remains to be zeroed, zero it all now.
4620 zeroes_done = align_down(zeroes_done, BytesPerInt);
4621 // if it is the last unused 4 bytes of an instance, forget about it
4622 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4623 if (zeroes_done + BytesPerLong >= size_limit) {
4624 AllocateNode* alloc = allocation();
4625 assert(alloc != nullptr, "must be present");
4626 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
4627 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4628 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4629 if (zeroes_done == k->layout_helper())
4630 zeroes_done = size_limit;
4631 }
4632 }
4633 if (zeroes_done < size_limit) {
4634 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4635 allocation()->in(AllocateNode::DefaultValue),
4636 allocation()->in(AllocateNode::RawDefaultValue),
4637 zeroes_done, size_in_bytes, phase);
4638 }
4639 }
4640
4641 set_complete(phase);
4642 return rawmem;
4643 }
4644
4645
4646 #ifdef ASSERT
4647 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
4648 if (is_complete())
4649 return true; // stores could be anything at this point
4650 assert(allocation() != nullptr, "must be present");
4651 intptr_t last_off = allocation()->minimum_header_size();
4652 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4653 Node* st = in(i);
4654 intptr_t st_off = get_store_offset(st, phase);
4655 if (st_off < 0) continue; // ignore dead garbage
4656 if (last_off > st_off) {
|