6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/regalloc.hpp"
38 #include "opto/compile.hpp"
39 #include "opto/connode.hpp"
40 #include "opto/convertnode.hpp"
41 #include "opto/loopnode.hpp"
42 #include "opto/machnode.hpp"
43 #include "opto/matcher.hpp"
44 #include "opto/memnode.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/phaseX.hpp"
48 #include "opto/regmask.hpp"
49 #include "opto/rootnode.hpp"
50 #include "opto/vectornode.hpp"
51 #include "utilities/align.hpp"
52 #include "utilities/copy.hpp"
53 #include "utilities/macros.hpp"
54 #include "utilities/powerOfTwo.hpp"
55 #include "utilities/vmError.hpp"
56
57 // Portions of code courtesy of Clifford Click
58
59 // Optimization - Graph Style
60
214 bool is_instance = t_oop->is_known_instance_field();
215 PhaseIterGVN *igvn = phase->is_IterGVN();
216 if (is_instance && igvn != nullptr && result->is_Phi()) {
217 PhiNode *mphi = result->as_Phi();
218 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
219 const TypePtr *t = mphi->adr_type();
220 bool do_split = false;
221 // In the following cases, Load memory input can be further optimized based on
222 // its precise address type
223 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
224 do_split = true;
225 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
226 const TypeOopPtr* mem_t =
227 t->is_oopptr()->cast_to_exactness(true)
228 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
229 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
230 if (t_oop->is_aryptr()) {
231 mem_t = mem_t->is_aryptr()
232 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
233 ->cast_to_size(t_oop->is_aryptr()->size())
234 ->with_offset(t_oop->is_aryptr()->offset())
235 ->is_aryptr();
236 }
237 do_split = mem_t == t_oop;
238 }
239 if (do_split) {
240 // clone the Phi with our address type
241 result = mphi->split_out_instance(t_adr, igvn);
242 } else {
243 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
244 }
245 }
246 return result;
247 }
248
249 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
250 uint alias_idx = phase->C->get_alias_index(tp);
251 Node *mem = mmem;
252 #ifdef ASSERT
253 {
254 // Check that current type is consistent with the alias index used during graph construction
255 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
256 bool consistent = adr_check == nullptr || adr_check->empty() ||
257 phase->C->must_alias(adr_check, alias_idx );
258 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
259 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
260 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
261 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
262 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
263 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
264 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
265 // don't assert if it is dead code.
266 consistent = true;
267 }
268 if( !consistent ) {
269 st->print("alias_idx==%d, adr_check==", alias_idx);
270 if( adr_check == nullptr ) {
271 st->print("null");
272 } else {
273 adr_check->dump();
274 }
275 st->cr();
276 print_alias_types();
277 assert(consistent, "adr_check must match alias idx");
278 }
279 }
280 #endif
973 Node* ld = gvn.transform(load);
974 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
975 }
976
977 return load;
978 }
979
980 //------------------------------hash-------------------------------------------
981 uint LoadNode::hash() const {
982 // unroll addition of interesting fields
983 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
984 }
985
986 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
987 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
988 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
989 bool is_stable_ary = FoldStableValues &&
990 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
991 tp->isa_aryptr()->is_stable();
992
993 return (eliminate_boxing && non_volatile) || is_stable_ary;
994 }
995
996 return false;
997 }
998
999 LoadNode* LoadNode::pin_array_access_node() const {
1000 const TypePtr* adr_type = this->adr_type();
1001 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1002 return clone_pinned();
1003 }
1004 return nullptr;
1005 }
1006
1007 // Is the value loaded previously stored by an arraycopy? If so return
1008 // a load node that reads from the source array so we may be able to
1009 // optimize out the ArrayCopy node later.
1010 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1011 Node* ld_adr = in(MemNode::Address);
1012 intptr_t ld_off = 0;
1013 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1030 assert(ld_alloc != nullptr, "need an alloc");
1031 assert(addp->is_AddP(), "address must be addp");
1032 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1033 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1034 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1035 addp->set_req(AddPNode::Base, src);
1036 addp->set_req(AddPNode::Address, src);
1037 } else {
1038 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1039 ac->as_ArrayCopy()->is_copyof_validated() ||
1040 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1041 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1042 addp->set_req(AddPNode::Base, src);
1043 addp->set_req(AddPNode::Address, src);
1044
1045 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1046 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1047 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1048
1049 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1050 uint shift = exact_log2(type2aelembytes(ary_elem));
1051
1052 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1053 #ifdef _LP64
1054 diff = phase->transform(new ConvI2LNode(diff));
1055 #endif
1056 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1057
1058 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1059 addp->set_req(AddPNode::Offset, offset);
1060 }
1061 addp = phase->transform(addp);
1062 #ifdef ASSERT
1063 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1064 ld->_adr_type = adr_type;
1065 #endif
1066 ld->set_req(MemNode::Address, addp);
1067 ld->set_req(0, ctl);
1068 ld->set_req(MemNode::Memory, mem);
1069 return ld;
1070 }
1149 // Same base, same offset.
1150 // Possible improvement for arrays: check index value instead of absolute offset.
1151
1152 // At this point we have proven something like this setup:
1153 // B = << base >>
1154 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1155 // S = StoreQ(AddP( B , #Off), V)
1156 // (Actually, we haven't yet proven the Q's are the same.)
1157 // In other words, we are loading from a casted version of
1158 // the same pointer-and-offset that we stored to.
1159 // Casted version may carry a dependency and it is respected.
1160 // Thus, we are able to replace L by V.
1161 }
1162 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1163 if (store_Opcode() != st->Opcode()) {
1164 return nullptr;
1165 }
1166 // LoadVector/StoreVector needs additional check to ensure the types match.
1167 if (st->is_StoreVector()) {
1168 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1169 const TypeVect* out_vt = as_LoadVector()->vect_type();
1170 if (in_vt != out_vt) {
1171 return nullptr;
1172 }
1173 }
1174 return st->in(MemNode::ValueIn);
1175 }
1176
1177 // A load from a freshly-created object always returns zero.
1178 // (This can happen after LoadNode::Ideal resets the load's memory input
1179 // to find_captured_store, which returned InitializeNode::zero_memory.)
1180 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1181 (st->in(0) == ld_alloc) &&
1182 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1183 // return a zero value for the load's basic type
1184 // (This is one of the few places where a generic PhaseTransform
1185 // can create new nodes. Think of it as lazily manifesting
1186 // virtually pre-existing constants.)
1187 if (memory_type() != T_VOID) {
1188 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1189 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1190 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1191 // by the ArrayCopyNode.
1192 return phase->zerocon(memory_type());
1193 }
1194 } else {
1195 // TODO: materialize all-zero vector constant
1196 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1197 }
1198 }
1199
1200 // A load from an initialization barrier can match a captured store.
1201 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1202 InitializeNode* init = st->in(0)->as_Initialize();
1203 AllocateNode* alloc = init->allocation();
1204 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1205 // examine a captured store value
1206 st = init->find_captured_store(ld_off, memory_size(), phase);
1234 //----------------------is_instance_field_load_with_local_phi------------------
1235 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1236 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1237 in(Address)->is_AddP() ) {
1238 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1239 // Only instances and boxed values.
1240 if( t_oop != nullptr &&
1241 (t_oop->is_ptr_to_boxed_value() ||
1242 t_oop->is_known_instance_field()) &&
1243 t_oop->offset() != Type::OffsetBot &&
1244 t_oop->offset() != Type::OffsetTop) {
1245 return true;
1246 }
1247 }
1248 return false;
1249 }
1250
1251 //------------------------------Identity---------------------------------------
1252 // Loads are identity if previous store is to same address
1253 Node* LoadNode::Identity(PhaseGVN* phase) {
1254 // If the previous store-maker is the right kind of Store, and the store is
1255 // to the same address, then we are equal to the value stored.
1256 Node* mem = in(Memory);
1257 Node* value = can_see_stored_value(mem, phase);
1258 if( value ) {
1259 // byte, short & char stores truncate naturally.
1260 // A load has to load the truncated value which requires
1261 // some sort of masking operation and that requires an
1262 // Ideal call instead of an Identity call.
1263 if (memory_size() < BytesPerInt) {
1264 // If the input to the store does not fit with the load's result type,
1265 // it must be truncated via an Ideal call.
1266 if (!phase->type(value)->higher_equal(phase->type(this)))
1267 return this;
1268 }
1269 // (This works even when value is a Con, but LoadNode::Value
1270 // usually runs first, producing the singleton type of the Con.)
1271 if (!has_pinned_control_dependency() || value->is_Con()) {
1272 return value;
1273 } else {
1992 }
1993 }
1994
1995 // Don't do this for integer types. There is only potential profit if
1996 // the element type t is lower than _type; that is, for int types, if _type is
1997 // more restrictive than t. This only happens here if one is short and the other
1998 // char (both 16 bits), and in those cases we've made an intentional decision
1999 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2000 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2001 //
2002 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2003 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2004 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2005 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2006 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2007 // In fact, that could have been the original type of p1, and p1 could have
2008 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2009 // expression (LShiftL quux 3) independently optimized to the constant 8.
2010 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2011 && (_type->isa_vect() == nullptr)
2012 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2013 // t might actually be lower than _type, if _type is a unique
2014 // concrete subclass of abstract class t.
2015 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2016 const Type* jt = t->join_speculative(_type);
2017 // In any case, do not allow the join, per se, to empty out the type.
2018 if (jt->empty() && !t->empty()) {
2019 // This can happen if a interface-typed array narrows to a class type.
2020 jt = _type;
2021 }
2022 #ifdef ASSERT
2023 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2024 // The pointers in the autobox arrays are always non-null
2025 Node* base = adr->in(AddPNode::Base);
2026 if ((base != nullptr) && base->is_DecodeN()) {
2027 // Get LoadN node which loads IntegerCache.cache field
2028 base = base->in(1);
2029 }
2030 if ((base != nullptr) && base->is_Con()) {
2031 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2032 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2033 // It could be narrow oop
2034 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2035 }
2036 }
2037 }
2038 #endif
2039 return jt;
2040 }
2041 }
2042 } else if (tp->base() == Type::InstPtr) {
2043 assert( off != Type::OffsetBot ||
2044 // arrays can be cast to Objects
2045 !tp->isa_instptr() ||
2046 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2047 // unsafe field access may not have a constant offset
2048 C->has_unsafe_access(),
2049 "Field accesses must be precise" );
2050 // For oop loads, we expect the _type to be precise.
2051
2052 // Optimize loads from constant fields.
2053 const TypeInstPtr* tinst = tp->is_instptr();
2054 ciObject* const_oop = tinst->const_oop();
2055 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2056 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
2057 if (con_type != nullptr) {
2058 return con_type;
2059 }
2060 }
2061 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2062 assert(off != Type::OffsetBot ||
2063 !tp->isa_instklassptr() ||
2064 // arrays can be cast to Objects
2065 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2066 // also allow array-loading from the primary supertype
2067 // array during subtype checks
2068 Opcode() == Op_LoadKlass,
2069 "Field accesses must be precise");
2070 // For klass/static loads, we expect the _type to be precise
2071 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2072 /* With mirrors being an indirect in the Klass*
2073 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2074 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2075 *
2076 * So check the type and klass of the node before the LoadP.
2077 */
2078 Node* adr2 = adr->in(MemNode::Address);
2079 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2080 if (tkls != nullptr && !StressReflectiveCode) {
2081 if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2082 ciKlass* klass = tkls->exact_klass();
2083 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2084 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2085 return TypeInstPtr::make(klass->java_mirror());
2086 }
2087 }
2088 }
2089
2090 const TypeKlassPtr *tkls = tp->isa_klassptr();
2091 if (tkls != nullptr) {
2092 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2093 ciKlass* klass = tkls->exact_klass();
2094 // We are loading a field from a Klass metaobject whose identity
2095 // is known at compile time (the type is "exact" or "precise").
2096 // Check for fields we know are maintained as constants by the VM.
2097 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2098 // The field is Klass::_super_check_offset. Return its (constant) value.
2099 // (Folds up type checking code.)
2100 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2101 return TypeInt::make(klass->super_check_offset());
2102 }
2103 // Compute index into primary_supers array
2104 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2105 // Check for overflowing; use unsigned compare to handle the negative case.
2171 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2172 Node* value = can_see_stored_value(mem,phase);
2173 if (value != nullptr && value->is_Con()) {
2174 assert(value->bottom_type()->higher_equal(_type),"sanity");
2175 return value->bottom_type();
2176 }
2177 }
2178
2179 bool is_vect = (_type->isa_vect() != nullptr);
2180 if (is_instance && !is_vect) {
2181 // If we have an instance type and our memory input is the
2182 // programs's initial memory state, there is no matching store,
2183 // so just return a zero of the appropriate type -
2184 // except if it is vectorized - then we have no zero constant.
2185 Node *mem = in(MemNode::Memory);
2186 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2187 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2188 return Type::get_zero_type(_type->basic_type());
2189 }
2190 }
2191
2192 Node* alloc = is_new_object_mark_load();
2193 if (alloc != nullptr) {
2194 return TypeX::make(markWord::prototype().value());
2195 }
2196
2197 return _type;
2198 }
2199
2200 //------------------------------match_edge-------------------------------------
2201 // Do we Match on this edge index or not? Match only the address.
2202 uint LoadNode::match_edge(uint idx) const {
2203 return idx == MemNode::Address;
2204 }
2205
2206 //--------------------------LoadBNode::Ideal--------------------------------------
2207 //
2208 // If the previous store is to the same address as this load,
2209 // and the value stored was larger than a byte, replace this load
2210 // with the value stored truncated to a byte. If no truncation is
2211 // needed, the replacement is done in LoadNode::Identity().
2212 //
2213 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2214 Node* mem = in(MemNode::Memory);
2325 return LoadNode::Ideal(phase, can_reshape);
2326 }
2327
2328 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2329 Node* mem = in(MemNode::Memory);
2330 Node* value = can_see_stored_value(mem,phase);
2331 if (value != nullptr && value->is_Con() &&
2332 !value->bottom_type()->higher_equal(_type)) {
2333 // If the input to the store does not fit with the load's result type,
2334 // it must be truncated. We can't delay until Ideal call since
2335 // a singleton Value is needed for split_thru_phi optimization.
2336 int con = value->get_int();
2337 return TypeInt::make((con << 16) >> 16);
2338 }
2339 return LoadNode::Value(phase);
2340 }
2341
2342 //=============================================================================
2343 //----------------------------LoadKlassNode::make------------------------------
2344 // Polymorphic factory method:
2345 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2346 // sanity check the alias category against the created node type
2347 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2348 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2349 #ifdef _LP64
2350 if (adr_type->is_ptr_to_narrowklass()) {
2351 assert(UseCompressedClassPointers, "no compressed klasses");
2352 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2353 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2354 }
2355 #endif
2356 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2357 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2358 }
2359
2360 //------------------------------Value------------------------------------------
2361 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2362 return klass_value_common(phase);
2363 }
2364
2365 // In most cases, LoadKlassNode does not have the control input set. If the control
2372 // Either input is TOP ==> the result is TOP
2373 const Type *t1 = phase->type( in(MemNode::Memory) );
2374 if (t1 == Type::TOP) return Type::TOP;
2375 Node *adr = in(MemNode::Address);
2376 const Type *t2 = phase->type( adr );
2377 if (t2 == Type::TOP) return Type::TOP;
2378 const TypePtr *tp = t2->is_ptr();
2379 if (TypePtr::above_centerline(tp->ptr()) ||
2380 tp->ptr() == TypePtr::Null) return Type::TOP;
2381
2382 // Return a more precise klass, if possible
2383 const TypeInstPtr *tinst = tp->isa_instptr();
2384 if (tinst != nullptr) {
2385 ciInstanceKlass* ik = tinst->instance_klass();
2386 int offset = tinst->offset();
2387 if (ik == phase->C->env()->Class_klass()
2388 && (offset == java_lang_Class::klass_offset() ||
2389 offset == java_lang_Class::array_klass_offset())) {
2390 // We are loading a special hidden field from a Class mirror object,
2391 // the field which points to the VM's Klass metaobject.
2392 ciType* t = tinst->java_mirror_type();
2393 // java_mirror_type returns non-null for compile-time Class constants.
2394 if (t != nullptr) {
2395 // constant oop => constant klass
2396 if (offset == java_lang_Class::array_klass_offset()) {
2397 if (t->is_void()) {
2398 // We cannot create a void array. Since void is a primitive type return null
2399 // klass. Users of this result need to do a null check on the returned klass.
2400 return TypePtr::NULL_PTR;
2401 }
2402 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2403 }
2404 if (!t->is_klass()) {
2405 // a primitive Class (e.g., int.class) has null for a klass field
2406 return TypePtr::NULL_PTR;
2407 }
2408 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2409 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2410 }
2411 // non-constant mirror, so we can't tell what's going on
2412 }
2413 if (!tinst->is_loaded())
2414 return _type; // Bail out if not loaded
2415 if (offset == oopDesc::klass_offset_in_bytes()) {
2416 return tinst->as_klass_type(true);
2417 }
2418 }
2419
2420 // Check for loading klass from an array
2421 const TypeAryPtr *tary = tp->isa_aryptr();
2422 if (tary != nullptr &&
2423 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2424 return tary->as_klass_type(true);
2425 }
2426
2427 // Check for loading klass from an array klass
2428 const TypeKlassPtr *tkls = tp->isa_klassptr();
2429 if (tkls != nullptr && !StressReflectiveCode) {
2430 if (!tkls->is_loaded())
2431 return _type; // Bail out if not loaded
2432 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2433 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2434 // // Always returning precise element type is incorrect,
2435 // // e.g., element type could be object and array may contain strings
2436 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2437
2438 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2439 // according to the element type's subclassing.
2440 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2441 }
2683
2684 // Since they are not commoned, do not hash them:
2685 return NO_HASH;
2686 }
2687
2688 //------------------------------Ideal------------------------------------------
2689 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2690 // When a store immediately follows a relevant allocation/initialization,
2691 // try to capture it into the initialization, or hoist it above.
2692 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2693 Node* p = MemNode::Ideal_common(phase, can_reshape);
2694 if (p) return (p == NodeSentinel) ? nullptr : p;
2695
2696 Node* mem = in(MemNode::Memory);
2697 Node* address = in(MemNode::Address);
2698 Node* value = in(MemNode::ValueIn);
2699 // Back-to-back stores to same address? Fold em up. Generally
2700 // unsafe if I have intervening uses... Also disallowed for StoreCM
2701 // since they must follow each StoreP operation. Redundant StoreCMs
2702 // are eliminated just before matching in final_graph_reshape.
2703 {
2704 Node* st = mem;
2705 // If Store 'st' has more than one use, we cannot fold 'st' away.
2706 // For example, 'st' might be the final state at a conditional
2707 // return. Or, 'st' might be used by some node which is live at
2708 // the same time 'st' is live, which might be unschedulable. So,
2709 // require exactly ONE user until such time as we clone 'mem' for
2710 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2711 // true).
2712 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2713 // Looking at a dead closed cycle of memory?
2714 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2715 assert(Opcode() == st->Opcode() ||
2716 st->Opcode() == Op_StoreVector ||
2717 Opcode() == Op_StoreVector ||
2718 st->Opcode() == Op_StoreVectorScatter ||
2719 Opcode() == Op_StoreVectorScatter ||
2720 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2721 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2722 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2723 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2724 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2725
2726 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2727 st->as_Store()->memory_size() <= this->memory_size()) {
2728 Node* use = st->raw_out(0);
2729 if (phase->is_IterGVN()) {
2730 phase->is_IterGVN()->rehash_node_delayed(use);
2731 }
2732 // It's OK to do this in the parser, since DU info is always accurate,
2733 // and the parser always refers to nodes via SafePointNode maps.
2734 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2735 return this;
2736 }
2737 st = st->in(MemNode::Memory);
2738 }
2739 }
2740
2741
2742 // Capture an unaliased, unconditional, simple store into an initializer.
2799 // Load then Store? Then the Store is useless
2800 if (val->is_Load() &&
2801 val->in(MemNode::Address)->eqv_uncast(adr) &&
2802 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2803 val->as_Load()->store_Opcode() == Opcode()) {
2804 result = mem;
2805 }
2806
2807 // Two stores in a row of the same value?
2808 if (result == this &&
2809 mem->is_Store() &&
2810 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2811 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2812 mem->Opcode() == Opcode()) {
2813 result = mem;
2814 }
2815
2816 // Store of zero anywhere into a freshly-allocated object?
2817 // Then the store is useless.
2818 // (It must already have been captured by the InitializeNode.)
2819 if (result == this &&
2820 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2821 // a newly allocated object is already all-zeroes everywhere
2822 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
2823 result = mem;
2824 }
2825
2826 if (result == this) {
2827 // the store may also apply to zero-bits in an earlier object
2828 Node* prev_mem = find_previous_store(phase);
2829 // Steps (a), (b): Walk past independent stores to find an exact match.
2830 if (prev_mem != nullptr) {
2831 Node* prev_val = can_see_stored_value(prev_mem, phase);
2832 if (prev_val != nullptr && prev_val == val) {
2833 // prev_val and val might differ by a cast; it would be good
2834 // to keep the more informative of the two.
2835 result = mem;
2836 }
2837 }
2838 }
2839 }
2840
2841 PhaseIterGVN* igvn = phase->is_IterGVN();
2842 if (result != this && igvn != nullptr) {
2843 MemBarNode* trailing = trailing_membar();
2844 if (trailing != nullptr) {
2845 #ifdef ASSERT
2846 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2991 Node* StoreCMNode::Identity(PhaseGVN* phase) {
2992 // No need to card mark when storing a null ptr
2993 Node* my_store = in(MemNode::OopStore);
2994 if (my_store->is_Store()) {
2995 const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
2996 if( t1 == TypePtr::NULL_PTR ) {
2997 return in(MemNode::Memory);
2998 }
2999 }
3000 return this;
3001 }
3002
3003 //=============================================================================
3004 //------------------------------Ideal---------------------------------------
3005 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
3006 Node* progress = StoreNode::Ideal(phase, can_reshape);
3007 if (progress != nullptr) return progress;
3008
3009 Node* my_store = in(MemNode::OopStore);
3010 if (my_store->is_MergeMem()) {
3011 Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
3012 set_req_X(MemNode::OopStore, mem, phase);
3013 return this;
3014 }
3015
3016 return nullptr;
3017 }
3018
3019 //------------------------------Value-----------------------------------------
3020 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
3021 // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
3022 // If extra input is TOP ==> the result is TOP
3023 const Type* t = phase->type(in(MemNode::OopStore));
3024 if (t == Type::TOP) {
3025 return Type::TOP;
3026 }
3027 return StoreNode::Value(phase);
3028 }
3029
3030
3031 //=============================================================================
3032 //----------------------------------SCMemProjNode------------------------------
3033 const Type* SCMemProjNode::Value(PhaseGVN* phase) const
3164 // Clearing a short array is faster with stores
3165 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3166 // Already know this is a large node, do not try to ideal it
3167 if (_is_large) return nullptr;
3168
3169 const int unit = BytesPerLong;
3170 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3171 if (!t) return nullptr;
3172 if (!t->is_con()) return nullptr;
3173 intptr_t raw_count = t->get_con();
3174 intptr_t size = raw_count;
3175 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3176 // Clearing nothing uses the Identity call.
3177 // Negative clears are possible on dead ClearArrays
3178 // (see jck test stmt114.stmt11402.val).
3179 if (size <= 0 || size % unit != 0) return nullptr;
3180 intptr_t count = size / unit;
3181 // Length too long; communicate this to matchers and assemblers.
3182 // Assemblers are responsible to produce fast hardware clears for it.
3183 if (size > InitArrayShortSize) {
3184 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3185 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3186 return nullptr;
3187 }
3188 if (!IdealizeClearArrayNode) return nullptr;
3189 Node *mem = in(1);
3190 if( phase->type(mem)==Type::TOP ) return nullptr;
3191 Node *adr = in(3);
3192 const Type* at = phase->type(adr);
3193 if( at==Type::TOP ) return nullptr;
3194 const TypePtr* atp = at->isa_ptr();
3195 // adjust atp to be the correct array element address type
3196 if (atp == nullptr) atp = TypePtr::BOTTOM;
3197 else atp = atp->add_offset(Type::OffsetBot);
3198 // Get base for derived pointer purposes
3199 if( adr->Opcode() != Op_AddP ) Unimplemented();
3200 Node *base = adr->in(1);
3201
3202 Node *zero = phase->makecon(TypeLong::ZERO);
3203 Node *off = phase->MakeConX(BytesPerLong);
3204 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3205 count--;
3206 while( count-- ) {
3207 mem = phase->transform(mem);
3208 adr = phase->transform(new AddPNode(base,adr,off));
3209 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3210 }
3211 return mem;
3212 }
3213
3214 //----------------------------step_through----------------------------------
3215 // Return allocation input memory edge if it is different instance
3216 // or itself if it is the one we are looking for.
3217 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3218 Node* n = *np;
3219 assert(n->is_ClearArray(), "sanity");
3220 intptr_t offset;
3221 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3222 // This method is called only before Allocate nodes are expanded
3223 // during macro nodes expansion. Before that ClearArray nodes are
3224 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3225 // Allocate nodes are expanded) which follows allocations.
3226 assert(alloc != nullptr, "should have allocation");
3227 if (alloc->_idx == instance_id) {
3228 // Can not bypass initialization of the instance we are looking for.
3229 return false;
3230 }
3231 // Otherwise skip it.
3232 InitializeNode* init = alloc->initialization();
3233 if (init != nullptr)
3234 *np = init->in(TypeFunc::Memory);
3235 else
3236 *np = alloc->in(TypeFunc::Memory);
3237 return true;
3238 }
3239
3240 //----------------------------clear_memory-------------------------------------
3241 // Generate code to initialize object storage to zero.
3242 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3243 intptr_t start_offset,
3244 Node* end_offset,
3245 PhaseGVN* phase) {
3246 intptr_t offset = start_offset;
3247
3248 int unit = BytesPerLong;
3249 if ((offset % unit) != 0) {
3250 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3251 adr = phase->transform(adr);
3252 const TypePtr* atp = TypeRawPtr::BOTTOM;
3253 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3254 mem = phase->transform(mem);
3255 offset += BytesPerInt;
3256 }
3257 assert((offset % unit) == 0, "");
3258
3259 // Initialize the remaining stuff, if any, with a ClearArray.
3260 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
3261 }
3262
3263 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3264 Node* start_offset,
3265 Node* end_offset,
3266 PhaseGVN* phase) {
3267 if (start_offset == end_offset) {
3268 // nothing to do
3269 return mem;
3270 }
3271
3272 int unit = BytesPerLong;
3273 Node* zbase = start_offset;
3274 Node* zend = end_offset;
3275
3276 // Scale to the unit required by the CPU:
3277 if (!Matcher::init_array_count_is_in_bytes) {
3278 Node* shift = phase->intcon(exact_log2(unit));
3279 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3280 zend = phase->transform(new URShiftXNode(zend, shift) );
3281 }
3282
3283 // Bulk clear double-words
3284 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3285 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3286 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
3287 return phase->transform(mem);
3288 }
3289
3290 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3291 intptr_t start_offset,
3292 intptr_t end_offset,
3293 PhaseGVN* phase) {
3294 if (start_offset == end_offset) {
3295 // nothing to do
3296 return mem;
3297 }
3298
3299 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3300 intptr_t done_offset = end_offset;
3301 if ((done_offset % BytesPerLong) != 0) {
3302 done_offset -= BytesPerInt;
3303 }
3304 if (done_offset > start_offset) {
3305 mem = clear_memory(ctl, mem, dest,
3306 start_offset, phase->MakeConX(done_offset), phase);
3307 }
3308 if (done_offset < end_offset) { // emit the final 32-bit store
3309 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3310 adr = phase->transform(adr);
3311 const TypePtr* atp = TypeRawPtr::BOTTOM;
3312 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3313 mem = phase->transform(mem);
3314 done_offset += BytesPerInt;
3315 }
3316 assert(done_offset == end_offset, "");
3317 return mem;
3318 }
3319
3320 //=============================================================================
3321 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3322 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3323 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3324 #ifdef ASSERT
3325 , _pair_idx(0)
3326 #endif
3327 {
3328 init_class_id(Class_MemBar);
3329 Node* top = C->top();
3330 init_req(TypeFunc::I_O,top);
3331 init_req(TypeFunc::FramePtr,top);
3332 init_req(TypeFunc::ReturnAdr,top);
3437 PhaseIterGVN* igvn = phase->is_IterGVN();
3438 remove(igvn);
3439 // Must return either the original node (now dead) or a new node
3440 // (Do not return a top here, since that would break the uniqueness of top.)
3441 return new ConINode(TypeInt::ZERO);
3442 }
3443 }
3444 return progress ? this : nullptr;
3445 }
3446
3447 //------------------------------Value------------------------------------------
3448 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3449 if( !in(0) ) return Type::TOP;
3450 if( phase->type(in(0)) == Type::TOP )
3451 return Type::TOP;
3452 return TypeTuple::MEMBAR;
3453 }
3454
3455 //------------------------------match------------------------------------------
3456 // Construct projections for memory.
3457 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3458 switch (proj->_con) {
3459 case TypeFunc::Control:
3460 case TypeFunc::Memory:
3461 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3462 }
3463 ShouldNotReachHere();
3464 return nullptr;
3465 }
3466
3467 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3468 trailing->_kind = TrailingStore;
3469 leading->_kind = LeadingStore;
3470 #ifdef ASSERT
3471 trailing->_pair_idx = leading->_idx;
3472 leading->_pair_idx = leading->_idx;
3473 #endif
3474 }
3475
3476 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3477 trailing->_kind = TrailingLoadStore;
3724 return (req() > RawStores);
3725 }
3726
3727 void InitializeNode::set_complete(PhaseGVN* phase) {
3728 assert(!is_complete(), "caller responsibility");
3729 _is_complete = Complete;
3730
3731 // After this node is complete, it contains a bunch of
3732 // raw-memory initializations. There is no need for
3733 // it to have anything to do with non-raw memory effects.
3734 // Therefore, tell all non-raw users to re-optimize themselves,
3735 // after skipping the memory effects of this initialization.
3736 PhaseIterGVN* igvn = phase->is_IterGVN();
3737 if (igvn) igvn->add_users_to_worklist(this);
3738 }
3739
3740 // convenience function
3741 // return false if the init contains any stores already
3742 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3743 InitializeNode* init = initialization();
3744 if (init == nullptr || init->is_complete()) return false;
3745 init->remove_extra_zeroes();
3746 // for now, if this allocation has already collected any inits, bail:
3747 if (init->is_non_zero()) return false;
3748 init->set_complete(phase);
3749 return true;
3750 }
3751
3752 void InitializeNode::remove_extra_zeroes() {
3753 if (req() == RawStores) return;
3754 Node* zmem = zero_memory();
3755 uint fill = RawStores;
3756 for (uint i = fill; i < req(); i++) {
3757 Node* n = in(i);
3758 if (n->is_top() || n == zmem) continue; // skip
3759 if (fill < i) set_req(fill, n); // compact
3760 ++fill;
3761 }
3762 // delete any empty spaces created:
3763 while (fill < req()) {
3764 del_req(fill);
3902 // store node that we'd like to capture. We need to check
3903 // the uses of the MergeMemNode.
3904 mems.push(n);
3905 }
3906 } else if (n->is_Mem()) {
3907 Node* other_adr = n->in(MemNode::Address);
3908 if (other_adr == adr) {
3909 failed = true;
3910 break;
3911 } else {
3912 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3913 if (other_t_adr != nullptr) {
3914 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3915 if (other_alias_idx == alias_idx) {
3916 // A load from the same memory slice as the store right
3917 // after the InitializeNode. We check the control of the
3918 // object/array that is loaded from. If it's the same as
3919 // the store control then we cannot capture the store.
3920 assert(!n->is_Store(), "2 stores to same slice on same control?");
3921 Node* base = other_adr;
3922 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3923 base = base->in(AddPNode::Base);
3924 if (base != nullptr) {
3925 base = base->uncast();
3926 if (base->is_Proj() && base->in(0) == alloc) {
3927 failed = true;
3928 break;
3929 }
3930 }
3931 }
3932 }
3933 }
3934 } else {
3935 failed = true;
3936 break;
3937 }
3938 }
3939 }
3940 }
3941 if (failed) {
4488 // z's_done 12 16 16 16 12 16 12
4489 // z's_needed 12 16 16 16 16 16 16
4490 // zsize 0 0 0 0 4 0 4
4491 if (next_full_store < 0) {
4492 // Conservative tack: Zero to end of current word.
4493 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4494 } else {
4495 // Zero to beginning of next fully initialized word.
4496 // Or, don't zero at all, if we are already in that word.
4497 assert(next_full_store >= zeroes_needed, "must go forward");
4498 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4499 zeroes_needed = next_full_store;
4500 }
4501 }
4502
4503 if (zeroes_needed > zeroes_done) {
4504 intptr_t zsize = zeroes_needed - zeroes_done;
4505 // Do some incremental zeroing on rawmem, in parallel with inits.
4506 zeroes_done = align_down(zeroes_done, BytesPerInt);
4507 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4508 zeroes_done, zeroes_needed,
4509 phase);
4510 zeroes_done = zeroes_needed;
4511 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4512 do_zeroing = false; // leave the hole, next time
4513 }
4514 }
4515
4516 // Collect the store and move on:
4517 phase->replace_input_of(st, MemNode::Memory, inits);
4518 inits = st; // put it on the linearized chain
4519 set_req(i, zmem); // unhook from previous position
4520
4521 if (zeroes_done == st_off)
4522 zeroes_done = next_init_off;
4523
4524 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4525
4526 #ifdef ASSERT
4527 // Various order invariants. Weaker than stores_are_sane because
4547 remove_extra_zeroes(); // clear out all the zmems left over
4548 add_req(inits);
4549
4550 if (!(UseTLAB && ZeroTLAB)) {
4551 // If anything remains to be zeroed, zero it all now.
4552 zeroes_done = align_down(zeroes_done, BytesPerInt);
4553 // if it is the last unused 4 bytes of an instance, forget about it
4554 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4555 if (zeroes_done + BytesPerLong >= size_limit) {
4556 AllocateNode* alloc = allocation();
4557 assert(alloc != nullptr, "must be present");
4558 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
4559 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4560 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4561 if (zeroes_done == k->layout_helper())
4562 zeroes_done = size_limit;
4563 }
4564 }
4565 if (zeroes_done < size_limit) {
4566 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4567 zeroes_done, size_in_bytes, phase);
4568 }
4569 }
4570
4571 set_complete(phase);
4572 return rawmem;
4573 }
4574
4575
4576 #ifdef ASSERT
4577 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
4578 if (is_complete())
4579 return true; // stores could be anything at this point
4580 assert(allocation() != nullptr, "must be present");
4581 intptr_t last_off = allocation()->minimum_header_size();
4582 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4583 Node* st = in(i);
4584 intptr_t st_off = get_store_offset(st, phase);
4585 if (st_off < 0) continue; // ignore dead garbage
4586 if (last_off > st_off) {
|
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "compiler/compileLog.hpp"
30 #include "gc/shared/barrierSet.hpp"
31 #include "gc/shared/c2/barrierSetC2.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "memory/allocation.inline.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "oops/objArrayKlass.hpp"
36 #include "opto/addnode.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/cfgnode.hpp"
39 #include "opto/regalloc.hpp"
40 #include "opto/compile.hpp"
41 #include "opto/connode.hpp"
42 #include "opto/convertnode.hpp"
43 #include "opto/inlinetypenode.hpp"
44 #include "opto/loopnode.hpp"
45 #include "opto/machnode.hpp"
46 #include "opto/matcher.hpp"
47 #include "opto/memnode.hpp"
48 #include "opto/mulnode.hpp"
49 #include "opto/narrowptrnode.hpp"
50 #include "opto/phaseX.hpp"
51 #include "opto/regmask.hpp"
52 #include "opto/rootnode.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/macros.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #include "utilities/vmError.hpp"
59
60 // Portions of code courtesy of Clifford Click
61
62 // Optimization - Graph Style
63
217 bool is_instance = t_oop->is_known_instance_field();
218 PhaseIterGVN *igvn = phase->is_IterGVN();
219 if (is_instance && igvn != nullptr && result->is_Phi()) {
220 PhiNode *mphi = result->as_Phi();
221 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
222 const TypePtr *t = mphi->adr_type();
223 bool do_split = false;
224 // In the following cases, Load memory input can be further optimized based on
225 // its precise address type
226 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
227 do_split = true;
228 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
229 const TypeOopPtr* mem_t =
230 t->is_oopptr()->cast_to_exactness(true)
231 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
232 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
233 if (t_oop->is_aryptr()) {
234 mem_t = mem_t->is_aryptr()
235 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
236 ->cast_to_size(t_oop->is_aryptr()->size())
237 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
238 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
239 ->with_offset(t_oop->is_aryptr()->offset())
240 ->is_aryptr();
241 }
242 do_split = mem_t == t_oop;
243 }
244 if (do_split) {
245 // clone the Phi with our address type
246 result = mphi->split_out_instance(t_adr, igvn);
247 } else {
248 // TODO 8325106
249 // assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
250 }
251 }
252 return result;
253 }
254
255 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
256 uint alias_idx = phase->C->get_alias_index(tp);
257 Node *mem = mmem;
258 #ifdef ASSERT
259 {
260 // Check that current type is consistent with the alias index used during graph construction
261 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
262 bool consistent = adr_check == nullptr || adr_check->empty() ||
263 phase->C->must_alias(adr_check, alias_idx );
264 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
265 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
266 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
267 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
268 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
269 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
270 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
271 // don't assert if it is dead code.
272 consistent = true;
273 }
274 if( !consistent ) {
275 st->print("alias_idx==%d, adr_check==", alias_idx);
276 if( adr_check == nullptr ) {
277 st->print("null");
278 } else {
279 adr_check->dump();
280 }
281 st->cr();
282 print_alias_types();
283 assert(consistent, "adr_check must match alias idx");
284 }
285 }
286 #endif
979 Node* ld = gvn.transform(load);
980 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
981 }
982
983 return load;
984 }
985
986 //------------------------------hash-------------------------------------------
987 uint LoadNode::hash() const {
988 // unroll addition of interesting fields
989 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
990 }
991
992 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
993 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
994 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
995 bool is_stable_ary = FoldStableValues &&
996 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
997 tp->isa_aryptr()->is_stable();
998
999 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1000 }
1001
1002 return false;
1003 }
1004
1005 LoadNode* LoadNode::pin_array_access_node() const {
1006 const TypePtr* adr_type = this->adr_type();
1007 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1008 return clone_pinned();
1009 }
1010 return nullptr;
1011 }
1012
1013 // Is the value loaded previously stored by an arraycopy? If so return
1014 // a load node that reads from the source array so we may be able to
1015 // optimize out the ArrayCopy node later.
1016 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1017 Node* ld_adr = in(MemNode::Address);
1018 intptr_t ld_off = 0;
1019 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1036 assert(ld_alloc != nullptr, "need an alloc");
1037 assert(addp->is_AddP(), "address must be addp");
1038 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1039 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1040 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1041 addp->set_req(AddPNode::Base, src);
1042 addp->set_req(AddPNode::Address, src);
1043 } else {
1044 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1045 ac->as_ArrayCopy()->is_copyof_validated() ||
1046 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1047 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1048 addp->set_req(AddPNode::Base, src);
1049 addp->set_req(AddPNode::Address, src);
1050
1051 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1052 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1053 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1054
1055 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1056 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1057
1058 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1059 #ifdef _LP64
1060 diff = phase->transform(new ConvI2LNode(diff));
1061 #endif
1062 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1063
1064 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1065 addp->set_req(AddPNode::Offset, offset);
1066 }
1067 addp = phase->transform(addp);
1068 #ifdef ASSERT
1069 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1070 ld->_adr_type = adr_type;
1071 #endif
1072 ld->set_req(MemNode::Address, addp);
1073 ld->set_req(0, ctl);
1074 ld->set_req(MemNode::Memory, mem);
1075 return ld;
1076 }
1155 // Same base, same offset.
1156 // Possible improvement for arrays: check index value instead of absolute offset.
1157
1158 // At this point we have proven something like this setup:
1159 // B = << base >>
1160 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1161 // S = StoreQ(AddP( B , #Off), V)
1162 // (Actually, we haven't yet proven the Q's are the same.)
1163 // In other words, we are loading from a casted version of
1164 // the same pointer-and-offset that we stored to.
1165 // Casted version may carry a dependency and it is respected.
1166 // Thus, we are able to replace L by V.
1167 }
1168 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1169 if (store_Opcode() != st->Opcode()) {
1170 return nullptr;
1171 }
1172 // LoadVector/StoreVector needs additional check to ensure the types match.
1173 if (st->is_StoreVector()) {
1174 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1175 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1176 if (in_vt != out_vt) {
1177 return nullptr;
1178 }
1179 }
1180 return st->in(MemNode::ValueIn);
1181 }
1182
1183 // A load from a freshly-created object always returns zero.
1184 // (This can happen after LoadNode::Ideal resets the load's memory input
1185 // to find_captured_store, which returned InitializeNode::zero_memory.)
1186 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1187 (st->in(0) == ld_alloc) &&
1188 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1189 // return a zero value for the load's basic type
1190 // (This is one of the few places where a generic PhaseTransform
1191 // can create new nodes. Think of it as lazily manifesting
1192 // virtually pre-existing constants.)
1193 Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1194 if (default_value != nullptr) {
1195 return default_value;
1196 }
1197 assert(ld_alloc->in(AllocateNode::RawDefaultValue) == nullptr, "default value may not be null");
1198 if (memory_type() != T_VOID) {
1199 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1200 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1201 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1202 // by the ArrayCopyNode.
1203 return phase->zerocon(memory_type());
1204 }
1205 } else {
1206 // TODO: materialize all-zero vector constant
1207 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1208 }
1209 }
1210
1211 // A load from an initialization barrier can match a captured store.
1212 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1213 InitializeNode* init = st->in(0)->as_Initialize();
1214 AllocateNode* alloc = init->allocation();
1215 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1216 // examine a captured store value
1217 st = init->find_captured_store(ld_off, memory_size(), phase);
1245 //----------------------is_instance_field_load_with_local_phi------------------
1246 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1247 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1248 in(Address)->is_AddP() ) {
1249 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1250 // Only instances and boxed values.
1251 if( t_oop != nullptr &&
1252 (t_oop->is_ptr_to_boxed_value() ||
1253 t_oop->is_known_instance_field()) &&
1254 t_oop->offset() != Type::OffsetBot &&
1255 t_oop->offset() != Type::OffsetTop) {
1256 return true;
1257 }
1258 }
1259 return false;
1260 }
1261
1262 //------------------------------Identity---------------------------------------
1263 // Loads are identity if previous store is to same address
1264 Node* LoadNode::Identity(PhaseGVN* phase) {
1265 // Loading from an InlineType? The InlineType has the values of
1266 // all fields as input. Look for the field with matching offset.
1267 Node* addr = in(Address);
1268 intptr_t offset;
1269 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1270 if (base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1271 Node* value = base->as_InlineType()->field_value_by_offset((int)offset, true);
1272 if (value != nullptr) {
1273 if (Opcode() == Op_LoadN) {
1274 // Encode oop value if we are loading a narrow oop
1275 assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1276 value = phase->transform(new EncodePNode(value, bottom_type()));
1277 }
1278 return value;
1279 }
1280 }
1281
1282 // If the previous store-maker is the right kind of Store, and the store is
1283 // to the same address, then we are equal to the value stored.
1284 Node* mem = in(Memory);
1285 Node* value = can_see_stored_value(mem, phase);
1286 if( value ) {
1287 // byte, short & char stores truncate naturally.
1288 // A load has to load the truncated value which requires
1289 // some sort of masking operation and that requires an
1290 // Ideal call instead of an Identity call.
1291 if (memory_size() < BytesPerInt) {
1292 // If the input to the store does not fit with the load's result type,
1293 // it must be truncated via an Ideal call.
1294 if (!phase->type(value)->higher_equal(phase->type(this)))
1295 return this;
1296 }
1297 // (This works even when value is a Con, but LoadNode::Value
1298 // usually runs first, producing the singleton type of the Con.)
1299 if (!has_pinned_control_dependency() || value->is_Con()) {
1300 return value;
1301 } else {
2020 }
2021 }
2022
2023 // Don't do this for integer types. There is only potential profit if
2024 // the element type t is lower than _type; that is, for int types, if _type is
2025 // more restrictive than t. This only happens here if one is short and the other
2026 // char (both 16 bits), and in those cases we've made an intentional decision
2027 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2028 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2029 //
2030 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2031 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2032 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2033 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2034 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2035 // In fact, that could have been the original type of p1, and p1 could have
2036 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2037 // expression (LShiftL quux 3) independently optimized to the constant 8.
2038 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2039 && (_type->isa_vect() == nullptr)
2040 && !ary->is_flat()
2041 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2042 // t might actually be lower than _type, if _type is a unique
2043 // concrete subclass of abstract class t.
2044 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2045 const Type* jt = t->join_speculative(_type);
2046 // In any case, do not allow the join, per se, to empty out the type.
2047 if (jt->empty() && !t->empty()) {
2048 // This can happen if a interface-typed array narrows to a class type.
2049 jt = _type;
2050 }
2051 #ifdef ASSERT
2052 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2053 // The pointers in the autobox arrays are always non-null
2054 Node* base = adr->in(AddPNode::Base);
2055 if ((base != nullptr) && base->is_DecodeN()) {
2056 // Get LoadN node which loads IntegerCache.cache field
2057 base = base->in(1);
2058 }
2059 if ((base != nullptr) && base->is_Con()) {
2060 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2061 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2062 // It could be narrow oop
2063 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2064 }
2065 }
2066 }
2067 #endif
2068 return jt;
2069 }
2070 }
2071 } else if (tp->base() == Type::InstPtr) {
2072 assert( off != Type::OffsetBot ||
2073 // arrays can be cast to Objects
2074 !tp->isa_instptr() ||
2075 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2076 // Default value load
2077 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2078 // unsafe field access may not have a constant offset
2079 C->has_unsafe_access(),
2080 "Field accesses must be precise" );
2081 // For oop loads, we expect the _type to be precise.
2082
2083 const TypeInstPtr* tinst = tp->is_instptr();
2084 BasicType bt = memory_type();
2085
2086 // Optimize loads from constant fields.
2087 ciObject* const_oop = tinst->const_oop();
2088 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2089 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2090 if (con_type != nullptr) {
2091 return con_type;
2092 }
2093 }
2094 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2095 assert(off != Type::OffsetBot ||
2096 !tp->isa_instklassptr() ||
2097 // arrays can be cast to Objects
2098 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2099 // also allow array-loading from the primary supertype
2100 // array during subtype checks
2101 Opcode() == Op_LoadKlass,
2102 "Field accesses must be precise");
2103 // For klass/static loads, we expect the _type to be precise
2104 } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
2105 if (adr->is_Load() && off == 0) {
2106 /* With mirrors being an indirect in the Klass*
2107 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2108 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2109 *
2110 * So check the type and klass of the node before the LoadP.
2111 */
2112 Node* adr2 = adr->in(MemNode::Address);
2113 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2114 if (tkls != nullptr) {
2115 if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2116 ciKlass* klass = tkls->exact_klass();
2117 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2118 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2119 return TypeInstPtr::make(klass->java_mirror());
2120 }
2121 }
2122 } else {
2123 // Check for a load of the default value offset from the InlineKlassFixedBlock:
2124 // LoadI(LoadP(inline_klass, adr_inlineklass_fixed_block_offset), default_value_offset_offset)
2125 // TODO 8325106 remove?
2126 intptr_t offset = 0;
2127 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2128 if (base != nullptr && base->is_Load() && offset == in_bytes(InlineKlass::default_value_offset_offset())) {
2129 const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
2130 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact() && tkls->exact_klass()->is_inlinetype() &&
2131 tkls->offset() == in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())) {
2132 assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
2133 assert(Opcode() == Op_LoadI, "must load an int from fixed block");
2134 return TypeInt::make(tkls->exact_klass()->as_inline_klass()->default_value_offset());
2135 }
2136 }
2137 }
2138 }
2139
2140 const TypeKlassPtr *tkls = tp->isa_klassptr();
2141 if (tkls != nullptr) {
2142 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2143 ciKlass* klass = tkls->exact_klass();
2144 // We are loading a field from a Klass metaobject whose identity
2145 // is known at compile time (the type is "exact" or "precise").
2146 // Check for fields we know are maintained as constants by the VM.
2147 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2148 // The field is Klass::_super_check_offset. Return its (constant) value.
2149 // (Folds up type checking code.)
2150 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2151 return TypeInt::make(klass->super_check_offset());
2152 }
2153 // Compute index into primary_supers array
2154 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2155 // Check for overflowing; use unsigned compare to handle the negative case.
2221 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2222 Node* value = can_see_stored_value(mem,phase);
2223 if (value != nullptr && value->is_Con()) {
2224 assert(value->bottom_type()->higher_equal(_type),"sanity");
2225 return value->bottom_type();
2226 }
2227 }
2228
2229 bool is_vect = (_type->isa_vect() != nullptr);
2230 if (is_instance && !is_vect) {
2231 // If we have an instance type and our memory input is the
2232 // programs's initial memory state, there is no matching store,
2233 // so just return a zero of the appropriate type -
2234 // except if it is vectorized - then we have no zero constant.
2235 Node *mem = in(MemNode::Memory);
2236 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2237 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2238 return Type::get_zero_type(_type->basic_type());
2239 }
2240 }
2241 Node* alloc = is_new_object_mark_load();
2242 if (alloc != nullptr) {
2243 if (EnableValhalla) {
2244 // The mark word may contain property bits (inline, flat, null-free)
2245 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2246 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2247 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2248 return TypeX::make(tkls->exact_klass()->prototype_header().value());
2249 }
2250 } else {
2251 return TypeX::make(markWord::prototype().value());
2252 }
2253 }
2254
2255 return _type;
2256 }
2257
2258 //------------------------------match_edge-------------------------------------
2259 // Do we Match on this edge index or not? Match only the address.
2260 uint LoadNode::match_edge(uint idx) const {
2261 return idx == MemNode::Address;
2262 }
2263
2264 //--------------------------LoadBNode::Ideal--------------------------------------
2265 //
2266 // If the previous store is to the same address as this load,
2267 // and the value stored was larger than a byte, replace this load
2268 // with the value stored truncated to a byte. If no truncation is
2269 // needed, the replacement is done in LoadNode::Identity().
2270 //
2271 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2272 Node* mem = in(MemNode::Memory);
2383 return LoadNode::Ideal(phase, can_reshape);
2384 }
2385
2386 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2387 Node* mem = in(MemNode::Memory);
2388 Node* value = can_see_stored_value(mem,phase);
2389 if (value != nullptr && value->is_Con() &&
2390 !value->bottom_type()->higher_equal(_type)) {
2391 // If the input to the store does not fit with the load's result type,
2392 // it must be truncated. We can't delay until Ideal call since
2393 // a singleton Value is needed for split_thru_phi optimization.
2394 int con = value->get_int();
2395 return TypeInt::make((con << 16) >> 16);
2396 }
2397 return LoadNode::Value(phase);
2398 }
2399
2400 //=============================================================================
2401 //----------------------------LoadKlassNode::make------------------------------
2402 // Polymorphic factory method:
2403 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2404 const TypeKlassPtr* tk) {
2405 // sanity check the alias category against the created node type
2406 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2407 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2408 #ifdef _LP64
2409 if (adr_type->is_ptr_to_narrowklass()) {
2410 assert(UseCompressedClassPointers, "no compressed klasses");
2411 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2412 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2413 }
2414 #endif
2415 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2416 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2417 }
2418
2419 //------------------------------Value------------------------------------------
2420 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2421 return klass_value_common(phase);
2422 }
2423
2424 // In most cases, LoadKlassNode does not have the control input set. If the control
2431 // Either input is TOP ==> the result is TOP
2432 const Type *t1 = phase->type( in(MemNode::Memory) );
2433 if (t1 == Type::TOP) return Type::TOP;
2434 Node *adr = in(MemNode::Address);
2435 const Type *t2 = phase->type( adr );
2436 if (t2 == Type::TOP) return Type::TOP;
2437 const TypePtr *tp = t2->is_ptr();
2438 if (TypePtr::above_centerline(tp->ptr()) ||
2439 tp->ptr() == TypePtr::Null) return Type::TOP;
2440
2441 // Return a more precise klass, if possible
2442 const TypeInstPtr *tinst = tp->isa_instptr();
2443 if (tinst != nullptr) {
2444 ciInstanceKlass* ik = tinst->instance_klass();
2445 int offset = tinst->offset();
2446 if (ik == phase->C->env()->Class_klass()
2447 && (offset == java_lang_Class::klass_offset() ||
2448 offset == java_lang_Class::array_klass_offset())) {
2449 // We are loading a special hidden field from a Class mirror object,
2450 // the field which points to the VM's Klass metaobject.
2451 bool is_null_free_array = false;
2452 ciType* t = tinst->java_mirror_type(&is_null_free_array);
2453 // java_mirror_type returns non-null for compile-time Class constants.
2454 if (t != nullptr) {
2455 // constant oop => constant klass
2456 if (offset == java_lang_Class::array_klass_offset()) {
2457 if (t->is_void()) {
2458 // We cannot create a void array. Since void is a primitive type return null
2459 // klass. Users of this result need to do a null check on the returned klass.
2460 return TypePtr::NULL_PTR;
2461 }
2462 const TypeKlassPtr* tklass = TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2463 if (is_null_free_array) {
2464 tklass = tklass->is_aryklassptr()->cast_to_null_free();
2465 }
2466 return tklass;
2467 }
2468 if (!t->is_klass()) {
2469 // a primitive Class (e.g., int.class) has null for a klass field
2470 return TypePtr::NULL_PTR;
2471 }
2472 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2473 const TypeKlassPtr* tklass = TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2474 if (is_null_free_array) {
2475 tklass = tklass->is_aryklassptr()->cast_to_null_free();
2476 }
2477 return tklass;
2478 }
2479 // non-constant mirror, so we can't tell what's going on
2480 }
2481 if (!tinst->is_loaded())
2482 return _type; // Bail out if not loaded
2483 if (offset == oopDesc::klass_offset_in_bytes()) {
2484 return tinst->as_klass_type(true);
2485 }
2486 }
2487
2488 // Check for loading klass from an array
2489 const TypeAryPtr* tary = tp->isa_aryptr();
2490 if (tary != nullptr &&
2491 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2492 return tary->as_klass_type(true);
2493 }
2494
2495 // Check for loading klass from an array klass
2496 const TypeKlassPtr *tkls = tp->isa_klassptr();
2497 if (tkls != nullptr && !StressReflectiveCode) {
2498 if (!tkls->is_loaded())
2499 return _type; // Bail out if not loaded
2500 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2501 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2502 // // Always returning precise element type is incorrect,
2503 // // e.g., element type could be object and array may contain strings
2504 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2505
2506 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2507 // according to the element type's subclassing.
2508 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2509 }
2751
2752 // Since they are not commoned, do not hash them:
2753 return NO_HASH;
2754 }
2755
2756 //------------------------------Ideal------------------------------------------
2757 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2758 // When a store immediately follows a relevant allocation/initialization,
2759 // try to capture it into the initialization, or hoist it above.
2760 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2761 Node* p = MemNode::Ideal_common(phase, can_reshape);
2762 if (p) return (p == NodeSentinel) ? nullptr : p;
2763
2764 Node* mem = in(MemNode::Memory);
2765 Node* address = in(MemNode::Address);
2766 Node* value = in(MemNode::ValueIn);
2767 // Back-to-back stores to same address? Fold em up. Generally
2768 // unsafe if I have intervening uses... Also disallowed for StoreCM
2769 // since they must follow each StoreP operation. Redundant StoreCMs
2770 // are eliminated just before matching in final_graph_reshape.
2771 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
2772 Node* st = mem;
2773 // If Store 'st' has more than one use, we cannot fold 'st' away.
2774 // For example, 'st' might be the final state at a conditional
2775 // return. Or, 'st' might be used by some node which is live at
2776 // the same time 'st' is live, which might be unschedulable. So,
2777 // require exactly ONE user until such time as we clone 'mem' for
2778 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2779 // true).
2780 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2781 // Looking at a dead closed cycle of memory?
2782 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2783 assert(Opcode() == st->Opcode() ||
2784 st->Opcode() == Op_StoreVector ||
2785 Opcode() == Op_StoreVector ||
2786 st->Opcode() == Op_StoreVectorScatter ||
2787 Opcode() == Op_StoreVectorScatter ||
2788 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2789 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2790 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2791 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
2792 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2793 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2794
2795 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2796 st->as_Store()->memory_size() <= this->memory_size()) {
2797 Node* use = st->raw_out(0);
2798 if (phase->is_IterGVN()) {
2799 phase->is_IterGVN()->rehash_node_delayed(use);
2800 }
2801 // It's OK to do this in the parser, since DU info is always accurate,
2802 // and the parser always refers to nodes via SafePointNode maps.
2803 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2804 return this;
2805 }
2806 st = st->in(MemNode::Memory);
2807 }
2808 }
2809
2810
2811 // Capture an unaliased, unconditional, simple store into an initializer.
2868 // Load then Store? Then the Store is useless
2869 if (val->is_Load() &&
2870 val->in(MemNode::Address)->eqv_uncast(adr) &&
2871 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2872 val->as_Load()->store_Opcode() == Opcode()) {
2873 result = mem;
2874 }
2875
2876 // Two stores in a row of the same value?
2877 if (result == this &&
2878 mem->is_Store() &&
2879 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2880 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2881 mem->Opcode() == Opcode()) {
2882 result = mem;
2883 }
2884
2885 // Store of zero anywhere into a freshly-allocated object?
2886 // Then the store is useless.
2887 // (It must already have been captured by the InitializeNode.)
2888 if (result == this && ReduceFieldZeroing) {
2889 // a newly allocated object is already all-zeroes everywhere
2890 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
2891 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
2892 result = mem;
2893 }
2894
2895 if (result == this && phase->type(val)->is_zero_type()) {
2896 // the store may also apply to zero-bits in an earlier object
2897 Node* prev_mem = find_previous_store(phase);
2898 // Steps (a), (b): Walk past independent stores to find an exact match.
2899 if (prev_mem != nullptr) {
2900 Node* prev_val = can_see_stored_value(prev_mem, phase);
2901 if (prev_val != nullptr && prev_val == val) {
2902 // prev_val and val might differ by a cast; it would be good
2903 // to keep the more informative of the two.
2904 result = mem;
2905 }
2906 }
2907 }
2908 }
2909
2910 PhaseIterGVN* igvn = phase->is_IterGVN();
2911 if (result != this && igvn != nullptr) {
2912 MemBarNode* trailing = trailing_membar();
2913 if (trailing != nullptr) {
2914 #ifdef ASSERT
2915 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
3060 Node* StoreCMNode::Identity(PhaseGVN* phase) {
3061 // No need to card mark when storing a null ptr
3062 Node* my_store = in(MemNode::OopStore);
3063 if (my_store->is_Store()) {
3064 const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
3065 if( t1 == TypePtr::NULL_PTR ) {
3066 return in(MemNode::Memory);
3067 }
3068 }
3069 return this;
3070 }
3071
3072 //=============================================================================
3073 //------------------------------Ideal---------------------------------------
3074 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
3075 Node* progress = StoreNode::Ideal(phase, can_reshape);
3076 if (progress != nullptr) return progress;
3077
3078 Node* my_store = in(MemNode::OopStore);
3079 if (my_store->is_MergeMem()) {
3080 if (oop_alias_idx() != phase->C->get_alias_index(TypeAryPtr::INLINES) ||
3081 phase->C->flat_accesses_share_alias()) {
3082 // The alias that was recorded is no longer accurate enough.
3083 Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
3084 set_req_X(MemNode::OopStore, mem, phase);
3085 return this;
3086 }
3087 }
3088
3089 return nullptr;
3090 }
3091
3092 //------------------------------Value-----------------------------------------
3093 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
3094 // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
3095 // If extra input is TOP ==> the result is TOP
3096 const Type* t = phase->type(in(MemNode::OopStore));
3097 if (t == Type::TOP) {
3098 return Type::TOP;
3099 }
3100 return StoreNode::Value(phase);
3101 }
3102
3103
3104 //=============================================================================
3105 //----------------------------------SCMemProjNode------------------------------
3106 const Type* SCMemProjNode::Value(PhaseGVN* phase) const
3237 // Clearing a short array is faster with stores
3238 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3239 // Already know this is a large node, do not try to ideal it
3240 if (_is_large) return nullptr;
3241
3242 const int unit = BytesPerLong;
3243 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3244 if (!t) return nullptr;
3245 if (!t->is_con()) return nullptr;
3246 intptr_t raw_count = t->get_con();
3247 intptr_t size = raw_count;
3248 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3249 // Clearing nothing uses the Identity call.
3250 // Negative clears are possible on dead ClearArrays
3251 // (see jck test stmt114.stmt11402.val).
3252 if (size <= 0 || size % unit != 0) return nullptr;
3253 intptr_t count = size / unit;
3254 // Length too long; communicate this to matchers and assemblers.
3255 // Assemblers are responsible to produce fast hardware clears for it.
3256 if (size > InitArrayShortSize) {
3257 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3258 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3259 return nullptr;
3260 }
3261 if (!IdealizeClearArrayNode) return nullptr;
3262 Node *mem = in(1);
3263 if( phase->type(mem)==Type::TOP ) return nullptr;
3264 Node *adr = in(3);
3265 const Type* at = phase->type(adr);
3266 if( at==Type::TOP ) return nullptr;
3267 const TypePtr* atp = at->isa_ptr();
3268 // adjust atp to be the correct array element address type
3269 if (atp == nullptr) atp = TypePtr::BOTTOM;
3270 else atp = atp->add_offset(Type::OffsetBot);
3271 // Get base for derived pointer purposes
3272 if( adr->Opcode() != Op_AddP ) Unimplemented();
3273 Node *base = adr->in(1);
3274
3275 Node *val = in(4);
3276 Node *off = phase->MakeConX(BytesPerLong);
3277 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3278 count--;
3279 while( count-- ) {
3280 mem = phase->transform(mem);
3281 adr = phase->transform(new AddPNode(base,adr,off));
3282 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3283 }
3284 return mem;
3285 }
3286
3287 //----------------------------step_through----------------------------------
3288 // Return allocation input memory edge if it is different instance
3289 // or itself if it is the one we are looking for.
3290 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3291 Node* n = *np;
3292 assert(n->is_ClearArray(), "sanity");
3293 intptr_t offset;
3294 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3295 // This method is called only before Allocate nodes are expanded
3296 // during macro nodes expansion. Before that ClearArray nodes are
3297 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3298 // Allocate nodes are expanded) which follows allocations.
3299 assert(alloc != nullptr, "should have allocation");
3300 if (alloc->_idx == instance_id) {
3301 // Can not bypass initialization of the instance we are looking for.
3302 return false;
3303 }
3304 // Otherwise skip it.
3305 InitializeNode* init = alloc->initialization();
3306 if (init != nullptr)
3307 *np = init->in(TypeFunc::Memory);
3308 else
3309 *np = alloc->in(TypeFunc::Memory);
3310 return true;
3311 }
3312
3313 //----------------------------clear_memory-------------------------------------
3314 // Generate code to initialize object storage to zero.
3315 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3316 Node* val,
3317 Node* raw_val,
3318 intptr_t start_offset,
3319 Node* end_offset,
3320 PhaseGVN* phase) {
3321 intptr_t offset = start_offset;
3322
3323 int unit = BytesPerLong;
3324 if ((offset % unit) != 0) {
3325 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3326 adr = phase->transform(adr);
3327 const TypePtr* atp = TypeRawPtr::BOTTOM;
3328 if (val != nullptr) {
3329 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3330 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3331 } else {
3332 assert(raw_val == nullptr, "val may not be null");
3333 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3334 }
3335 mem = phase->transform(mem);
3336 offset += BytesPerInt;
3337 }
3338 assert((offset % unit) == 0, "");
3339
3340 // Initialize the remaining stuff, if any, with a ClearArray.
3341 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3342 }
3343
3344 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3345 Node* raw_val,
3346 Node* start_offset,
3347 Node* end_offset,
3348 PhaseGVN* phase) {
3349 if (start_offset == end_offset) {
3350 // nothing to do
3351 return mem;
3352 }
3353
3354 int unit = BytesPerLong;
3355 Node* zbase = start_offset;
3356 Node* zend = end_offset;
3357
3358 // Scale to the unit required by the CPU:
3359 if (!Matcher::init_array_count_is_in_bytes) {
3360 Node* shift = phase->intcon(exact_log2(unit));
3361 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3362 zend = phase->transform(new URShiftXNode(zend, shift) );
3363 }
3364
3365 // Bulk clear double-words
3366 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3367 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3368 if (raw_val == nullptr) {
3369 raw_val = phase->MakeConX(0);
3370 }
3371 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3372 return phase->transform(mem);
3373 }
3374
3375 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3376 Node* val,
3377 Node* raw_val,
3378 intptr_t start_offset,
3379 intptr_t end_offset,
3380 PhaseGVN* phase) {
3381 if (start_offset == end_offset) {
3382 // nothing to do
3383 return mem;
3384 }
3385
3386 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3387 intptr_t done_offset = end_offset;
3388 if ((done_offset % BytesPerLong) != 0) {
3389 done_offset -= BytesPerInt;
3390 }
3391 if (done_offset > start_offset) {
3392 mem = clear_memory(ctl, mem, dest, val, raw_val,
3393 start_offset, phase->MakeConX(done_offset), phase);
3394 }
3395 if (done_offset < end_offset) { // emit the final 32-bit store
3396 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3397 adr = phase->transform(adr);
3398 const TypePtr* atp = TypeRawPtr::BOTTOM;
3399 if (val != nullptr) {
3400 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3401 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3402 } else {
3403 assert(raw_val == nullptr, "val may not be null");
3404 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3405 }
3406 mem = phase->transform(mem);
3407 done_offset += BytesPerInt;
3408 }
3409 assert(done_offset == end_offset, "");
3410 return mem;
3411 }
3412
3413 //=============================================================================
3414 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3415 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3416 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3417 #ifdef ASSERT
3418 , _pair_idx(0)
3419 #endif
3420 {
3421 init_class_id(Class_MemBar);
3422 Node* top = C->top();
3423 init_req(TypeFunc::I_O,top);
3424 init_req(TypeFunc::FramePtr,top);
3425 init_req(TypeFunc::ReturnAdr,top);
3530 PhaseIterGVN* igvn = phase->is_IterGVN();
3531 remove(igvn);
3532 // Must return either the original node (now dead) or a new node
3533 // (Do not return a top here, since that would break the uniqueness of top.)
3534 return new ConINode(TypeInt::ZERO);
3535 }
3536 }
3537 return progress ? this : nullptr;
3538 }
3539
3540 //------------------------------Value------------------------------------------
3541 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3542 if( !in(0) ) return Type::TOP;
3543 if( phase->type(in(0)) == Type::TOP )
3544 return Type::TOP;
3545 return TypeTuple::MEMBAR;
3546 }
3547
3548 //------------------------------match------------------------------------------
3549 // Construct projections for memory.
3550 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
3551 switch (proj->_con) {
3552 case TypeFunc::Control:
3553 case TypeFunc::Memory:
3554 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3555 }
3556 ShouldNotReachHere();
3557 return nullptr;
3558 }
3559
3560 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3561 trailing->_kind = TrailingStore;
3562 leading->_kind = LeadingStore;
3563 #ifdef ASSERT
3564 trailing->_pair_idx = leading->_idx;
3565 leading->_pair_idx = leading->_idx;
3566 #endif
3567 }
3568
3569 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3570 trailing->_kind = TrailingLoadStore;
3817 return (req() > RawStores);
3818 }
3819
3820 void InitializeNode::set_complete(PhaseGVN* phase) {
3821 assert(!is_complete(), "caller responsibility");
3822 _is_complete = Complete;
3823
3824 // After this node is complete, it contains a bunch of
3825 // raw-memory initializations. There is no need for
3826 // it to have anything to do with non-raw memory effects.
3827 // Therefore, tell all non-raw users to re-optimize themselves,
3828 // after skipping the memory effects of this initialization.
3829 PhaseIterGVN* igvn = phase->is_IterGVN();
3830 if (igvn) igvn->add_users_to_worklist(this);
3831 }
3832
3833 // convenience function
3834 // return false if the init contains any stores already
3835 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3836 InitializeNode* init = initialization();
3837 if (init == nullptr || init->is_complete()) {
3838 return false;
3839 }
3840 init->remove_extra_zeroes();
3841 // for now, if this allocation has already collected any inits, bail:
3842 if (init->is_non_zero()) return false;
3843 init->set_complete(phase);
3844 return true;
3845 }
3846
3847 void InitializeNode::remove_extra_zeroes() {
3848 if (req() == RawStores) return;
3849 Node* zmem = zero_memory();
3850 uint fill = RawStores;
3851 for (uint i = fill; i < req(); i++) {
3852 Node* n = in(i);
3853 if (n->is_top() || n == zmem) continue; // skip
3854 if (fill < i) set_req(fill, n); // compact
3855 ++fill;
3856 }
3857 // delete any empty spaces created:
3858 while (fill < req()) {
3859 del_req(fill);
3997 // store node that we'd like to capture. We need to check
3998 // the uses of the MergeMemNode.
3999 mems.push(n);
4000 }
4001 } else if (n->is_Mem()) {
4002 Node* other_adr = n->in(MemNode::Address);
4003 if (other_adr == adr) {
4004 failed = true;
4005 break;
4006 } else {
4007 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4008 if (other_t_adr != nullptr) {
4009 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4010 if (other_alias_idx == alias_idx) {
4011 // A load from the same memory slice as the store right
4012 // after the InitializeNode. We check the control of the
4013 // object/array that is loaded from. If it's the same as
4014 // the store control then we cannot capture the store.
4015 assert(!n->is_Store(), "2 stores to same slice on same control?");
4016 Node* base = other_adr;
4017 if (base->is_Phi()) {
4018 // In rare case, base may be a PhiNode and it may read
4019 // the same memory slice between InitializeNode and store.
4020 failed = true;
4021 break;
4022 }
4023 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4024 base = base->in(AddPNode::Base);
4025 if (base != nullptr) {
4026 base = base->uncast();
4027 if (base->is_Proj() && base->in(0) == alloc) {
4028 failed = true;
4029 break;
4030 }
4031 }
4032 }
4033 }
4034 }
4035 } else {
4036 failed = true;
4037 break;
4038 }
4039 }
4040 }
4041 }
4042 if (failed) {
4589 // z's_done 12 16 16 16 12 16 12
4590 // z's_needed 12 16 16 16 16 16 16
4591 // zsize 0 0 0 0 4 0 4
4592 if (next_full_store < 0) {
4593 // Conservative tack: Zero to end of current word.
4594 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4595 } else {
4596 // Zero to beginning of next fully initialized word.
4597 // Or, don't zero at all, if we are already in that word.
4598 assert(next_full_store >= zeroes_needed, "must go forward");
4599 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4600 zeroes_needed = next_full_store;
4601 }
4602 }
4603
4604 if (zeroes_needed > zeroes_done) {
4605 intptr_t zsize = zeroes_needed - zeroes_done;
4606 // Do some incremental zeroing on rawmem, in parallel with inits.
4607 zeroes_done = align_down(zeroes_done, BytesPerInt);
4608 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4609 allocation()->in(AllocateNode::DefaultValue),
4610 allocation()->in(AllocateNode::RawDefaultValue),
4611 zeroes_done, zeroes_needed,
4612 phase);
4613 zeroes_done = zeroes_needed;
4614 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4615 do_zeroing = false; // leave the hole, next time
4616 }
4617 }
4618
4619 // Collect the store and move on:
4620 phase->replace_input_of(st, MemNode::Memory, inits);
4621 inits = st; // put it on the linearized chain
4622 set_req(i, zmem); // unhook from previous position
4623
4624 if (zeroes_done == st_off)
4625 zeroes_done = next_init_off;
4626
4627 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4628
4629 #ifdef ASSERT
4630 // Various order invariants. Weaker than stores_are_sane because
4650 remove_extra_zeroes(); // clear out all the zmems left over
4651 add_req(inits);
4652
4653 if (!(UseTLAB && ZeroTLAB)) {
4654 // If anything remains to be zeroed, zero it all now.
4655 zeroes_done = align_down(zeroes_done, BytesPerInt);
4656 // if it is the last unused 4 bytes of an instance, forget about it
4657 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4658 if (zeroes_done + BytesPerLong >= size_limit) {
4659 AllocateNode* alloc = allocation();
4660 assert(alloc != nullptr, "must be present");
4661 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
4662 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4663 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4664 if (zeroes_done == k->layout_helper())
4665 zeroes_done = size_limit;
4666 }
4667 }
4668 if (zeroes_done < size_limit) {
4669 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4670 allocation()->in(AllocateNode::DefaultValue),
4671 allocation()->in(AllocateNode::RawDefaultValue),
4672 zeroes_done, size_in_bytes, phase);
4673 }
4674 }
4675
4676 set_complete(phase);
4677 return rawmem;
4678 }
4679
4680
4681 #ifdef ASSERT
4682 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
4683 if (is_complete())
4684 return true; // stores could be anything at this point
4685 assert(allocation() != nullptr, "must be present");
4686 intptr_t last_off = allocation()->minimum_header_size();
4687 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4688 Node* st = in(i);
4689 intptr_t st_off = get_store_offset(st, phase);
4690 if (st_off < 0) continue; // ignore dead garbage
4691 if (last_off > st_off) {
|