6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/regalloc.hpp"
38 #include "opto/compile.hpp"
39 #include "opto/connode.hpp"
40 #include "opto/convertnode.hpp"
41 #include "opto/loopnode.hpp"
42 #include "opto/machnode.hpp"
43 #include "opto/matcher.hpp"
44 #include "opto/memnode.hpp"
45 #include "opto/mempointer.hpp"
46 #include "opto/mulnode.hpp"
47 #include "opto/narrowptrnode.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regmask.hpp"
50 #include "opto/rootnode.hpp"
51 #include "opto/traceMergeStoresTag.hpp"
52 #include "opto/vectornode.hpp"
53 #include "utilities/align.hpp"
54 #include "utilities/copy.hpp"
55 #include "utilities/macros.hpp"
56 #include "utilities/powerOfTwo.hpp"
57 #include "utilities/vmError.hpp"
58
59 // Portions of code courtesy of Clifford Click
60
123 st->print(", idx=Bot;");
124 else if (atp->index() == Compile::AliasIdxTop)
125 st->print(", idx=Top;");
126 else if (atp->index() == Compile::AliasIdxRaw)
127 st->print(", idx=Raw;");
128 else {
129 ciField* field = atp->field();
130 if (field) {
131 st->print(", name=");
132 field->print_name_on(st);
133 }
134 st->print(", idx=%d;", atp->index());
135 }
136 }
137 }
138
139 extern void print_alias_types();
140
141 #endif
142
143 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
144 assert((t_oop != nullptr), "sanity");
145 bool is_instance = t_oop->is_known_instance_field();
146 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
147 (load != nullptr) && load->is_Load() &&
148 (phase->is_IterGVN() != nullptr);
149 if (!(is_instance || is_boxed_value_load))
150 return mchain; // don't try to optimize non-instance types
151 uint instance_id = t_oop->instance_id();
152 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
153 Node *prev = nullptr;
154 Node *result = mchain;
155 while (prev != result) {
156 prev = result;
157 if (result == start_mem)
158 break; // hit one of our sentinels
159 // skip over a call which does not affect this memory slice
160 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
161 Node *proj_in = result->in(0);
162 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
163 break; // hit one of our sentinels
164 } else if (proj_in->is_Call()) {
165 // ArrayCopyNodes processed here as well
166 CallNode *call = proj_in->as_Call();
167 if (!call->may_modify(t_oop, phase)) { // returns false for instances
168 result = call->in(TypeFunc::Memory);
169 }
170 } else if (proj_in->is_Initialize()) {
171 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
172 // Stop if this is the initialization for the object instance which
173 // contains this memory slice, otherwise skip over it.
174 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
175 break;
176 }
177 if (is_instance) {
178 result = proj_in->in(TypeFunc::Memory);
179 } else if (is_boxed_value_load) {
180 Node* klass = alloc->in(AllocateNode::KlassNode);
181 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
182 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
183 result = proj_in->in(TypeFunc::Memory); // not related allocation
184 }
185 }
186 } else if (proj_in->is_MemBar()) {
187 ArrayCopyNode* ac = nullptr;
188 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
189 break;
190 }
191 result = proj_in->in(TypeFunc::Memory);
192 } else if (proj_in->is_top()) {
193 break; // dead code
194 } else {
195 assert(false, "unexpected projection");
196 }
197 } else if (result->is_ClearArray()) {
198 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
199 // Can not bypass initialization of the instance
200 // we are looking for.
201 break;
202 }
203 // Otherwise skip it (the call updated 'result' value).
216 bool is_instance = t_oop->is_known_instance_field();
217 PhaseIterGVN *igvn = phase->is_IterGVN();
218 if (is_instance && igvn != nullptr && result->is_Phi()) {
219 PhiNode *mphi = result->as_Phi();
220 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
221 const TypePtr *t = mphi->adr_type();
222 bool do_split = false;
223 // In the following cases, Load memory input can be further optimized based on
224 // its precise address type
225 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
226 do_split = true;
227 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
228 const TypeOopPtr* mem_t =
229 t->is_oopptr()->cast_to_exactness(true)
230 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
231 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
232 if (t_oop->isa_aryptr()) {
233 mem_t = mem_t->is_aryptr()
234 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
235 ->cast_to_size(t_oop->is_aryptr()->size())
236 ->with_offset(t_oop->is_aryptr()->offset())
237 ->is_aryptr();
238 }
239 do_split = mem_t == t_oop;
240 }
241 if (do_split) {
242 // clone the Phi with our address type
243 result = mphi->split_out_instance(t_adr, igvn);
244 } else {
245 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
246 }
247 }
248 return result;
249 }
250
251 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
252 uint alias_idx = phase->C->get_alias_index(tp);
253 Node *mem = mmem;
254 #ifdef ASSERT
255 {
256 // Check that current type is consistent with the alias index used during graph construction
257 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
258 bool consistent = adr_check == nullptr || adr_check->empty() ||
259 phase->C->must_alias(adr_check, alias_idx );
260 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
261 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
262 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
263 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
264 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
265 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
266 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
267 // don't assert if it is dead code.
268 consistent = true;
269 }
270 if( !consistent ) {
271 st->print("alias_idx==%d, adr_check==", alias_idx);
272 if( adr_check == nullptr ) {
273 st->print("null");
274 } else {
275 adr_check->dump();
276 }
277 st->cr();
278 print_alias_types();
279 assert(consistent, "adr_check must match alias idx");
280 }
281 }
282 #endif
995 Node* ld = gvn.transform(load);
996 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
997 }
998
999 return load;
1000 }
1001
1002 //------------------------------hash-------------------------------------------
1003 uint LoadNode::hash() const {
1004 // unroll addition of interesting fields
1005 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1006 }
1007
1008 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1009 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1010 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1011 bool is_stable_ary = FoldStableValues &&
1012 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1013 tp->isa_aryptr()->is_stable();
1014
1015 return (eliminate_boxing && non_volatile) || is_stable_ary;
1016 }
1017
1018 return false;
1019 }
1020
1021 LoadNode* LoadNode::pin_array_access_node() const {
1022 const TypePtr* adr_type = this->adr_type();
1023 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1024 return clone_pinned();
1025 }
1026 return nullptr;
1027 }
1028
1029 // Is the value loaded previously stored by an arraycopy? If so return
1030 // a load node that reads from the source array so we may be able to
1031 // optimize out the ArrayCopy node later.
1032 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1033 Node* ld_adr = in(MemNode::Address);
1034 intptr_t ld_off = 0;
1035 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1052 assert(ld_alloc != nullptr, "need an alloc");
1053 assert(addp->is_AddP(), "address must be addp");
1054 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1055 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1056 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1057 addp->set_req(AddPNode::Base, src);
1058 addp->set_req(AddPNode::Address, src);
1059 } else {
1060 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1061 ac->as_ArrayCopy()->is_copyof_validated() ||
1062 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1063 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1064 addp->set_req(AddPNode::Base, src);
1065 addp->set_req(AddPNode::Address, src);
1066
1067 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1068 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1069 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1070
1071 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1072 uint shift = exact_log2(type2aelembytes(ary_elem));
1073
1074 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1075 #ifdef _LP64
1076 diff = phase->transform(new ConvI2LNode(diff));
1077 #endif
1078 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1079
1080 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1081 addp->set_req(AddPNode::Offset, offset);
1082 }
1083 addp = phase->transform(addp);
1084 #ifdef ASSERT
1085 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1086 ld->_adr_type = adr_type;
1087 #endif
1088 ld->set_req(MemNode::Address, addp);
1089 ld->set_req(0, ctl);
1090 ld->set_req(MemNode::Memory, mem);
1091 return ld;
1092 }
1093 return nullptr;
1094 }
1095
1096
1097 //---------------------------can_see_stored_value------------------------------
1098 // This routine exists to make sure this set of tests is done the same
1099 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1100 // will change the graph shape in a way which makes memory alive twice at the
1101 // same time (uses the Oracle model of aliasing), then some
1102 // LoadXNode::Identity will fold things back to the equivalence-class model
1103 // of aliasing.
1104 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1105 Node* ld_adr = in(MemNode::Address);
1106 intptr_t ld_off = 0;
1107 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1108 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1109 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1110 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1111 // This is more general than load from boxing objects.
1112 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1113 uint alias_idx = atp->index();
1114 Node* result = nullptr;
1115 Node* current = st;
1116 // Skip through chains of MemBarNodes checking the MergeMems for
1117 // new states for the slice of this load. Stop once any other
1118 // kind of node is encountered. Loads from final memory can skip
1119 // through any kind of MemBar but normal loads shouldn't skip
1120 // through MemBarAcquire since the could allow them to move out of
1121 // a synchronized region. It is not safe to step over MemBarCPUOrder,
1122 // because alias info above them may be inaccurate (e.g., due to
1123 // mixed/mismatched unsafe accesses).
1124 bool is_final_mem = !atp->is_rewritable();
1125 while (current->is_Proj()) {
1126 int opc = current->in(0)->Opcode();
1127 if ((is_final_mem && (opc == Op_MemBarAcquire ||
1171 // Same base, same offset.
1172 // Possible improvement for arrays: check index value instead of absolute offset.
1173
1174 // At this point we have proven something like this setup:
1175 // B = << base >>
1176 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1177 // S = StoreQ(AddP( B , #Off), V)
1178 // (Actually, we haven't yet proven the Q's are the same.)
1179 // In other words, we are loading from a casted version of
1180 // the same pointer-and-offset that we stored to.
1181 // Casted version may carry a dependency and it is respected.
1182 // Thus, we are able to replace L by V.
1183 }
1184 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1185 if (store_Opcode() != st->Opcode()) {
1186 return nullptr;
1187 }
1188 // LoadVector/StoreVector needs additional check to ensure the types match.
1189 if (st->is_StoreVector()) {
1190 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1191 const TypeVect* out_vt = as_LoadVector()->vect_type();
1192 if (in_vt != out_vt) {
1193 return nullptr;
1194 }
1195 }
1196 return st->in(MemNode::ValueIn);
1197 }
1198
1199 // A load from a freshly-created object always returns zero.
1200 // (This can happen after LoadNode::Ideal resets the load's memory input
1201 // to find_captured_store, which returned InitializeNode::zero_memory.)
1202 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1203 (st->in(0) == ld_alloc) &&
1204 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1205 // return a zero value for the load's basic type
1206 // (This is one of the few places where a generic PhaseTransform
1207 // can create new nodes. Think of it as lazily manifesting
1208 // virtually pre-existing constants.)
1209 if (memory_type() != T_VOID) {
1210 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1211 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1212 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1213 // by the ArrayCopyNode.
1214 return phase->zerocon(memory_type());
1215 }
1216 } else {
1217 // TODO: materialize all-zero vector constant
1218 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1219 }
1220 }
1221
1222 // A load from an initialization barrier can match a captured store.
1223 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1224 InitializeNode* init = st->in(0)->as_Initialize();
1225 AllocateNode* alloc = init->allocation();
1226 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1227 // examine a captured store value
1228 st = init->find_captured_store(ld_off, memory_size(), phase);
1849 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1850 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1851
1852 // Skip up past a SafePoint control. Cannot do this for Stores because
1853 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1854 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1855 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1856 !addr_mark &&
1857 (depends_only_on_test() || has_unknown_control_dependency())) {
1858 ctrl = ctrl->in(0);
1859 set_req(MemNode::Control,ctrl);
1860 progress = true;
1861 }
1862
1863 intptr_t ignore = 0;
1864 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1865 if (base != nullptr
1866 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1867 // Check for useless control edge in some common special cases
1868 if (in(MemNode::Control) != nullptr
1869 && can_remove_control()
1870 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1871 && all_controls_dominate(base, phase->C->start())) {
1872 // A method-invariant, non-null address (constant or 'this' argument).
1873 set_req(MemNode::Control, nullptr);
1874 progress = true;
1875 }
1876 }
1877
1878 Node* mem = in(MemNode::Memory);
1879 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1880
1881 if (can_reshape && (addr_t != nullptr)) {
1882 // try to optimize our memory input
1883 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1884 if (opt_mem != mem) {
1885 set_req_X(MemNode::Memory, opt_mem, phase);
1886 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1887 return this;
1888 }
1945 // fold up, do so.
1946 Node* prev_mem = find_previous_store(phase);
1947 if (prev_mem != nullptr) {
1948 Node* value = can_see_arraycopy_value(prev_mem, phase);
1949 if (value != nullptr) {
1950 return value;
1951 }
1952 }
1953 // Steps (a), (b): Walk past independent stores to find an exact match.
1954 if (prev_mem != nullptr && prev_mem != in(MemNode::Memory)) {
1955 // (c) See if we can fold up on the spot, but don't fold up here.
1956 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
1957 // just return a prior value, which is done by Identity calls.
1958 if (can_see_stored_value(prev_mem, phase)) {
1959 // Make ready for step (d):
1960 set_req_X(MemNode::Memory, prev_mem, phase);
1961 return this;
1962 }
1963 }
1964
1965 return progress ? this : nullptr;
1966 }
1967
1968 // Helper to recognize certain Klass fields which are invariant across
1969 // some group of array types (e.g., int[] or all T[] where T < Object).
1970 const Type*
1971 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
1972 ciKlass* klass) const {
1973 assert(!UseCompactObjectHeaders || tkls->offset() != in_bytes(Klass::prototype_header_offset()),
1974 "must not happen");
1975 if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
1976 // The field is Klass::_access_flags. Return its (constant) value.
1977 // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
1978 assert(Opcode() == Op_LoadUS, "must load an unsigned short from _access_flags");
1979 return TypeInt::make(klass->access_flags());
1980 }
1981 if (tkls->offset() == in_bytes(Klass::misc_flags_offset())) {
1982 // The field is Klass::_misc_flags. Return its (constant) value.
1983 // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
1984 assert(Opcode() == Op_LoadUB, "must load an unsigned byte from _misc_flags");
1985 return TypeInt::make(klass->misc_flags());
2047 }
2048 }
2049
2050 // Don't do this for integer types. There is only potential profit if
2051 // the element type t is lower than _type; that is, for int types, if _type is
2052 // more restrictive than t. This only happens here if one is short and the other
2053 // char (both 16 bits), and in those cases we've made an intentional decision
2054 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2055 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2056 //
2057 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2058 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2059 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2060 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2061 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2062 // In fact, that could have been the original type of p1, and p1 could have
2063 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2064 // expression (LShiftL quux 3) independently optimized to the constant 8.
2065 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2066 && (_type->isa_vect() == nullptr)
2067 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2068 // t might actually be lower than _type, if _type is a unique
2069 // concrete subclass of abstract class t.
2070 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2071 const Type* jt = t->join_speculative(_type);
2072 // In any case, do not allow the join, per se, to empty out the type.
2073 if (jt->empty() && !t->empty()) {
2074 // This can happen if a interface-typed array narrows to a class type.
2075 jt = _type;
2076 }
2077 #ifdef ASSERT
2078 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2079 // The pointers in the autobox arrays are always non-null
2080 Node* base = adr->in(AddPNode::Base);
2081 if ((base != nullptr) && base->is_DecodeN()) {
2082 // Get LoadN node which loads IntegerCache.cache field
2083 base = base->in(1);
2084 }
2085 if ((base != nullptr) && base->is_Con()) {
2086 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2087 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2088 // It could be narrow oop
2089 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2090 }
2091 }
2092 }
2093 #endif
2094 return jt;
2095 }
2096 }
2097 } else if (tp->base() == Type::InstPtr) {
2098 assert( off != Type::OffsetBot ||
2099 // arrays can be cast to Objects
2100 !tp->isa_instptr() ||
2101 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2102 // unsafe field access may not have a constant offset
2103 C->has_unsafe_access(),
2104 "Field accesses must be precise" );
2105 // For oop loads, we expect the _type to be precise.
2106
2107 // Optimize loads from constant fields.
2108 const TypeInstPtr* tinst = tp->is_instptr();
2109 ciObject* const_oop = tinst->const_oop();
2110 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2111 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
2112 if (con_type != nullptr) {
2113 return con_type;
2114 }
2115 }
2116 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2117 assert(off != Type::OffsetBot ||
2118 !tp->isa_instklassptr() ||
2119 // arrays can be cast to Objects
2120 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2121 // also allow array-loading from the primary supertype
2122 // array during subtype checks
2123 Opcode() == Op_LoadKlass,
2124 "Field accesses must be precise");
2125 // For klass/static loads, we expect the _type to be precise
2126 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2127 /* With mirrors being an indirect in the Klass*
2128 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2129 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2130 *
2131 * So check the type and klass of the node before the LoadP.
2138 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2139 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2140 return TypeInstPtr::make(klass->java_mirror());
2141 }
2142 }
2143 }
2144
2145 const TypeKlassPtr *tkls = tp->isa_klassptr();
2146 if (tkls != nullptr) {
2147 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2148 ciKlass* klass = tkls->exact_klass();
2149 // We are loading a field from a Klass metaobject whose identity
2150 // is known at compile time (the type is "exact" or "precise").
2151 // Check for fields we know are maintained as constants by the VM.
2152 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2153 // The field is Klass::_super_check_offset. Return its (constant) value.
2154 // (Folds up type checking code.)
2155 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2156 return TypeInt::make(klass->super_check_offset());
2157 }
2158 if (UseCompactObjectHeaders) {
2159 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2160 // The field is Klass::_prototype_header. Return its (constant) value.
2161 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2162 return TypeX::make(klass->prototype_header());
2163 }
2164 }
2165 // Compute index into primary_supers array
2166 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2167 // Check for overflowing; use unsigned compare to handle the negative case.
2168 if( depth < ciKlass::primary_super_limit() ) {
2169 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2170 // (Folds up type checking code.)
2171 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2172 ciKlass *ss = klass->super_of_depth(depth);
2173 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2174 }
2175 const Type* aift = load_array_final_field(tkls, klass);
2176 if (aift != nullptr) return aift;
2177 }
2178
2216 // Note: When interfaces are reliable, we can narrow the interface
2217 // test to (klass != Serializable && klass != Cloneable).
2218 assert(Opcode() == Op_LoadI, "must load an int from _layout_helper");
2219 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2220 // The key property of this type is that it folds up tests
2221 // for array-ness, since it proves that the layout_helper is positive.
2222 // Thus, a generic value like the basic object layout helper works fine.
2223 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2224 }
2225 }
2226
2227 bool is_vect = (_type->isa_vect() != nullptr);
2228 if (is_instance && !is_vect) {
2229 // If we have an instance type and our memory input is the
2230 // programs's initial memory state, there is no matching store,
2231 // so just return a zero of the appropriate type -
2232 // except if it is vectorized - then we have no zero constant.
2233 Node *mem = in(MemNode::Memory);
2234 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2235 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2236 return Type::get_zero_type(_type->basic_type());
2237 }
2238 }
2239
2240 if (!UseCompactObjectHeaders) {
2241 Node* alloc = is_new_object_mark_load();
2242 if (alloc != nullptr) {
2243 return TypeX::make(markWord::prototype().value());
2244 }
2245 }
2246
2247 return _type;
2248 }
2249
2250 //------------------------------match_edge-------------------------------------
2251 // Do we Match on this edge index or not? Match only the address.
2252 uint LoadNode::match_edge(uint idx) const {
2253 return idx == MemNode::Address;
2254 }
2255
2256 //--------------------------LoadBNode::Ideal--------------------------------------
2257 //
2258 // If the previous store is to the same address as this load,
2259 // and the value stored was larger than a byte, replace this load
2260 // with the value stored truncated to a byte. If no truncation is
2261 // needed, the replacement is done in LoadNode::Identity().
2262 //
2263 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2372 }
2373 }
2374 // Identity call will handle the case where truncation is not needed.
2375 return LoadNode::Ideal(phase, can_reshape);
2376 }
2377
2378 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2379 Node* mem = in(MemNode::Memory);
2380 Node* value = can_see_stored_value(mem,phase);
2381 if (value != nullptr && value->is_Con() &&
2382 !value->bottom_type()->higher_equal(_type)) {
2383 // If the input to the store does not fit with the load's result type,
2384 // it must be truncated. We can't delay until Ideal call since
2385 // a singleton Value is needed for split_thru_phi optimization.
2386 int con = value->get_int();
2387 return TypeInt::make((con << 16) >> 16);
2388 }
2389 return LoadNode::Value(phase);
2390 }
2391
2392 //=============================================================================
2393 //----------------------------LoadKlassNode::make------------------------------
2394 // Polymorphic factory method:
2395 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2396 // sanity check the alias category against the created node type
2397 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2398 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2399 #ifdef _LP64
2400 if (adr_type->is_ptr_to_narrowklass()) {
2401 assert(UseCompressedClassPointers, "no compressed klasses");
2402 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2403 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2404 }
2405 #endif
2406 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2407 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2408 }
2409
2410 //------------------------------Value------------------------------------------
2411 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2416 // Either input is TOP ==> the result is TOP
2417 const Type *t1 = phase->type( in(MemNode::Memory) );
2418 if (t1 == Type::TOP) return Type::TOP;
2419 Node *adr = in(MemNode::Address);
2420 const Type *t2 = phase->type( adr );
2421 if (t2 == Type::TOP) return Type::TOP;
2422 const TypePtr *tp = t2->is_ptr();
2423 if (TypePtr::above_centerline(tp->ptr()) ||
2424 tp->ptr() == TypePtr::Null) return Type::TOP;
2425
2426 // Return a more precise klass, if possible
2427 const TypeInstPtr *tinst = tp->isa_instptr();
2428 if (tinst != nullptr) {
2429 ciInstanceKlass* ik = tinst->instance_klass();
2430 int offset = tinst->offset();
2431 if (ik == phase->C->env()->Class_klass()
2432 && (offset == java_lang_Class::klass_offset() ||
2433 offset == java_lang_Class::array_klass_offset())) {
2434 // We are loading a special hidden field from a Class mirror object,
2435 // the field which points to the VM's Klass metaobject.
2436 ciType* t = tinst->java_mirror_type();
2437 // java_mirror_type returns non-null for compile-time Class constants.
2438 if (t != nullptr) {
2439 // constant oop => constant klass
2440 if (offset == java_lang_Class::array_klass_offset()) {
2441 if (t->is_void()) {
2442 // We cannot create a void array. Since void is a primitive type return null
2443 // klass. Users of this result need to do a null check on the returned klass.
2444 return TypePtr::NULL_PTR;
2445 }
2446 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2447 }
2448 if (!t->is_klass()) {
2449 // a primitive Class (e.g., int.class) has null for a klass field
2450 return TypePtr::NULL_PTR;
2451 }
2452 // Fold up the load of the hidden field
2453 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2454 }
2455 // non-constant mirror, so we can't tell what's going on
2456 }
2457 if (!tinst->is_loaded())
2458 return _type; // Bail out if not loaded
2459 if (offset == oopDesc::klass_offset_in_bytes()) {
2460 return tinst->as_klass_type(true);
2461 }
2462 }
2463
2464 // Check for loading klass from an array
2465 const TypeAryPtr *tary = tp->isa_aryptr();
2466 if (tary != nullptr &&
2467 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2468 return tary->as_klass_type(true);
2469 }
2470
2471 // Check for loading klass from an array klass
2472 const TypeKlassPtr *tkls = tp->isa_klassptr();
2473 if (tkls != nullptr && !StressReflectiveCode) {
2474 if (!tkls->is_loaded())
2475 return _type; // Bail out if not loaded
2476 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2477 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2478 // // Always returning precise element type is incorrect,
2479 // // e.g., element type could be object and array may contain strings
2480 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2481
2482 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2483 // according to the element type's subclassing.
2484 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2485 }
3361 }
3362 ss.print_cr("[TraceMergeStores]: with");
3363 merged_input_value->dump("\n", false, &ss);
3364 merged_store->dump("\n", false, &ss);
3365 tty->print("%s", ss.as_string());
3366 }
3367 #endif
3368
3369 //------------------------------Ideal------------------------------------------
3370 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3371 // When a store immediately follows a relevant allocation/initialization,
3372 // try to capture it into the initialization, or hoist it above.
3373 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3374 Node* p = MemNode::Ideal_common(phase, can_reshape);
3375 if (p) return (p == NodeSentinel) ? nullptr : p;
3376
3377 Node* mem = in(MemNode::Memory);
3378 Node* address = in(MemNode::Address);
3379 Node* value = in(MemNode::ValueIn);
3380 // Back-to-back stores to same address? Fold em up. Generally
3381 // unsafe if I have intervening uses.
3382 {
3383 Node* st = mem;
3384 // If Store 'st' has more than one use, we cannot fold 'st' away.
3385 // For example, 'st' might be the final state at a conditional
3386 // return. Or, 'st' might be used by some node which is live at
3387 // the same time 'st' is live, which might be unschedulable. So,
3388 // require exactly ONE user until such time as we clone 'mem' for
3389 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3390 // true).
3391 while (st->is_Store() && st->outcnt() == 1) {
3392 // Looking at a dead closed cycle of memory?
3393 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3394 assert(Opcode() == st->Opcode() ||
3395 st->Opcode() == Op_StoreVector ||
3396 Opcode() == Op_StoreVector ||
3397 st->Opcode() == Op_StoreVectorScatter ||
3398 Opcode() == Op_StoreVectorScatter ||
3399 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3400 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3401 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3402 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3403 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3404
3405 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3406 st->as_Store()->memory_size() <= this->memory_size()) {
3407 Node* use = st->raw_out(0);
3408 if (phase->is_IterGVN()) {
3409 phase->is_IterGVN()->rehash_node_delayed(use);
3410 }
3411 // It's OK to do this in the parser, since DU info is always accurate,
3412 // and the parser always refers to nodes via SafePointNode maps.
3413 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3414 return this;
3415 }
3416 st = st->in(MemNode::Memory);
3417 }
3418 }
3419
3420
3421 // Capture an unaliased, unconditional, simple store into an initializer.
3519 const StoreVectorNode* store_vector = as_StoreVector();
3520 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3521 const Node* store_indices = store_vector->indices();
3522 const Node* mem_indices = mem_vector->indices();
3523 const Node* store_mask = store_vector->mask();
3524 const Node* mem_mask = mem_vector->mask();
3525 // Ensure types, indices, and masks match
3526 if (store_vector->vect_type() == mem_vector->vect_type() &&
3527 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3528 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3529 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3530 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3531 result = mem;
3532 }
3533 }
3534 }
3535
3536 // Store of zero anywhere into a freshly-allocated object?
3537 // Then the store is useless.
3538 // (It must already have been captured by the InitializeNode.)
3539 if (result == this &&
3540 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3541 // a newly allocated object is already all-zeroes everywhere
3542 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3543 result = mem;
3544 }
3545
3546 if (result == this) {
3547 // the store may also apply to zero-bits in an earlier object
3548 Node* prev_mem = find_previous_store(phase);
3549 // Steps (a), (b): Walk past independent stores to find an exact match.
3550 if (prev_mem != nullptr) {
3551 Node* prev_val = can_see_stored_value(prev_mem, phase);
3552 if (prev_val != nullptr && prev_val == val) {
3553 // prev_val and val might differ by a cast; it would be good
3554 // to keep the more informative of the two.
3555 result = mem;
3556 }
3557 }
3558 }
3559 }
3560
3561 PhaseIterGVN* igvn = phase->is_IterGVN();
3562 if (result != this && igvn != nullptr) {
3563 MemBarNode* trailing = trailing_membar();
3564 if (trailing != nullptr) {
3565 #ifdef ASSERT
3566 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
4030 // Clearing a short array is faster with stores
4031 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4032 // Already know this is a large node, do not try to ideal it
4033 if (_is_large) return nullptr;
4034
4035 const int unit = BytesPerLong;
4036 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4037 if (!t) return nullptr;
4038 if (!t->is_con()) return nullptr;
4039 intptr_t raw_count = t->get_con();
4040 intptr_t size = raw_count;
4041 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4042 // Clearing nothing uses the Identity call.
4043 // Negative clears are possible on dead ClearArrays
4044 // (see jck test stmt114.stmt11402.val).
4045 if (size <= 0 || size % unit != 0) return nullptr;
4046 intptr_t count = size / unit;
4047 // Length too long; communicate this to matchers and assemblers.
4048 // Assemblers are responsible to produce fast hardware clears for it.
4049 if (size > InitArrayShortSize) {
4050 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4051 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4052 return nullptr;
4053 }
4054 if (!IdealizeClearArrayNode) return nullptr;
4055 Node *mem = in(1);
4056 if( phase->type(mem)==Type::TOP ) return nullptr;
4057 Node *adr = in(3);
4058 const Type* at = phase->type(adr);
4059 if( at==Type::TOP ) return nullptr;
4060 const TypePtr* atp = at->isa_ptr();
4061 // adjust atp to be the correct array element address type
4062 if (atp == nullptr) atp = TypePtr::BOTTOM;
4063 else atp = atp->add_offset(Type::OffsetBot);
4064 // Get base for derived pointer purposes
4065 if( adr->Opcode() != Op_AddP ) Unimplemented();
4066 Node *base = adr->in(1);
4067
4068 Node *zero = phase->makecon(TypeLong::ZERO);
4069 Node *off = phase->MakeConX(BytesPerLong);
4070 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4071 count--;
4072 while( count-- ) {
4073 mem = phase->transform(mem);
4074 adr = phase->transform(new AddPNode(base,adr,off));
4075 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4076 }
4077 return mem;
4078 }
4079
4080 //----------------------------step_through----------------------------------
4081 // Return allocation input memory edge if it is different instance
4082 // or itself if it is the one we are looking for.
4083 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4084 Node* n = *np;
4085 assert(n->is_ClearArray(), "sanity");
4086 intptr_t offset;
4087 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4088 // This method is called only before Allocate nodes are expanded
4089 // during macro nodes expansion. Before that ClearArray nodes are
4090 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4091 // Allocate nodes are expanded) which follows allocations.
4092 assert(alloc != nullptr, "should have allocation");
4093 if (alloc->_idx == instance_id) {
4094 // Can not bypass initialization of the instance we are looking for.
4095 return false;
4096 }
4097 // Otherwise skip it.
4098 InitializeNode* init = alloc->initialization();
4099 if (init != nullptr)
4100 *np = init->in(TypeFunc::Memory);
4101 else
4102 *np = alloc->in(TypeFunc::Memory);
4103 return true;
4104 }
4105
4106 //----------------------------clear_memory-------------------------------------
4107 // Generate code to initialize object storage to zero.
4108 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4109 intptr_t start_offset,
4110 Node* end_offset,
4111 PhaseGVN* phase) {
4112 intptr_t offset = start_offset;
4113
4114 int unit = BytesPerLong;
4115 if ((offset % unit) != 0) {
4116 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
4117 adr = phase->transform(adr);
4118 const TypePtr* atp = TypeRawPtr::BOTTOM;
4119 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4120 mem = phase->transform(mem);
4121 offset += BytesPerInt;
4122 }
4123 assert((offset % unit) == 0, "");
4124
4125 // Initialize the remaining stuff, if any, with a ClearArray.
4126 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
4127 }
4128
4129 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4130 Node* start_offset,
4131 Node* end_offset,
4132 PhaseGVN* phase) {
4133 if (start_offset == end_offset) {
4134 // nothing to do
4135 return mem;
4136 }
4137
4138 int unit = BytesPerLong;
4139 Node* zbase = start_offset;
4140 Node* zend = end_offset;
4141
4142 // Scale to the unit required by the CPU:
4143 if (!Matcher::init_array_count_is_in_bytes) {
4144 Node* shift = phase->intcon(exact_log2(unit));
4145 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4146 zend = phase->transform(new URShiftXNode(zend, shift) );
4147 }
4148
4149 // Bulk clear double-words
4150 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4151 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
4152 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4153 return phase->transform(mem);
4154 }
4155
4156 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4157 intptr_t start_offset,
4158 intptr_t end_offset,
4159 PhaseGVN* phase) {
4160 if (start_offset == end_offset) {
4161 // nothing to do
4162 return mem;
4163 }
4164
4165 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4166 intptr_t done_offset = end_offset;
4167 if ((done_offset % BytesPerLong) != 0) {
4168 done_offset -= BytesPerInt;
4169 }
4170 if (done_offset > start_offset) {
4171 mem = clear_memory(ctl, mem, dest,
4172 start_offset, phase->MakeConX(done_offset), phase);
4173 }
4174 if (done_offset < end_offset) { // emit the final 32-bit store
4175 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
4176 adr = phase->transform(adr);
4177 const TypePtr* atp = TypeRawPtr::BOTTOM;
4178 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4179 mem = phase->transform(mem);
4180 done_offset += BytesPerInt;
4181 }
4182 assert(done_offset == end_offset, "");
4183 return mem;
4184 }
4185
4186 //=============================================================================
4187 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4188 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4189 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4190 #ifdef ASSERT
4191 , _pair_idx(0)
4192 #endif
4193 {
4194 init_class_id(Class_MemBar);
4195 Node* top = C->top();
4196 init_req(TypeFunc::I_O,top);
4197 init_req(TypeFunc::FramePtr,top);
4198 init_req(TypeFunc::ReturnAdr,top);
4304 PhaseIterGVN* igvn = phase->is_IterGVN();
4305 remove(igvn);
4306 // Must return either the original node (now dead) or a new node
4307 // (Do not return a top here, since that would break the uniqueness of top.)
4308 return new ConINode(TypeInt::ZERO);
4309 }
4310 }
4311 return progress ? this : nullptr;
4312 }
4313
4314 //------------------------------Value------------------------------------------
4315 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4316 if( !in(0) ) return Type::TOP;
4317 if( phase->type(in(0)) == Type::TOP )
4318 return Type::TOP;
4319 return TypeTuple::MEMBAR;
4320 }
4321
4322 //------------------------------match------------------------------------------
4323 // Construct projections for memory.
4324 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4325 switch (proj->_con) {
4326 case TypeFunc::Control:
4327 case TypeFunc::Memory:
4328 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4329 }
4330 ShouldNotReachHere();
4331 return nullptr;
4332 }
4333
4334 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4335 trailing->_kind = TrailingStore;
4336 leading->_kind = LeadingStore;
4337 #ifdef ASSERT
4338 trailing->_pair_idx = leading->_idx;
4339 leading->_pair_idx = leading->_idx;
4340 #endif
4341 }
4342
4343 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4344 trailing->_kind = TrailingLoadStore;
4591 return (req() > RawStores);
4592 }
4593
4594 void InitializeNode::set_complete(PhaseGVN* phase) {
4595 assert(!is_complete(), "caller responsibility");
4596 _is_complete = Complete;
4597
4598 // After this node is complete, it contains a bunch of
4599 // raw-memory initializations. There is no need for
4600 // it to have anything to do with non-raw memory effects.
4601 // Therefore, tell all non-raw users to re-optimize themselves,
4602 // after skipping the memory effects of this initialization.
4603 PhaseIterGVN* igvn = phase->is_IterGVN();
4604 if (igvn) igvn->add_users_to_worklist(this);
4605 }
4606
4607 // convenience function
4608 // return false if the init contains any stores already
4609 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4610 InitializeNode* init = initialization();
4611 if (init == nullptr || init->is_complete()) return false;
4612 init->remove_extra_zeroes();
4613 // for now, if this allocation has already collected any inits, bail:
4614 if (init->is_non_zero()) return false;
4615 init->set_complete(phase);
4616 return true;
4617 }
4618
4619 void InitializeNode::remove_extra_zeroes() {
4620 if (req() == RawStores) return;
4621 Node* zmem = zero_memory();
4622 uint fill = RawStores;
4623 for (uint i = fill; i < req(); i++) {
4624 Node* n = in(i);
4625 if (n->is_top() || n == zmem) continue; // skip
4626 if (fill < i) set_req(fill, n); // compact
4627 ++fill;
4628 }
4629 // delete any empty spaces created:
4630 while (fill < req()) {
4631 del_req(fill);
4775 // store node that we'd like to capture. We need to check
4776 // the uses of the MergeMemNode.
4777 mems.push(n);
4778 }
4779 } else if (n->is_Mem()) {
4780 Node* other_adr = n->in(MemNode::Address);
4781 if (other_adr == adr) {
4782 failed = true;
4783 break;
4784 } else {
4785 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4786 if (other_t_adr != nullptr) {
4787 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4788 if (other_alias_idx == alias_idx) {
4789 // A load from the same memory slice as the store right
4790 // after the InitializeNode. We check the control of the
4791 // object/array that is loaded from. If it's the same as
4792 // the store control then we cannot capture the store.
4793 assert(!n->is_Store(), "2 stores to same slice on same control?");
4794 Node* base = other_adr;
4795 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4796 base = base->in(AddPNode::Base);
4797 if (base != nullptr) {
4798 base = base->uncast();
4799 if (base->is_Proj() && base->in(0) == alloc) {
4800 failed = true;
4801 break;
4802 }
4803 }
4804 }
4805 }
4806 }
4807 } else {
4808 failed = true;
4809 break;
4810 }
4811 }
4812 }
4813 }
4814 if (failed) {
5361 // z's_done 12 16 16 16 12 16 12
5362 // z's_needed 12 16 16 16 16 16 16
5363 // zsize 0 0 0 0 4 0 4
5364 if (next_full_store < 0) {
5365 // Conservative tack: Zero to end of current word.
5366 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5367 } else {
5368 // Zero to beginning of next fully initialized word.
5369 // Or, don't zero at all, if we are already in that word.
5370 assert(next_full_store >= zeroes_needed, "must go forward");
5371 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5372 zeroes_needed = next_full_store;
5373 }
5374 }
5375
5376 if (zeroes_needed > zeroes_done) {
5377 intptr_t zsize = zeroes_needed - zeroes_done;
5378 // Do some incremental zeroing on rawmem, in parallel with inits.
5379 zeroes_done = align_down(zeroes_done, BytesPerInt);
5380 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5381 zeroes_done, zeroes_needed,
5382 phase);
5383 zeroes_done = zeroes_needed;
5384 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5385 do_zeroing = false; // leave the hole, next time
5386 }
5387 }
5388
5389 // Collect the store and move on:
5390 phase->replace_input_of(st, MemNode::Memory, inits);
5391 inits = st; // put it on the linearized chain
5392 set_req(i, zmem); // unhook from previous position
5393
5394 if (zeroes_done == st_off)
5395 zeroes_done = next_init_off;
5396
5397 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5398
5399 #ifdef ASSERT
5400 // Various order invariants. Weaker than stores_are_sane because
5420 remove_extra_zeroes(); // clear out all the zmems left over
5421 add_req(inits);
5422
5423 if (!(UseTLAB && ZeroTLAB)) {
5424 // If anything remains to be zeroed, zero it all now.
5425 zeroes_done = align_down(zeroes_done, BytesPerInt);
5426 // if it is the last unused 4 bytes of an instance, forget about it
5427 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5428 if (zeroes_done + BytesPerLong >= size_limit) {
5429 AllocateNode* alloc = allocation();
5430 assert(alloc != nullptr, "must be present");
5431 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5432 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5433 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5434 if (zeroes_done == k->layout_helper())
5435 zeroes_done = size_limit;
5436 }
5437 }
5438 if (zeroes_done < size_limit) {
5439 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5440 zeroes_done, size_in_bytes, phase);
5441 }
5442 }
5443
5444 set_complete(phase);
5445 return rawmem;
5446 }
5447
5448
5449 #ifdef ASSERT
5450 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5451 if (is_complete())
5452 return true; // stores could be anything at this point
5453 assert(allocation() != nullptr, "must be present");
5454 intptr_t last_off = allocation()->minimum_header_size();
5455 for (uint i = InitializeNode::RawStores; i < req(); i++) {
5456 Node* st = in(i);
5457 intptr_t st_off = get_store_offset(st, phase);
5458 if (st_off < 0) continue; // ignore dead garbage
5459 if (last_off > st_off) {
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "compiler/compileLog.hpp"
30 #include "gc/shared/barrierSet.hpp"
31 #include "gc/shared/c2/barrierSetC2.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "memory/allocation.inline.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "oops/objArrayKlass.hpp"
36 #include "opto/addnode.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/cfgnode.hpp"
39 #include "opto/regalloc.hpp"
40 #include "opto/compile.hpp"
41 #include "opto/connode.hpp"
42 #include "opto/convertnode.hpp"
43 #include "opto/inlinetypenode.hpp"
44 #include "opto/loopnode.hpp"
45 #include "opto/machnode.hpp"
46 #include "opto/matcher.hpp"
47 #include "opto/memnode.hpp"
48 #include "opto/mempointer.hpp"
49 #include "opto/mulnode.hpp"
50 #include "opto/narrowptrnode.hpp"
51 #include "opto/phaseX.hpp"
52 #include "opto/regmask.hpp"
53 #include "opto/rootnode.hpp"
54 #include "opto/traceMergeStoresTag.hpp"
55 #include "opto/vectornode.hpp"
56 #include "utilities/align.hpp"
57 #include "utilities/copy.hpp"
58 #include "utilities/macros.hpp"
59 #include "utilities/powerOfTwo.hpp"
60 #include "utilities/vmError.hpp"
61
62 // Portions of code courtesy of Clifford Click
63
126 st->print(", idx=Bot;");
127 else if (atp->index() == Compile::AliasIdxTop)
128 st->print(", idx=Top;");
129 else if (atp->index() == Compile::AliasIdxRaw)
130 st->print(", idx=Raw;");
131 else {
132 ciField* field = atp->field();
133 if (field) {
134 st->print(", name=");
135 field->print_name_on(st);
136 }
137 st->print(", idx=%d;", atp->index());
138 }
139 }
140 }
141
142 extern void print_alias_types();
143
144 #endif
145
146 // If call is a constructor call on receiver, returns the class which declares the target method,
147 // else returns nullptr. This information can then be used to deduce if call modifies a field of
148 // receiver. Specifically, if the field is declared in a class that is a subclass of the one
149 // declaring the constructor, then the field is set inside the constructor, else the field must be
150 // set before the constructor invocation. E.g. A field Super.x will be set during the execution of
151 // Sub::<init>, while a field Sub.y must be set before Super::<init> is invoked.
152 static ciInstanceKlass* find_constructor_call_method_holder(Node* call, Node* receiver) {
153 if (!call->is_CallJava()) {
154 return nullptr;
155 }
156
157 ciMethod* target = call->as_CallJava()->method();
158 if (target == nullptr || !target->is_object_constructor()) {
159 return nullptr;
160 }
161
162 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
163 Node* parm = call->in(TypeFunc::Parms)->uncast();
164 receiver = receiver->uncast();
165 if (parm == receiver || (parm->is_InlineType() && parm->as_InlineType()->get_oop()->uncast() == receiver)) {
166 return target->holder();
167 }
168
169 return nullptr;
170 }
171
172 // Find the memory output corresponding to the fall-through path of a call
173 static Node* find_call_fallthrough_mem_output(CallNode* call) {
174 ResourceMark rm;
175 CallProjections* projs = call->extract_projections(false, false);
176 Node* res = projs->fallthrough_memproj;
177 assert(res != nullptr, "must have a fallthrough mem output");
178 return res;
179 }
180
181 // Try to find a better memory input for a load from a strict final field of an object that is
182 // allocated in the current compilation unit, or is the first parameter when we are in a
183 // constructor
184 static Node* optimize_strict_final_load_memory_from_local_object(ciField* field, ProjNode* base_uncasted) {
185 if (!EnableValhalla) {
186 // In this method we depends on the fact that strict fields are set before the invocation of
187 // super(), I'm not sure if this is true without Valhalla
188 return nullptr;
189 }
190
191 // The node that can be passed into a constructor
192 Node* base = base_uncasted;
193 if (!base_uncasted->is_Parm()) {
194 assert(base_uncasted->_con == AllocateNode::RawAddress && base_uncasted->in(0)->is_Allocate(), "must be the RawAddress of an AllocateNode");
195 base = base_uncasted->in(0)->as_Allocate()->result_cast();
196 assert(base != nullptr && base->in(1) == base_uncasted, "must find a valid base");
197 }
198
199 // Try to see if there is a constructor call on the base
200 for (DUIterator_Fast imax, i = base->fast_outs(imax); i < imax; i++) {
201 Node* out = base->fast_out(i);
202 ciInstanceKlass* target_holder = find_constructor_call_method_holder(out, base);
203 if (target_holder == nullptr) {
204 continue;
205 } else if (target_holder->is_subclass_of(field->holder())) {
206 return find_call_fallthrough_mem_output(out->as_CallJava());
207 } else {
208 Node* res = out->in(TypeFunc::Memory);
209 assert(res != nullptr, "should have a memory input");
210 return res;
211 }
212 }
213
214 return nullptr;
215 }
216
217 // Try to find a better memory input for a load from a strict final field
218 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, ciField* field, Node* adr, ProjNode*& base_local) {
219 intptr_t offset = 0;
220 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
221 if (base == nullptr) {
222 return nullptr;
223 }
224
225 Node* base_uncasted = base->uncast();
226 if (base_uncasted->is_Proj()) {
227 MultiNode* multi = base_uncasted->in(0)->as_Multi();
228 if (multi->is_Allocate()) {
229 // The result of an AllocateNode, try to find the constructor call
230 base_local = base_uncasted->as_Proj();
231 return optimize_strict_final_load_memory_from_local_object(field, base_uncasted->as_Proj());
232 } else if (multi->is_Call()) {
233 // The oop is returned from a call, the memory can be the fallthrough output of the call
234 return find_call_fallthrough_mem_output(multi->as_Call());
235 } else if (multi->is_Start()) {
236 // The oop is a parameter
237 if (phase->C->method()->is_object_constructor() && base_uncasted->as_Proj()->_con == TypeFunc::Parms) {
238 // The receiver of a constructor is similar to the result of an AllocateNode
239 base_local = base_uncasted->as_Proj();
240 return optimize_strict_final_load_memory_from_local_object(field, base_uncasted->as_Proj());
241 } else {
242 // Use the start memory otherwise
243 return multi->proj_out(TypeFunc::Memory);
244 }
245 }
246 }
247
248 return nullptr;
249 }
250
251 // Whether a call can modify a strict final field of base_local, given that base_local is allocated
252 // inside the current compilation unit, or is the first parameter when the compilation root is a
253 // constructor. This is equivalent to asking whether base_local is the receiver of the constructor
254 // invocation call and the class declaring the target method is a subclass of the class declaring
255 // field.
256 static bool call_can_modify_local_object(ciField* field, CallNode* call, Node* base_local) {
257 ciInstanceKlass* target_holder = find_constructor_call_method_holder(call, base_local);
258 return target_holder != nullptr && target_holder->is_subclass_of(field->holder());
259 }
260
261 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
262 assert(t_oop != nullptr, "sanity");
263 bool is_instance = t_oop->is_known_instance_field();
264
265 ciField* field = phase->C->alias_type(t_oop)->field();
266 bool is_strict_final_load = false;
267
268 // After macro expansion, an allocation may become a call, changing the memory input to the
269 // memory output of that call would be illegal. As a result, disallow this transformation after
270 // macro expansion.
271 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
272 if (EnableValhalla) {
273 if (field != nullptr && (field->holder()->is_inlinetype() || field->holder()->is_abstract_value_klass())) {
274 is_strict_final_load = true;
275 }
276 #ifdef ASSERT
277 if (t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) {
278 assert(is_strict_final_load, "sanity check for basic cases");
279 }
280 #endif
281 } else {
282 is_strict_final_load = field != nullptr && t_oop->is_ptr_to_boxed_value();
283 }
284 }
285
286 if (!is_instance && !is_strict_final_load) {
287 return mchain;
288 }
289
290 Node* result = mchain;
291 ProjNode* base_local = nullptr;
292
293 if (is_strict_final_load) {
294 Node* adr = load->in(MemNode::Address);
295 assert(phase->type(adr) == t_oop, "inconsistent type");
296 Node* tmp = try_optimize_strict_final_load_memory(phase, field, adr, base_local);
297 if (tmp != nullptr) {
298 result = tmp;
299 }
300 }
301
302 uint instance_id = t_oop->instance_id();
303 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
304 Node* prev = nullptr;
305 while (prev != result) {
306 prev = result;
307 if (result == start_mem) {
308 // start_mem is the earliest memory possible
309 break;
310 }
311
312 // skip over a call which does not affect this memory slice
313 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
314 Node* proj_in = result->in(0);
315 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
316 // This is the allocation that creates the object from which we are loading from
317 break;
318 } else if (proj_in->is_Call()) {
319 // ArrayCopyNodes processed here as well
320 CallNode* call = proj_in->as_Call();
321 if (!call->may_modify(t_oop, phase)) {
322 result = call->in(TypeFunc::Memory);
323 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call, base_local)) {
324 result = call->in(TypeFunc::Memory);
325 }
326 } else if (proj_in->is_Initialize()) {
327 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
328 // Stop if this is the initialization for the object instance which
329 // contains this memory slice, otherwise skip over it.
330 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
331 break;
332 }
333 if (is_instance) {
334 result = proj_in->in(TypeFunc::Memory);
335 } else if (is_strict_final_load) {
336 Node* klass = alloc->in(AllocateNode::KlassNode);
337 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
338 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
339 // Allocation of another type, must be another object
340 result = proj_in->in(TypeFunc::Memory);
341 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
342 // Allocation of another object
343 result = proj_in->in(TypeFunc::Memory);
344 }
345 }
346 } else if (proj_in->is_MemBar()) {
347 ArrayCopyNode* ac = nullptr;
348 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
349 break;
350 }
351 result = proj_in->in(TypeFunc::Memory);
352 } else if (proj_in->is_top()) {
353 break; // dead code
354 } else {
355 assert(false, "unexpected projection");
356 }
357 } else if (result->is_ClearArray()) {
358 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
359 // Can not bypass initialization of the instance
360 // we are looking for.
361 break;
362 }
363 // Otherwise skip it (the call updated 'result' value).
376 bool is_instance = t_oop->is_known_instance_field();
377 PhaseIterGVN *igvn = phase->is_IterGVN();
378 if (is_instance && igvn != nullptr && result->is_Phi()) {
379 PhiNode *mphi = result->as_Phi();
380 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
381 const TypePtr *t = mphi->adr_type();
382 bool do_split = false;
383 // In the following cases, Load memory input can be further optimized based on
384 // its precise address type
385 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
386 do_split = true;
387 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
388 const TypeOopPtr* mem_t =
389 t->is_oopptr()->cast_to_exactness(true)
390 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
391 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
392 if (t_oop->isa_aryptr()) {
393 mem_t = mem_t->is_aryptr()
394 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
395 ->cast_to_size(t_oop->is_aryptr()->size())
396 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
397 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
398 ->with_offset(t_oop->is_aryptr()->offset())
399 ->is_aryptr();
400 }
401 do_split = mem_t == t_oop;
402 }
403 if (do_split) {
404 // clone the Phi with our address type
405 result = mphi->split_out_instance(t_adr, igvn);
406 } else {
407 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
408 }
409 }
410 return result;
411 }
412
413 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
414 uint alias_idx = phase->C->get_alias_index(tp);
415 Node *mem = mmem;
416 #ifdef ASSERT
417 {
418 // Check that current type is consistent with the alias index used during graph construction
419 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
420 bool consistent = adr_check == nullptr || adr_check->empty() ||
421 phase->C->must_alias(adr_check, alias_idx );
422 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
423 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
424 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
425 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
426 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
427 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
428 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
429 // don't assert if it is dead code.
430 consistent = true;
431 }
432 if( !consistent ) {
433 st->print("alias_idx==%d, adr_check==", alias_idx);
434 if( adr_check == nullptr ) {
435 st->print("null");
436 } else {
437 adr_check->dump();
438 }
439 st->cr();
440 print_alias_types();
441 assert(consistent, "adr_check must match alias idx");
442 }
443 }
444 #endif
1157 Node* ld = gvn.transform(load);
1158 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1159 }
1160
1161 return load;
1162 }
1163
1164 //------------------------------hash-------------------------------------------
1165 uint LoadNode::hash() const {
1166 // unroll addition of interesting fields
1167 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1168 }
1169
1170 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1171 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1172 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1173 bool is_stable_ary = FoldStableValues &&
1174 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1175 tp->isa_aryptr()->is_stable();
1176
1177 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1178 }
1179
1180 return false;
1181 }
1182
1183 LoadNode* LoadNode::pin_array_access_node() const {
1184 const TypePtr* adr_type = this->adr_type();
1185 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1186 return clone_pinned();
1187 }
1188 return nullptr;
1189 }
1190
1191 // Is the value loaded previously stored by an arraycopy? If so return
1192 // a load node that reads from the source array so we may be able to
1193 // optimize out the ArrayCopy node later.
1194 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1195 Node* ld_adr = in(MemNode::Address);
1196 intptr_t ld_off = 0;
1197 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1214 assert(ld_alloc != nullptr, "need an alloc");
1215 assert(addp->is_AddP(), "address must be addp");
1216 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1217 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1218 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1219 addp->set_req(AddPNode::Base, src);
1220 addp->set_req(AddPNode::Address, src);
1221 } else {
1222 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1223 ac->as_ArrayCopy()->is_copyof_validated() ||
1224 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1225 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1226 addp->set_req(AddPNode::Base, src);
1227 addp->set_req(AddPNode::Address, src);
1228
1229 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1230 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1231 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1232
1233 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1234 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1235
1236 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1237 #ifdef _LP64
1238 diff = phase->transform(new ConvI2LNode(diff));
1239 #endif
1240 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1241
1242 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1243 addp->set_req(AddPNode::Offset, offset);
1244 }
1245 addp = phase->transform(addp);
1246 #ifdef ASSERT
1247 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1248 ld->_adr_type = adr_type;
1249 #endif
1250 ld->set_req(MemNode::Address, addp);
1251 ld->set_req(0, ctl);
1252 ld->set_req(MemNode::Memory, mem);
1253 return ld;
1254 }
1255 return nullptr;
1256 }
1257
1258 static Node* see_through_inline_type(PhaseValues* phase, const MemNode* load, Node* base, int offset) {
1259 if (!load->is_mismatched_access() && base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1260 InlineTypeNode* vt = base->as_InlineType();
1261 assert(!vt->is_larval(), "must not load from a larval object");
1262 Node* value = vt->field_value_by_offset(offset, true);
1263 assert(value != nullptr, "must see some value");
1264 return value;
1265 }
1266
1267 return nullptr;
1268 }
1269
1270 //---------------------------can_see_stored_value------------------------------
1271 // This routine exists to make sure this set of tests is done the same
1272 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1273 // will change the graph shape in a way which makes memory alive twice at the
1274 // same time (uses the Oracle model of aliasing), then some
1275 // LoadXNode::Identity will fold things back to the equivalence-class model
1276 // of aliasing.
1277 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1278 Node* ld_adr = in(MemNode::Address);
1279 intptr_t ld_off = 0;
1280 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1281 // Try to see through an InlineTypeNode
1282 // LoadN is special because the input is not compressed
1283 if (Opcode() != Op_LoadN) {
1284 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1285 if (value != nullptr) {
1286 return value;
1287 }
1288 }
1289
1290 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1291 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1292 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1293 // This is more general than load from boxing objects.
1294 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1295 uint alias_idx = atp->index();
1296 Node* result = nullptr;
1297 Node* current = st;
1298 // Skip through chains of MemBarNodes checking the MergeMems for
1299 // new states for the slice of this load. Stop once any other
1300 // kind of node is encountered. Loads from final memory can skip
1301 // through any kind of MemBar but normal loads shouldn't skip
1302 // through MemBarAcquire since the could allow them to move out of
1303 // a synchronized region. It is not safe to step over MemBarCPUOrder,
1304 // because alias info above them may be inaccurate (e.g., due to
1305 // mixed/mismatched unsafe accesses).
1306 bool is_final_mem = !atp->is_rewritable();
1307 while (current->is_Proj()) {
1308 int opc = current->in(0)->Opcode();
1309 if ((is_final_mem && (opc == Op_MemBarAcquire ||
1353 // Same base, same offset.
1354 // Possible improvement for arrays: check index value instead of absolute offset.
1355
1356 // At this point we have proven something like this setup:
1357 // B = << base >>
1358 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1359 // S = StoreQ(AddP( B , #Off), V)
1360 // (Actually, we haven't yet proven the Q's are the same.)
1361 // In other words, we are loading from a casted version of
1362 // the same pointer-and-offset that we stored to.
1363 // Casted version may carry a dependency and it is respected.
1364 // Thus, we are able to replace L by V.
1365 }
1366 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1367 if (store_Opcode() != st->Opcode()) {
1368 return nullptr;
1369 }
1370 // LoadVector/StoreVector needs additional check to ensure the types match.
1371 if (st->is_StoreVector()) {
1372 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1373 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1374 if (in_vt != out_vt) {
1375 return nullptr;
1376 }
1377 }
1378 return st->in(MemNode::ValueIn);
1379 }
1380
1381 // A load from a freshly-created object always returns zero.
1382 // (This can happen after LoadNode::Ideal resets the load's memory input
1383 // to find_captured_store, which returned InitializeNode::zero_memory.)
1384 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1385 (st->in(0) == ld_alloc) &&
1386 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1387 // return a zero value for the load's basic type
1388 // (This is one of the few places where a generic PhaseTransform
1389 // can create new nodes. Think of it as lazily manifesting
1390 // virtually pre-existing constants.)
1391 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1392 if (init_value != nullptr) {
1393 // TODO 8350865 Is this correct for non-all-zero init values? Don't we need field_value_by_offset?
1394 return init_value;
1395 }
1396 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1397 if (memory_type() != T_VOID) {
1398 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1399 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1400 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1401 // by the ArrayCopyNode.
1402 return phase->zerocon(memory_type());
1403 }
1404 } else {
1405 // TODO: materialize all-zero vector constant
1406 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1407 }
1408 }
1409
1410 // A load from an initialization barrier can match a captured store.
1411 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1412 InitializeNode* init = st->in(0)->as_Initialize();
1413 AllocateNode* alloc = init->allocation();
1414 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1415 // examine a captured store value
1416 st = init->find_captured_store(ld_off, memory_size(), phase);
2037 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2038 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2039
2040 // Skip up past a SafePoint control. Cannot do this for Stores because
2041 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2042 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2043 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2044 !addr_mark &&
2045 (depends_only_on_test() || has_unknown_control_dependency())) {
2046 ctrl = ctrl->in(0);
2047 set_req(MemNode::Control,ctrl);
2048 progress = true;
2049 }
2050
2051 intptr_t ignore = 0;
2052 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2053 if (base != nullptr
2054 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2055 // Check for useless control edge in some common special cases
2056 if (in(MemNode::Control) != nullptr
2057 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2058 && can_remove_control()
2059 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2060 && all_controls_dominate(base, phase->C->start())) {
2061 // A method-invariant, non-null address (constant or 'this' argument).
2062 set_req(MemNode::Control, nullptr);
2063 progress = true;
2064 }
2065 }
2066
2067 Node* mem = in(MemNode::Memory);
2068 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2069
2070 if (can_reshape && (addr_t != nullptr)) {
2071 // try to optimize our memory input
2072 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2073 if (opt_mem != mem) {
2074 set_req_X(MemNode::Memory, opt_mem, phase);
2075 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2076 return this;
2077 }
2134 // fold up, do so.
2135 Node* prev_mem = find_previous_store(phase);
2136 if (prev_mem != nullptr) {
2137 Node* value = can_see_arraycopy_value(prev_mem, phase);
2138 if (value != nullptr) {
2139 return value;
2140 }
2141 }
2142 // Steps (a), (b): Walk past independent stores to find an exact match.
2143 if (prev_mem != nullptr && prev_mem != in(MemNode::Memory)) {
2144 // (c) See if we can fold up on the spot, but don't fold up here.
2145 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
2146 // just return a prior value, which is done by Identity calls.
2147 if (can_see_stored_value(prev_mem, phase)) {
2148 // Make ready for step (d):
2149 set_req_X(MemNode::Memory, prev_mem, phase);
2150 return this;
2151 }
2152 }
2153
2154 if (progress) {
2155 return this;
2156 }
2157
2158 if (!can_reshape) {
2159 phase->record_for_igvn(this);
2160 }
2161 return nullptr;
2162 }
2163
2164 // Helper to recognize certain Klass fields which are invariant across
2165 // some group of array types (e.g., int[] or all T[] where T < Object).
2166 const Type*
2167 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
2168 ciKlass* klass) const {
2169 assert(!UseCompactObjectHeaders || tkls->offset() != in_bytes(Klass::prototype_header_offset()),
2170 "must not happen");
2171 if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
2172 // The field is Klass::_access_flags. Return its (constant) value.
2173 // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
2174 assert(Opcode() == Op_LoadUS, "must load an unsigned short from _access_flags");
2175 return TypeInt::make(klass->access_flags());
2176 }
2177 if (tkls->offset() == in_bytes(Klass::misc_flags_offset())) {
2178 // The field is Klass::_misc_flags. Return its (constant) value.
2179 // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
2180 assert(Opcode() == Op_LoadUB, "must load an unsigned byte from _misc_flags");
2181 return TypeInt::make(klass->misc_flags());
2243 }
2244 }
2245
2246 // Don't do this for integer types. There is only potential profit if
2247 // the element type t is lower than _type; that is, for int types, if _type is
2248 // more restrictive than t. This only happens here if one is short and the other
2249 // char (both 16 bits), and in those cases we've made an intentional decision
2250 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2251 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2252 //
2253 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2254 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2255 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2256 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2257 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2258 // In fact, that could have been the original type of p1, and p1 could have
2259 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2260 // expression (LShiftL quux 3) independently optimized to the constant 8.
2261 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2262 && (_type->isa_vect() == nullptr)
2263 && !ary->is_flat()
2264 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2265 // t might actually be lower than _type, if _type is a unique
2266 // concrete subclass of abstract class t.
2267 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2268 const Type* jt = t->join_speculative(_type);
2269 // In any case, do not allow the join, per se, to empty out the type.
2270 if (jt->empty() && !t->empty()) {
2271 // This can happen if a interface-typed array narrows to a class type.
2272 jt = _type;
2273 }
2274 #ifdef ASSERT
2275 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2276 // The pointers in the autobox arrays are always non-null
2277 Node* base = adr->in(AddPNode::Base);
2278 if ((base != nullptr) && base->is_DecodeN()) {
2279 // Get LoadN node which loads IntegerCache.cache field
2280 base = base->in(1);
2281 }
2282 if ((base != nullptr) && base->is_Con()) {
2283 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2284 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2285 // It could be narrow oop
2286 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2287 }
2288 }
2289 }
2290 #endif
2291 return jt;
2292 }
2293 }
2294 } else if (tp->base() == Type::InstPtr) {
2295 assert( off != Type::OffsetBot ||
2296 // arrays can be cast to Objects
2297 !tp->isa_instptr() ||
2298 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2299 // Default value load
2300 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2301 // unsafe field access may not have a constant offset
2302 C->has_unsafe_access(),
2303 "Field accesses must be precise" );
2304 // For oop loads, we expect the _type to be precise.
2305
2306 const TypeInstPtr* tinst = tp->is_instptr();
2307 BasicType bt = memory_type();
2308
2309 // Optimize loads from constant fields.
2310 ciObject* const_oop = tinst->const_oop();
2311 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2312 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2313 if (con_type != nullptr) {
2314 return con_type;
2315 }
2316 }
2317 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2318 assert(off != Type::OffsetBot ||
2319 !tp->isa_instklassptr() ||
2320 // arrays can be cast to Objects
2321 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2322 // also allow array-loading from the primary supertype
2323 // array during subtype checks
2324 Opcode() == Op_LoadKlass,
2325 "Field accesses must be precise");
2326 // For klass/static loads, we expect the _type to be precise
2327 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2328 /* With mirrors being an indirect in the Klass*
2329 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2330 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2331 *
2332 * So check the type and klass of the node before the LoadP.
2339 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2340 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2341 return TypeInstPtr::make(klass->java_mirror());
2342 }
2343 }
2344 }
2345
2346 const TypeKlassPtr *tkls = tp->isa_klassptr();
2347 if (tkls != nullptr) {
2348 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2349 ciKlass* klass = tkls->exact_klass();
2350 // We are loading a field from a Klass metaobject whose identity
2351 // is known at compile time (the type is "exact" or "precise").
2352 // Check for fields we know are maintained as constants by the VM.
2353 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2354 // The field is Klass::_super_check_offset. Return its (constant) value.
2355 // (Folds up type checking code.)
2356 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2357 return TypeInt::make(klass->super_check_offset());
2358 }
2359 if (UseCompactObjectHeaders) { // TODO: Should EnableValhalla also take this path ?
2360 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2361 // The field is Klass::_prototype_header. Return its (constant) value.
2362 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2363 return TypeX::make(klass->prototype_header());
2364 }
2365 }
2366 // Compute index into primary_supers array
2367 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2368 // Check for overflowing; use unsigned compare to handle the negative case.
2369 if( depth < ciKlass::primary_super_limit() ) {
2370 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2371 // (Folds up type checking code.)
2372 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2373 ciKlass *ss = klass->super_of_depth(depth);
2374 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2375 }
2376 const Type* aift = load_array_final_field(tkls, klass);
2377 if (aift != nullptr) return aift;
2378 }
2379
2417 // Note: When interfaces are reliable, we can narrow the interface
2418 // test to (klass != Serializable && klass != Cloneable).
2419 assert(Opcode() == Op_LoadI, "must load an int from _layout_helper");
2420 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2421 // The key property of this type is that it folds up tests
2422 // for array-ness, since it proves that the layout_helper is positive.
2423 // Thus, a generic value like the basic object layout helper works fine.
2424 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2425 }
2426 }
2427
2428 bool is_vect = (_type->isa_vect() != nullptr);
2429 if (is_instance && !is_vect) {
2430 // If we have an instance type and our memory input is the
2431 // programs's initial memory state, there is no matching store,
2432 // so just return a zero of the appropriate type -
2433 // except if it is vectorized - then we have no zero constant.
2434 Node *mem = in(MemNode::Memory);
2435 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2436 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2437 // TODO 8350865 This is needed for flat array accesses, somehow the memory of the loads bypasses the intrinsic
2438 // Run TestArrays.test6 in Scenario4, we need more tests for this. TestBasicFunctionality::test20 also needs this.
2439 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && !UseFieldFlattening) {
2440 return _type;
2441 }
2442 return Type::get_zero_type(_type->basic_type());
2443 }
2444 }
2445 if (!UseCompactObjectHeaders) {
2446 Node* alloc = is_new_object_mark_load();
2447 if (alloc != nullptr) {
2448 if (EnableValhalla) {
2449 // The mark word may contain property bits (inline, flat, null-free)
2450 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2451 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2452 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2453 return TypeX::make(tkls->exact_klass()->prototype_header());
2454 }
2455 } else {
2456 return TypeX::make(markWord::prototype().value());
2457 }
2458 }
2459 }
2460
2461 return _type;
2462 }
2463
2464 //------------------------------match_edge-------------------------------------
2465 // Do we Match on this edge index or not? Match only the address.
2466 uint LoadNode::match_edge(uint idx) const {
2467 return idx == MemNode::Address;
2468 }
2469
2470 //--------------------------LoadBNode::Ideal--------------------------------------
2471 //
2472 // If the previous store is to the same address as this load,
2473 // and the value stored was larger than a byte, replace this load
2474 // with the value stored truncated to a byte. If no truncation is
2475 // needed, the replacement is done in LoadNode::Identity().
2476 //
2477 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2586 }
2587 }
2588 // Identity call will handle the case where truncation is not needed.
2589 return LoadNode::Ideal(phase, can_reshape);
2590 }
2591
2592 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2593 Node* mem = in(MemNode::Memory);
2594 Node* value = can_see_stored_value(mem,phase);
2595 if (value != nullptr && value->is_Con() &&
2596 !value->bottom_type()->higher_equal(_type)) {
2597 // If the input to the store does not fit with the load's result type,
2598 // it must be truncated. We can't delay until Ideal call since
2599 // a singleton Value is needed for split_thru_phi optimization.
2600 int con = value->get_int();
2601 return TypeInt::make((con << 16) >> 16);
2602 }
2603 return LoadNode::Value(phase);
2604 }
2605
2606 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2607 // Loading from an InlineType, find the input and make an EncodeP
2608 Node* addr = in(Address);
2609 intptr_t offset;
2610 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
2611 Node* value = see_through_inline_type(phase, this, base, offset);
2612 if (value != nullptr) {
2613 return new EncodePNode(value, type());
2614 }
2615
2616 return LoadNode::Ideal(phase, can_reshape);
2617 }
2618
2619 //=============================================================================
2620 //----------------------------LoadKlassNode::make------------------------------
2621 // Polymorphic factory method:
2622 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2623 // sanity check the alias category against the created node type
2624 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2625 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2626 #ifdef _LP64
2627 if (adr_type->is_ptr_to_narrowklass()) {
2628 assert(UseCompressedClassPointers, "no compressed klasses");
2629 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2630 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2631 }
2632 #endif
2633 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2634 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2635 }
2636
2637 //------------------------------Value------------------------------------------
2638 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2643 // Either input is TOP ==> the result is TOP
2644 const Type *t1 = phase->type( in(MemNode::Memory) );
2645 if (t1 == Type::TOP) return Type::TOP;
2646 Node *adr = in(MemNode::Address);
2647 const Type *t2 = phase->type( adr );
2648 if (t2 == Type::TOP) return Type::TOP;
2649 const TypePtr *tp = t2->is_ptr();
2650 if (TypePtr::above_centerline(tp->ptr()) ||
2651 tp->ptr() == TypePtr::Null) return Type::TOP;
2652
2653 // Return a more precise klass, if possible
2654 const TypeInstPtr *tinst = tp->isa_instptr();
2655 if (tinst != nullptr) {
2656 ciInstanceKlass* ik = tinst->instance_klass();
2657 int offset = tinst->offset();
2658 if (ik == phase->C->env()->Class_klass()
2659 && (offset == java_lang_Class::klass_offset() ||
2660 offset == java_lang_Class::array_klass_offset())) {
2661 // We are loading a special hidden field from a Class mirror object,
2662 // the field which points to the VM's Klass metaobject.
2663 bool is_null_free_array = false;
2664 ciType* t = tinst->java_mirror_type(&is_null_free_array);
2665 // java_mirror_type returns non-null for compile-time Class constants.
2666 if (t != nullptr) {
2667 // constant oop => constant klass
2668 if (offset == java_lang_Class::array_klass_offset()) {
2669 if (t->is_void()) {
2670 // We cannot create a void array. Since void is a primitive type return null
2671 // klass. Users of this result need to do a null check on the returned klass.
2672 return TypePtr::NULL_PTR;
2673 }
2674 const TypeKlassPtr* tklass = TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2675 if (is_null_free_array) {
2676 tklass = tklass->is_aryklassptr()->cast_to_null_free();
2677 }
2678 return tklass;
2679 }
2680 if (!t->is_klass()) {
2681 // a primitive Class (e.g., int.class) has null for a klass field
2682 return TypePtr::NULL_PTR;
2683 }
2684 // Fold up the load of the hidden field
2685 const TypeKlassPtr* tklass = TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2686 if (is_null_free_array) {
2687 tklass = tklass->is_aryklassptr()->cast_to_null_free();
2688 }
2689 return tklass;
2690 }
2691 // non-constant mirror, so we can't tell what's going on
2692 }
2693 if (!tinst->is_loaded())
2694 return _type; // Bail out if not loaded
2695 if (offset == oopDesc::klass_offset_in_bytes()) {
2696 return tinst->as_klass_type(true);
2697 }
2698 }
2699
2700 // Check for loading klass from an array
2701 const TypeAryPtr* tary = tp->isa_aryptr();
2702 if (tary != nullptr &&
2703 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2704 return tary->as_klass_type(true);
2705 }
2706
2707 // Check for loading klass from an array klass
2708 const TypeKlassPtr *tkls = tp->isa_klassptr();
2709 if (tkls != nullptr && !StressReflectiveCode) {
2710 if (!tkls->is_loaded())
2711 return _type; // Bail out if not loaded
2712 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2713 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2714 // // Always returning precise element type is incorrect,
2715 // // e.g., element type could be object and array may contain strings
2716 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2717
2718 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2719 // according to the element type's subclassing.
2720 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2721 }
3597 }
3598 ss.print_cr("[TraceMergeStores]: with");
3599 merged_input_value->dump("\n", false, &ss);
3600 merged_store->dump("\n", false, &ss);
3601 tty->print("%s", ss.as_string());
3602 }
3603 #endif
3604
3605 //------------------------------Ideal------------------------------------------
3606 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3607 // When a store immediately follows a relevant allocation/initialization,
3608 // try to capture it into the initialization, or hoist it above.
3609 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3610 Node* p = MemNode::Ideal_common(phase, can_reshape);
3611 if (p) return (p == NodeSentinel) ? nullptr : p;
3612
3613 Node* mem = in(MemNode::Memory);
3614 Node* address = in(MemNode::Address);
3615 Node* value = in(MemNode::ValueIn);
3616 // Back-to-back stores to same address? Fold em up. Generally
3617 // unsafe if I have intervening uses...
3618 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3619 Node* st = mem;
3620 // If Store 'st' has more than one use, we cannot fold 'st' away.
3621 // For example, 'st' might be the final state at a conditional
3622 // return. Or, 'st' might be used by some node which is live at
3623 // the same time 'st' is live, which might be unschedulable. So,
3624 // require exactly ONE user until such time as we clone 'mem' for
3625 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3626 // true).
3627 while (st->is_Store() && st->outcnt() == 1) {
3628 // Looking at a dead closed cycle of memory?
3629 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3630 assert(Opcode() == st->Opcode() ||
3631 st->Opcode() == Op_StoreVector ||
3632 Opcode() == Op_StoreVector ||
3633 st->Opcode() == Op_StoreVectorScatter ||
3634 Opcode() == Op_StoreVectorScatter ||
3635 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3636 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3637 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3638 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3639 (st->adr_type()->isa_aryptr() && st->adr_type()->is_aryptr()->is_flat()) || // TODO 8343835
3640 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3641 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3642
3643 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3644 st->as_Store()->memory_size() <= this->memory_size()) {
3645 Node* use = st->raw_out(0);
3646 if (phase->is_IterGVN()) {
3647 phase->is_IterGVN()->rehash_node_delayed(use);
3648 }
3649 // It's OK to do this in the parser, since DU info is always accurate,
3650 // and the parser always refers to nodes via SafePointNode maps.
3651 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3652 return this;
3653 }
3654 st = st->in(MemNode::Memory);
3655 }
3656 }
3657
3658
3659 // Capture an unaliased, unconditional, simple store into an initializer.
3757 const StoreVectorNode* store_vector = as_StoreVector();
3758 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3759 const Node* store_indices = store_vector->indices();
3760 const Node* mem_indices = mem_vector->indices();
3761 const Node* store_mask = store_vector->mask();
3762 const Node* mem_mask = mem_vector->mask();
3763 // Ensure types, indices, and masks match
3764 if (store_vector->vect_type() == mem_vector->vect_type() &&
3765 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3766 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3767 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3768 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3769 result = mem;
3770 }
3771 }
3772 }
3773
3774 // Store of zero anywhere into a freshly-allocated object?
3775 // Then the store is useless.
3776 // (It must already have been captured by the InitializeNode.)
3777 if (result == this && ReduceFieldZeroing) {
3778 // a newly allocated object is already all-zeroes everywhere
3779 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3780 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3781 result = mem;
3782 }
3783
3784 if (result == this && phase->type(val)->is_zero_type()) {
3785 // the store may also apply to zero-bits in an earlier object
3786 Node* prev_mem = find_previous_store(phase);
3787 // Steps (a), (b): Walk past independent stores to find an exact match.
3788 if (prev_mem != nullptr) {
3789 Node* prev_val = can_see_stored_value(prev_mem, phase);
3790 if (prev_val != nullptr && prev_val == val) {
3791 // prev_val and val might differ by a cast; it would be good
3792 // to keep the more informative of the two.
3793 result = mem;
3794 }
3795 }
3796 }
3797 }
3798
3799 PhaseIterGVN* igvn = phase->is_IterGVN();
3800 if (result != this && igvn != nullptr) {
3801 MemBarNode* trailing = trailing_membar();
3802 if (trailing != nullptr) {
3803 #ifdef ASSERT
3804 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
4268 // Clearing a short array is faster with stores
4269 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4270 // Already know this is a large node, do not try to ideal it
4271 if (_is_large) return nullptr;
4272
4273 const int unit = BytesPerLong;
4274 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4275 if (!t) return nullptr;
4276 if (!t->is_con()) return nullptr;
4277 intptr_t raw_count = t->get_con();
4278 intptr_t size = raw_count;
4279 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4280 // Clearing nothing uses the Identity call.
4281 // Negative clears are possible on dead ClearArrays
4282 // (see jck test stmt114.stmt11402.val).
4283 if (size <= 0 || size % unit != 0) return nullptr;
4284 intptr_t count = size / unit;
4285 // Length too long; communicate this to matchers and assemblers.
4286 // Assemblers are responsible to produce fast hardware clears for it.
4287 if (size > InitArrayShortSize) {
4288 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4289 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4290 return nullptr;
4291 }
4292 if (!IdealizeClearArrayNode) return nullptr;
4293 Node *mem = in(1);
4294 if( phase->type(mem)==Type::TOP ) return nullptr;
4295 Node *adr = in(3);
4296 const Type* at = phase->type(adr);
4297 if( at==Type::TOP ) return nullptr;
4298 const TypePtr* atp = at->isa_ptr();
4299 // adjust atp to be the correct array element address type
4300 if (atp == nullptr) atp = TypePtr::BOTTOM;
4301 else atp = atp->add_offset(Type::OffsetBot);
4302 // Get base for derived pointer purposes
4303 if( adr->Opcode() != Op_AddP ) Unimplemented();
4304 Node *base = adr->in(1);
4305
4306 Node *val = in(4);
4307 Node *off = phase->MakeConX(BytesPerLong);
4308 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4309 count--;
4310 while( count-- ) {
4311 mem = phase->transform(mem);
4312 adr = phase->transform(new AddPNode(base,adr,off));
4313 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4314 }
4315 return mem;
4316 }
4317
4318 //----------------------------step_through----------------------------------
4319 // Return allocation input memory edge if it is different instance
4320 // or itself if it is the one we are looking for.
4321 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4322 Node* n = *np;
4323 assert(n->is_ClearArray(), "sanity");
4324 intptr_t offset;
4325 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4326 // This method is called only before Allocate nodes are expanded
4327 // during macro nodes expansion. Before that ClearArray nodes are
4328 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4329 // Allocate nodes are expanded) which follows allocations.
4330 assert(alloc != nullptr, "should have allocation");
4331 if (alloc->_idx == instance_id) {
4332 // Can not bypass initialization of the instance we are looking for.
4333 return false;
4334 }
4335 // Otherwise skip it.
4336 InitializeNode* init = alloc->initialization();
4337 if (init != nullptr)
4338 *np = init->in(TypeFunc::Memory);
4339 else
4340 *np = alloc->in(TypeFunc::Memory);
4341 return true;
4342 }
4343
4344 //----------------------------clear_memory-------------------------------------
4345 // Generate code to initialize object storage to zero.
4346 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4347 Node* val,
4348 Node* raw_val,
4349 intptr_t start_offset,
4350 Node* end_offset,
4351 PhaseGVN* phase) {
4352 intptr_t offset = start_offset;
4353
4354 int unit = BytesPerLong;
4355 if ((offset % unit) != 0) {
4356 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
4357 adr = phase->transform(adr);
4358 const TypePtr* atp = TypeRawPtr::BOTTOM;
4359 if (val != nullptr) {
4360 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4361 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4362 } else {
4363 assert(raw_val == nullptr, "val may not be null");
4364 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4365 }
4366 mem = phase->transform(mem);
4367 offset += BytesPerInt;
4368 }
4369 assert((offset % unit) == 0, "");
4370
4371 // Initialize the remaining stuff, if any, with a ClearArray.
4372 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
4373 }
4374
4375 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4376 Node* raw_val,
4377 Node* start_offset,
4378 Node* end_offset,
4379 PhaseGVN* phase) {
4380 if (start_offset == end_offset) {
4381 // nothing to do
4382 return mem;
4383 }
4384
4385 int unit = BytesPerLong;
4386 Node* zbase = start_offset;
4387 Node* zend = end_offset;
4388
4389 // Scale to the unit required by the CPU:
4390 if (!Matcher::init_array_count_is_in_bytes) {
4391 Node* shift = phase->intcon(exact_log2(unit));
4392 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4393 zend = phase->transform(new URShiftXNode(zend, shift) );
4394 }
4395
4396 // Bulk clear double-words
4397 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4398 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
4399 if (raw_val == nullptr) {
4400 raw_val = phase->MakeConX(0);
4401 }
4402 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4403 return phase->transform(mem);
4404 }
4405
4406 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4407 Node* val,
4408 Node* raw_val,
4409 intptr_t start_offset,
4410 intptr_t end_offset,
4411 PhaseGVN* phase) {
4412 if (start_offset == end_offset) {
4413 // nothing to do
4414 return mem;
4415 }
4416
4417 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4418 intptr_t done_offset = end_offset;
4419 if ((done_offset % BytesPerLong) != 0) {
4420 done_offset -= BytesPerInt;
4421 }
4422 if (done_offset > start_offset) {
4423 mem = clear_memory(ctl, mem, dest, val, raw_val,
4424 start_offset, phase->MakeConX(done_offset), phase);
4425 }
4426 if (done_offset < end_offset) { // emit the final 32-bit store
4427 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
4428 adr = phase->transform(adr);
4429 const TypePtr* atp = TypeRawPtr::BOTTOM;
4430 if (val != nullptr) {
4431 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4432 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4433 } else {
4434 assert(raw_val == nullptr, "val may not be null");
4435 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4436 }
4437 mem = phase->transform(mem);
4438 done_offset += BytesPerInt;
4439 }
4440 assert(done_offset == end_offset, "");
4441 return mem;
4442 }
4443
4444 //=============================================================================
4445 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4446 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4447 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4448 #ifdef ASSERT
4449 , _pair_idx(0)
4450 #endif
4451 {
4452 init_class_id(Class_MemBar);
4453 Node* top = C->top();
4454 init_req(TypeFunc::I_O,top);
4455 init_req(TypeFunc::FramePtr,top);
4456 init_req(TypeFunc::ReturnAdr,top);
4562 PhaseIterGVN* igvn = phase->is_IterGVN();
4563 remove(igvn);
4564 // Must return either the original node (now dead) or a new node
4565 // (Do not return a top here, since that would break the uniqueness of top.)
4566 return new ConINode(TypeInt::ZERO);
4567 }
4568 }
4569 return progress ? this : nullptr;
4570 }
4571
4572 //------------------------------Value------------------------------------------
4573 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4574 if( !in(0) ) return Type::TOP;
4575 if( phase->type(in(0)) == Type::TOP )
4576 return Type::TOP;
4577 return TypeTuple::MEMBAR;
4578 }
4579
4580 //------------------------------match------------------------------------------
4581 // Construct projections for memory.
4582 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4583 switch (proj->_con) {
4584 case TypeFunc::Control:
4585 case TypeFunc::Memory:
4586 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4587 }
4588 ShouldNotReachHere();
4589 return nullptr;
4590 }
4591
4592 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4593 trailing->_kind = TrailingStore;
4594 leading->_kind = LeadingStore;
4595 #ifdef ASSERT
4596 trailing->_pair_idx = leading->_idx;
4597 leading->_pair_idx = leading->_idx;
4598 #endif
4599 }
4600
4601 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4602 trailing->_kind = TrailingLoadStore;
4849 return (req() > RawStores);
4850 }
4851
4852 void InitializeNode::set_complete(PhaseGVN* phase) {
4853 assert(!is_complete(), "caller responsibility");
4854 _is_complete = Complete;
4855
4856 // After this node is complete, it contains a bunch of
4857 // raw-memory initializations. There is no need for
4858 // it to have anything to do with non-raw memory effects.
4859 // Therefore, tell all non-raw users to re-optimize themselves,
4860 // after skipping the memory effects of this initialization.
4861 PhaseIterGVN* igvn = phase->is_IterGVN();
4862 if (igvn) igvn->add_users_to_worklist(this);
4863 }
4864
4865 // convenience function
4866 // return false if the init contains any stores already
4867 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4868 InitializeNode* init = initialization();
4869 if (init == nullptr || init->is_complete()) {
4870 return false;
4871 }
4872 init->remove_extra_zeroes();
4873 // for now, if this allocation has already collected any inits, bail:
4874 if (init->is_non_zero()) return false;
4875 init->set_complete(phase);
4876 return true;
4877 }
4878
4879 void InitializeNode::remove_extra_zeroes() {
4880 if (req() == RawStores) return;
4881 Node* zmem = zero_memory();
4882 uint fill = RawStores;
4883 for (uint i = fill; i < req(); i++) {
4884 Node* n = in(i);
4885 if (n->is_top() || n == zmem) continue; // skip
4886 if (fill < i) set_req(fill, n); // compact
4887 ++fill;
4888 }
4889 // delete any empty spaces created:
4890 while (fill < req()) {
4891 del_req(fill);
5035 // store node that we'd like to capture. We need to check
5036 // the uses of the MergeMemNode.
5037 mems.push(n);
5038 }
5039 } else if (n->is_Mem()) {
5040 Node* other_adr = n->in(MemNode::Address);
5041 if (other_adr == adr) {
5042 failed = true;
5043 break;
5044 } else {
5045 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5046 if (other_t_adr != nullptr) {
5047 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5048 if (other_alias_idx == alias_idx) {
5049 // A load from the same memory slice as the store right
5050 // after the InitializeNode. We check the control of the
5051 // object/array that is loaded from. If it's the same as
5052 // the store control then we cannot capture the store.
5053 assert(!n->is_Store(), "2 stores to same slice on same control?");
5054 Node* base = other_adr;
5055 if (base->is_Phi()) {
5056 // In rare case, base may be a PhiNode and it may read
5057 // the same memory slice between InitializeNode and store.
5058 failed = true;
5059 break;
5060 }
5061 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5062 base = base->in(AddPNode::Base);
5063 if (base != nullptr) {
5064 base = base->uncast();
5065 if (base->is_Proj() && base->in(0) == alloc) {
5066 failed = true;
5067 break;
5068 }
5069 }
5070 }
5071 }
5072 }
5073 } else {
5074 failed = true;
5075 break;
5076 }
5077 }
5078 }
5079 }
5080 if (failed) {
5627 // z's_done 12 16 16 16 12 16 12
5628 // z's_needed 12 16 16 16 16 16 16
5629 // zsize 0 0 0 0 4 0 4
5630 if (next_full_store < 0) {
5631 // Conservative tack: Zero to end of current word.
5632 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5633 } else {
5634 // Zero to beginning of next fully initialized word.
5635 // Or, don't zero at all, if we are already in that word.
5636 assert(next_full_store >= zeroes_needed, "must go forward");
5637 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5638 zeroes_needed = next_full_store;
5639 }
5640 }
5641
5642 if (zeroes_needed > zeroes_done) {
5643 intptr_t zsize = zeroes_needed - zeroes_done;
5644 // Do some incremental zeroing on rawmem, in parallel with inits.
5645 zeroes_done = align_down(zeroes_done, BytesPerInt);
5646 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5647 allocation()->in(AllocateNode::InitValue),
5648 allocation()->in(AllocateNode::RawInitValue),
5649 zeroes_done, zeroes_needed,
5650 phase);
5651 zeroes_done = zeroes_needed;
5652 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5653 do_zeroing = false; // leave the hole, next time
5654 }
5655 }
5656
5657 // Collect the store and move on:
5658 phase->replace_input_of(st, MemNode::Memory, inits);
5659 inits = st; // put it on the linearized chain
5660 set_req(i, zmem); // unhook from previous position
5661
5662 if (zeroes_done == st_off)
5663 zeroes_done = next_init_off;
5664
5665 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5666
5667 #ifdef ASSERT
5668 // Various order invariants. Weaker than stores_are_sane because
5688 remove_extra_zeroes(); // clear out all the zmems left over
5689 add_req(inits);
5690
5691 if (!(UseTLAB && ZeroTLAB)) {
5692 // If anything remains to be zeroed, zero it all now.
5693 zeroes_done = align_down(zeroes_done, BytesPerInt);
5694 // if it is the last unused 4 bytes of an instance, forget about it
5695 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5696 if (zeroes_done + BytesPerLong >= size_limit) {
5697 AllocateNode* alloc = allocation();
5698 assert(alloc != nullptr, "must be present");
5699 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5700 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5701 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5702 if (zeroes_done == k->layout_helper())
5703 zeroes_done = size_limit;
5704 }
5705 }
5706 if (zeroes_done < size_limit) {
5707 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5708 allocation()->in(AllocateNode::InitValue),
5709 allocation()->in(AllocateNode::RawInitValue),
5710 zeroes_done, size_in_bytes, phase);
5711 }
5712 }
5713
5714 set_complete(phase);
5715 return rawmem;
5716 }
5717
5718
5719 #ifdef ASSERT
5720 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5721 if (is_complete())
5722 return true; // stores could be anything at this point
5723 assert(allocation() != nullptr, "must be present");
5724 intptr_t last_off = allocation()->minimum_header_size();
5725 for (uint i = InitializeNode::RawStores; i < req(); i++) {
5726 Node* st = in(i);
5727 intptr_t st_off = get_store_offset(st, phase);
5728 if (st_off < 0) continue; // ignore dead garbage
5729 if (last_off > st_off) {
|