6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/compile.hpp"
38 #include "opto/connode.hpp"
39 #include "opto/convertnode.hpp"
40 #include "opto/loopnode.hpp"
41 #include "opto/machnode.hpp"
42 #include "opto/matcher.hpp"
43 #include "opto/memnode.hpp"
44 #include "opto/mempointer.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/opcodes.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regalloc.hpp"
50 #include "opto/regmask.hpp"
51 #include "opto/rootnode.hpp"
52 #include "opto/traceMergeStoresTag.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/macros.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #include "utilities/vmError.hpp"
59
60 // Portions of code courtesy of Clifford Click
61
62 // Optimization - Graph Style
63
64 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
65
66 //=============================================================================
67 uint MemNode::size_of() const { return sizeof(*this); }
68
69 const TypePtr *MemNode::adr_type() const {
70 Node* adr = in(Address);
71 if (adr == nullptr) return nullptr; // node is dead
72 const TypePtr* cross_check = nullptr;
73 DEBUG_ONLY(cross_check = _adr_type);
74 return calculate_adr_type(adr->bottom_type(), cross_check);
75 }
124 st->print(", idx=Bot;");
125 else if (atp->index() == Compile::AliasIdxTop)
126 st->print(", idx=Top;");
127 else if (atp->index() == Compile::AliasIdxRaw)
128 st->print(", idx=Raw;");
129 else {
130 ciField* field = atp->field();
131 if (field) {
132 st->print(", name=");
133 field->print_name_on(st);
134 }
135 st->print(", idx=%d;", atp->index());
136 }
137 }
138 }
139
140 extern void print_alias_types();
141
142 #endif
143
144 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
145 assert((t_oop != nullptr), "sanity");
146 bool is_instance = t_oop->is_known_instance_field();
147 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
148 (load != nullptr) && load->is_Load() &&
149 (phase->is_IterGVN() != nullptr);
150 if (!(is_instance || is_boxed_value_load))
151 return mchain; // don't try to optimize non-instance types
152 uint instance_id = t_oop->instance_id();
153 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
154 Node *prev = nullptr;
155 Node *result = mchain;
156 while (prev != result) {
157 prev = result;
158 if (result == start_mem)
159 break; // hit one of our sentinels
160 // skip over a call which does not affect this memory slice
161 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
162 Node *proj_in = result->in(0);
163 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
164 break; // hit one of our sentinels
165 } else if (proj_in->is_Call()) {
166 // ArrayCopyNodes processed here as well
167 CallNode *call = proj_in->as_Call();
168 if (!call->may_modify(t_oop, phase)) { // returns false for instances
169 result = call->in(TypeFunc::Memory);
170 }
171 } else if (proj_in->is_Initialize()) {
172 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
173 // Stop if this is the initialization for the object instance which
174 // contains this memory slice, otherwise skip over it.
175 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
176 break;
177 }
178 if (is_instance) {
179 result = proj_in->in(TypeFunc::Memory);
180 } else if (is_boxed_value_load) {
181 Node* klass = alloc->in(AllocateNode::KlassNode);
182 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
183 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
184 result = proj_in->in(TypeFunc::Memory); // not related allocation
185 }
186 }
187 } else if (proj_in->is_MemBar()) {
188 ArrayCopyNode* ac = nullptr;
189 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
190 break;
191 }
192 result = proj_in->in(TypeFunc::Memory);
193 } else if (proj_in->is_top()) {
194 break; // dead code
195 } else {
196 assert(false, "unexpected projection");
197 }
198 } else if (result->is_ClearArray()) {
199 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
200 // Can not bypass initialization of the instance
201 // we are looking for.
202 break;
203 }
204 // Otherwise skip it (the call updated 'result' value).
205 } else if (result->is_MergeMem()) {
206 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
207 }
208 }
209 return result;
210 }
211
212 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
213 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
214 if (t_oop == nullptr)
215 return mchain; // don't try to optimize non-oop types
216 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
217 bool is_instance = t_oop->is_known_instance_field();
218 PhaseIterGVN *igvn = phase->is_IterGVN();
219 if (is_instance && igvn != nullptr && result->is_Phi()) {
220 PhiNode *mphi = result->as_Phi();
221 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
222 const TypePtr *t = mphi->adr_type();
223 bool do_split = false;
224 // In the following cases, Load memory input can be further optimized based on
225 // its precise address type
226 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
227 do_split = true;
228 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
229 const TypeOopPtr* mem_t =
230 t->is_oopptr()->cast_to_exactness(true)
231 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
232 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
233 if (t_oop->isa_aryptr()) {
234 mem_t = mem_t->is_aryptr()
235 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
236 ->cast_to_size(t_oop->is_aryptr()->size())
237 ->with_offset(t_oop->is_aryptr()->offset())
238 ->is_aryptr();
239 }
240 do_split = mem_t == t_oop;
241 }
242 if (do_split) {
243 // clone the Phi with our address type
244 result = mphi->split_out_instance(t_adr, igvn);
245 } else {
246 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
247 }
248 }
249 return result;
250 }
251
252 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
253 uint alias_idx = phase->C->get_alias_index(tp);
254 Node *mem = mmem;
255 #ifdef ASSERT
256 {
257 // Check that current type is consistent with the alias index used during graph construction
258 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
259 bool consistent = adr_check == nullptr || adr_check->empty() ||
260 phase->C->must_alias(adr_check, alias_idx );
261 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
262 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
263 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
264 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
265 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
266 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
267 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
268 // don't assert if it is dead code.
269 consistent = true;
270 }
271 if( !consistent ) {
272 st->print("alias_idx==%d, adr_check==", alias_idx);
273 if( adr_check == nullptr ) {
274 st->print("null");
275 } else {
276 adr_check->dump();
277 }
278 st->cr();
279 print_alias_types();
280 assert(consistent, "adr_check must match alias idx");
281 }
282 }
283 #endif
993 in_bytes(JavaThread::vthread_offset()),
994 in_bytes(JavaThread::scopedValueCache_offset()),
995 };
996
997 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
998 if (offset == offsets[i]) {
999 return true;
1000 }
1001 }
1002 }
1003
1004 return false;
1005 }
1006 #endif
1007
1008 //----------------------------LoadNode::make-----------------------------------
1009 // Polymorphic factory method:
1010 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1011 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1012 Compile* C = gvn.C;
1013 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
1014
1015 // sanity check the alias category against the created node type
1016 assert(!(adr_type->isa_oopptr() &&
1017 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1018 "use LoadKlassNode instead");
1019 assert(!(adr_type->isa_aryptr() &&
1020 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1021 "use LoadRangeNode instead");
1022 // Check control edge of raw loads
1023 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1024 // oop will be recorded in oop map if load crosses safepoint
1025 rt->isa_oopptr() || is_immutable_value(adr),
1026 "raw memory operations should have control edge");
1027 LoadNode* load = nullptr;
1028 switch (bt) {
1029 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1030 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1031 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1032 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1033 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1034 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1035 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1036 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1037 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1038 case T_OBJECT:
1039 case T_NARROWOOP:
1040 #ifdef _LP64
1041 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1042 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1043 } else
1044 #endif
1045 {
1046 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1047 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1048 }
1049 break;
1050 default:
1051 ShouldNotReachHere();
1052 break;
1053 }
1054 assert(load != nullptr, "LoadNode should have been created");
1055 if (unaligned) {
1056 load->set_unaligned_access();
1057 }
1058 if (mismatched) {
1059 load->set_mismatched_access();
1060 }
1061 if (unsafe) {
1062 load->set_unsafe_access();
1063 }
1064 load->set_barrier_data(barrier_data);
1065 if (load->Opcode() == Op_LoadN) {
1066 Node* ld = gvn.transform(load);
1067 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1068 }
1069
1070 return load;
1071 }
1072
1073 //------------------------------hash-------------------------------------------
1074 uint LoadNode::hash() const {
1075 // unroll addition of interesting fields
1076 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1077 }
1078
1079 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1080 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1081 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1082 bool is_stable_ary = FoldStableValues &&
1083 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1084 tp->isa_aryptr()->is_stable();
1085
1086 return (eliminate_boxing && non_volatile) || is_stable_ary;
1087 }
1088
1089 return false;
1090 }
1091
1092 // Is the value loaded previously stored by an arraycopy? If so return
1093 // a load node that reads from the source array so we may be able to
1094 // optimize out the ArrayCopy node later.
1095 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1096 Node* ld_adr = in(MemNode::Address);
1097 intptr_t ld_off = 0;
1098 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1099 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1100 if (ac != nullptr) {
1101 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1102
1103 Node* mem = ac->in(TypeFunc::Memory);
1104 Node* ctl = ac->in(0);
1105 Node* src = ac->in(ArrayCopyNode::Src);
1106
1114 if (ac->as_ArrayCopy()->is_clonebasic()) {
1115 assert(ld_alloc != nullptr, "need an alloc");
1116 assert(addp->is_AddP(), "address must be addp");
1117 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1118 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1119 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1120 addp->set_req(AddPNode::Base, src);
1121 addp->set_req(AddPNode::Address, src);
1122 } else {
1123 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1124 ac->as_ArrayCopy()->is_copyof_validated() ||
1125 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1126 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1127 addp->set_req(AddPNode::Base, src);
1128 addp->set_req(AddPNode::Address, src);
1129
1130 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1131 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1132 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1133
1134 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1135 uint shift = exact_log2(type2aelembytes(ary_elem));
1136
1137 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1138 #ifdef _LP64
1139 diff = phase->transform(new ConvI2LNode(diff));
1140 #endif
1141 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1142
1143 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1144 addp->set_req(AddPNode::Offset, offset);
1145 }
1146 addp = phase->transform(addp);
1147 #ifdef ASSERT
1148 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1149 ld->_adr_type = adr_type;
1150 #endif
1151 ld->set_req(MemNode::Address, addp);
1152 ld->set_req(0, ctl);
1153 ld->set_req(MemNode::Memory, mem);
1154 return ld;
1155 }
1156 return nullptr;
1157 }
1158
1159 // This routine exists to make sure this set of tests is done the same
1160 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1161 // will change the graph shape in a way which makes memory alive twice at the
1162 // same time (uses the Oracle model of aliasing), then some
1163 // LoadXNode::Identity will fold things back to the equivalence-class model
1164 // of aliasing.
1165 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1166 Node* ld_adr = in(MemNode::Address);
1167 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1168 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1169
1170 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1171 uint alias_idx = atp->index();
1172 Node* result = nullptr;
1173 Node* current = st;
1174 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1175 // this load. Stop once any other kind of node is encountered.
1176 //
1177 // In principle, folding a load is moving it up until it meets a matching store.
1178 //
1179 // store(ptr, v); store(ptr, v); store(ptr, v);
1180 // membar1; -> membar1; -> load(ptr);
1181 // membar2; load(ptr); membar1;
1182 // load(ptr); membar2; membar2;
1183 //
1184 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1185 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1186 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1266
1267 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1268 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1269 if (in_vt != out_vt) {
1270 return nullptr;
1271 }
1272 }
1273 return st->in(MemNode::ValueIn);
1274 }
1275
1276 // A load from a freshly-created object always returns zero.
1277 // (This can happen after LoadNode::Ideal resets the load's memory input
1278 // to find_captured_store, which returned InitializeNode::zero_memory.)
1279 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1280 (st->in(0) == ld_alloc) &&
1281 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1282 // return a zero value for the load's basic type
1283 // (This is one of the few places where a generic PhaseTransform
1284 // can create new nodes. Think of it as lazily manifesting
1285 // virtually pre-existing constants.)
1286 if (value_basic_type() != T_VOID) {
1287 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1288 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1289 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1290 // by the ArrayCopyNode.
1291 return phase->zerocon(value_basic_type());
1292 }
1293 } else {
1294 // TODO: materialize all-zero vector constant
1295 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1296 }
1297 }
1298
1299 // A load from an initialization barrier can match a captured store.
1300 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1301 InitializeNode* init = st->in(0)->as_Initialize();
1302 AllocateNode* alloc = init->allocation();
1303 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1304 // examine a captured store value
1305 st = init->find_captured_store(ld_off, memory_size(), phase);
1318 base = bs->step_over_gc_barrier(base);
1319 if (base != nullptr && base->is_Proj() &&
1320 base->as_Proj()->_con == TypeFunc::Parms &&
1321 base->in(0)->is_CallStaticJava() &&
1322 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1323 return base->in(0)->in(TypeFunc::Parms);
1324 }
1325 }
1326
1327 break;
1328 }
1329
1330 return nullptr;
1331 }
1332
1333 //----------------------is_instance_field_load_with_local_phi------------------
1334 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1335 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1336 in(Address)->is_AddP() ) {
1337 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1338 // Only instances and boxed values.
1339 if( t_oop != nullptr &&
1340 (t_oop->is_ptr_to_boxed_value() ||
1341 t_oop->is_known_instance_field()) &&
1342 t_oop->offset() != Type::OffsetBot &&
1343 t_oop->offset() != Type::OffsetTop) {
1344 return true;
1345 }
1346 }
1347 return false;
1348 }
1349
1350 //------------------------------Identity---------------------------------------
1351 // Loads are identity if previous store is to same address
1352 Node* LoadNode::Identity(PhaseGVN* phase) {
1353 // If the previous store-maker is the right kind of Store, and the store is
1354 // to the same address, then we are equal to the value stored.
1355 Node* mem = in(Memory);
1356 Node* value = can_see_stored_value_through_membars(mem, phase);
1357 if( value ) {
1358 // byte, short & char stores truncate naturally.
1359 // A load has to load the truncated value which requires
1360 // some sort of masking operation and that requires an
1361 // Ideal call instead of an Identity call.
1362 if (memory_size() < BytesPerInt) {
1363 // If the input to the store does not fit with the load's result type,
1364 // it must be truncated via an Ideal call.
1365 if (!phase->type(value)->higher_equal(phase->type(this)))
1366 return this;
1367 }
1368 // (This works even when value is a Con, but LoadNode::Value
1369 // usually runs first, producing the singleton type of the Con.)
1370 if (!has_pinned_control_dependency() || value->is_Con()) {
1371 return value;
1372 } else {
1373 return this;
1374 }
1375 }
1376
1377 if (has_pinned_control_dependency()) {
1378 return this;
1379 }
1380 // Search for an existing data phi which was generated before for the same
1381 // instance's field to avoid infinite generation of phis in a loop.
1382 Node *region = mem->in(0);
1383 if (is_instance_field_load_with_local_phi(region)) {
1384 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1385 int this_index = phase->C->get_alias_index(addr_t);
1386 int this_offset = addr_t->offset();
1387 int this_iid = addr_t->instance_id();
1388 if (!addr_t->is_known_instance() &&
1389 addr_t->is_ptr_to_boxed_value()) {
1390 // Use _idx of address base (could be Phi node) for boxed values.
1391 intptr_t ignore = 0;
1392 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1393 if (base == nullptr) {
1394 return this;
1395 }
1396 this_iid = base->_idx;
1397 }
1398 const Type* this_type = bottom_type();
1399 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1400 Node* phi = region->fast_out(i);
1401 if (phi->is_Phi() && phi != mem &&
1402 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1403 return phi;
1404 }
1405 }
1406 }
1407
1408 return this;
1409 }
1410
1654 Node* in = phi->in(i);
1655 if (in == nullptr || phase->type(in) == Type::TOP)
1656 return false; // Wait stable graph
1657 }
1658 return true;
1659 }
1660
1661 //------------------------------split_through_phi------------------------------
1662 // Check whether a call to 'split_through_phi' would split this load through the
1663 // Phi *base*. This method is essentially a copy of the validations performed
1664 // by 'split_through_phi'. The first use of this method was in EA code as part
1665 // of simplification of allocation merges.
1666 // Some differences from original method (split_through_phi):
1667 // - If base->is_CastPP(): base = base->in(1)
1668 bool LoadNode::can_split_through_phi_base(PhaseGVN* phase) {
1669 Node* mem = in(Memory);
1670 Node* address = in(Address);
1671 intptr_t ignore = 0;
1672 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1673
1674 if (base->is_CastPP()) {
1675 base = base->in(1);
1676 }
1677
1678 if (req() > 3 || base == nullptr || !base->is_Phi()) {
1679 return false;
1680 }
1681
1682 if (!mem->is_Phi()) {
1683 if (!MemNode::all_controls_dominate(mem, base->in(0))) {
1684 return false;
1685 }
1686 } else if (base->in(0) != mem->in(0)) {
1687 if (!MemNode::all_controls_dominate(mem, base->in(0))) {
1688 return false;
1689 }
1690 }
1691
1692 return true;
1693 }
1694
1941 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1942 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1943
1944 // Skip up past a SafePoint control. Cannot do this for Stores because
1945 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1946 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1947 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1948 !addr_mark &&
1949 (depends_only_on_test() || has_unknown_control_dependency())) {
1950 ctrl = ctrl->in(0);
1951 set_req(MemNode::Control,ctrl);
1952 return this;
1953 }
1954
1955 intptr_t ignore = 0;
1956 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1957 if (base != nullptr
1958 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1959 // Check for useless control edge in some common special cases
1960 if (in(MemNode::Control) != nullptr
1961 && can_remove_control()
1962 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1963 && all_controls_dominate(base, phase->C->start())) {
1964 // A method-invariant, non-null address (constant or 'this' argument).
1965 set_req(MemNode::Control, nullptr);
1966 return this;
1967 }
1968 }
1969
1970 Node* mem = in(MemNode::Memory);
1971 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1972
1973 if (can_reshape && (addr_t != nullptr)) {
1974 // try to optimize our memory input
1975 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1976 if (opt_mem != mem) {
1977 set_req_X(MemNode::Memory, opt_mem, phase);
1978 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1979 return this;
1980 }
2093 // No match.
2094 return nullptr;
2095 }
2096
2097 //------------------------------Value-----------------------------------------
2098 const Type* LoadNode::Value(PhaseGVN* phase) const {
2099 // Either input is TOP ==> the result is TOP
2100 Node* mem = in(MemNode::Memory);
2101 const Type *t1 = phase->type(mem);
2102 if (t1 == Type::TOP) return Type::TOP;
2103 Node* adr = in(MemNode::Address);
2104 const TypePtr* tp = phase->type(adr)->isa_ptr();
2105 if (tp == nullptr || tp->empty()) return Type::TOP;
2106 int off = tp->offset();
2107 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2108 Compile* C = phase->C;
2109
2110 // If load can see a previous constant store, use that.
2111 Node* value = can_see_stored_value_through_membars(mem, phase);
2112 if (value != nullptr && value->is_Con()) {
2113 assert(value->bottom_type()->higher_equal(_type), "sanity");
2114 return value->bottom_type();
2115 }
2116
2117 // Try to guess loaded type from pointer type
2118 if (tp->isa_aryptr()) {
2119 const TypeAryPtr* ary = tp->is_aryptr();
2120 const Type* t = ary->elem();
2121
2122 // Determine whether the reference is beyond the header or not, by comparing
2123 // the offset against the offset of the start of the array's data.
2124 // Different array types begin at slightly different offsets (12 vs. 16).
2125 // We choose T_BYTE as an example base type that is least restrictive
2126 // as to alignment, which will therefore produce the smallest
2127 // possible base offset.
2128 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2129 const bool off_beyond_header = (off >= min_base_off);
2130
2131 // Try to constant-fold a stable array element.
2132 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2133 // Make sure the reference is not into the header and the offset is constant
2134 ciObject* aobj = ary->const_oop();
2135 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2136 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2137 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off,
2138 stable_dimension,
2139 value_basic_type(), is_unsigned());
2140 if (con_type != nullptr) {
2141 return con_type;
2142 }
2143 }
2144 }
2145
2146 // Don't do this for integer types. There is only potential profit if
2147 // the element type t is lower than _type; that is, for int types, if _type is
2148 // more restrictive than t. This only happens here if one is short and the other
2149 // char (both 16 bits), and in those cases we've made an intentional decision
2150 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2151 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2152 //
2153 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2154 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2155 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2156 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2157 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2158 // In fact, that could have been the original type of p1, and p1 could have
2159 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2160 // expression (LShiftL quux 3) independently optimized to the constant 8.
2161 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2162 && (_type->isa_vect() == nullptr)
2163 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2164 // t might actually be lower than _type, if _type is a unique
2165 // concrete subclass of abstract class t.
2166 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2167 const Type* jt = t->join_speculative(_type);
2168 // In any case, do not allow the join, per se, to empty out the type.
2169 if (jt->empty() && !t->empty()) {
2170 // This can happen if a interface-typed array narrows to a class type.
2171 jt = _type;
2172 }
2173 #ifdef ASSERT
2174 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2175 // The pointers in the autobox arrays are always non-null
2176 Node* base = adr->in(AddPNode::Base);
2177 if ((base != nullptr) && base->is_DecodeN()) {
2178 // Get LoadN node which loads IntegerCache.cache field
2179 base = base->in(1);
2180 }
2181 if ((base != nullptr) && base->is_Con()) {
2182 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2183 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2184 // It could be narrow oop
2185 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2186 }
2187 }
2188 }
2189 #endif
2190 return jt;
2191 }
2192 }
2193 } else if (tp->base() == Type::InstPtr) {
2194 assert( off != Type::OffsetBot ||
2195 // arrays can be cast to Objects
2196 !tp->isa_instptr() ||
2197 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2198 // unsafe field access may not have a constant offset
2199 C->has_unsafe_access(),
2200 "Field accesses must be precise" );
2201 // For oop loads, we expect the _type to be precise.
2202
2203 // Optimize loads from constant fields.
2204 const TypeInstPtr* tinst = tp->is_instptr();
2205 ciObject* const_oop = tinst->const_oop();
2206 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2207 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2208 if (con_type != nullptr) {
2209 return con_type;
2210 }
2211 }
2212 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2213 assert(off != Type::OffsetBot ||
2214 !tp->isa_instklassptr() ||
2215 // arrays can be cast to Objects
2216 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2217 // also allow array-loading from the primary supertype
2218 // array during subtype checks
2219 Opcode() == Op_LoadKlass,
2220 "Field accesses must be precise");
2221 // For klass/static loads, we expect the _type to be precise
2222 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2223 /* With mirrors being an indirect in the Klass*
2224 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2225 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2226 *
2227 * So check the type and klass of the node before the LoadP.
2234 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2235 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2236 return TypeInstPtr::make(klass->java_mirror());
2237 }
2238 }
2239 }
2240
2241 const TypeKlassPtr *tkls = tp->isa_klassptr();
2242 if (tkls != nullptr) {
2243 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2244 ciKlass* klass = tkls->exact_klass();
2245 // We are loading a field from a Klass metaobject whose identity
2246 // is known at compile time (the type is "exact" or "precise").
2247 // Check for fields we know are maintained as constants by the VM.
2248 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2249 // The field is Klass::_super_check_offset. Return its (constant) value.
2250 // (Folds up type checking code.)
2251 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2252 return TypeInt::make(klass->super_check_offset());
2253 }
2254 if (UseCompactObjectHeaders) {
2255 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2256 // The field is Klass::_prototype_header. Return its (constant) value.
2257 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2258 return TypeX::make(klass->prototype_header());
2259 }
2260 }
2261 // Compute index into primary_supers array
2262 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2263 // Check for overflowing; use unsigned compare to handle the negative case.
2264 if( depth < ciKlass::primary_super_limit() ) {
2265 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2266 // (Folds up type checking code.)
2267 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2268 ciKlass *ss = klass->super_of_depth(depth);
2269 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2270 }
2271 const Type* aift = load_array_final_field(tkls, klass);
2272 if (aift != nullptr) return aift;
2273 }
2274
2275 // We can still check if we are loading from the primary_supers array at a
2276 // shallow enough depth. Even though the klass is not exact, entries less
2277 // than or equal to its super depth are correct.
2278 if (tkls->is_loaded()) {
2279 ciKlass* klass = nullptr;
2313 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2314 // The key property of this type is that it folds up tests
2315 // for array-ness, since it proves that the layout_helper is positive.
2316 // Thus, a generic value like the basic object layout helper works fine.
2317 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2318 }
2319 }
2320
2321 // If we are loading from a freshly-allocated object/array, produce a zero.
2322 // Things to check:
2323 // 1. Load is beyond the header: headers are not guaranteed to be zero
2324 // 2. Load is not vectorized: vectors have no zero constant
2325 // 3. Load has no matching store, i.e. the input is the initial memory state
2326 const TypeOopPtr* tinst = tp->isa_oopptr();
2327 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2328 bool is_not_vect = (_type->isa_vect() == nullptr);
2329 if (is_not_header && is_not_vect) {
2330 Node* mem = in(MemNode::Memory);
2331 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2332 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2333 return Type::get_zero_type(_type->basic_type());
2334 }
2335 }
2336
2337 if (!UseCompactObjectHeaders) {
2338 Node* alloc = is_new_object_mark_load();
2339 if (alloc != nullptr) {
2340 return TypeX::make(markWord::prototype().value());
2341 }
2342 }
2343
2344 return _type;
2345 }
2346
2347 //------------------------------match_edge-------------------------------------
2348 // Do we Match on this edge index or not? Match only the address.
2349 uint LoadNode::match_edge(uint idx) const {
2350 return idx == MemNode::Address;
2351 }
2352
2353 //--------------------------LoadBNode::Ideal--------------------------------------
2354 //
2355 // If the previous store is to the same address as this load,
2356 // and the value stored was larger than a byte, replace this load
2357 // with the value stored truncated to a byte. If no truncation is
2358 // needed, the replacement is done in LoadNode::Identity().
2359 //
2360 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2469 }
2470 }
2471 // Identity call will handle the case where truncation is not needed.
2472 return LoadNode::Ideal(phase, can_reshape);
2473 }
2474
2475 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2476 Node* mem = in(MemNode::Memory);
2477 Node* value = can_see_stored_value_through_membars(mem, phase);
2478 if (value != nullptr && value->is_Con() &&
2479 !value->bottom_type()->higher_equal(_type)) {
2480 // If the input to the store does not fit with the load's result type,
2481 // it must be truncated. We can't delay until Ideal call since
2482 // a singleton Value is needed for split_thru_phi optimization.
2483 int con = value->get_int();
2484 return TypeInt::make((con << 16) >> 16);
2485 }
2486 return LoadNode::Value(phase);
2487 }
2488
2489 //=============================================================================
2490 //----------------------------LoadKlassNode::make------------------------------
2491 // Polymorphic factory method:
2492 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2493 // sanity check the alias category against the created node type
2494 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2495 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2496 #ifdef _LP64
2497 if (adr_type->is_ptr_to_narrowklass()) {
2498 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2499 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2500 }
2501 #endif
2502 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2503 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2504 }
2505
2506 //------------------------------Value------------------------------------------
2507 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2508 return klass_value_common(phase);
2541 }
2542 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2543 }
2544 if (!t->is_klass()) {
2545 // a primitive Class (e.g., int.class) has null for a klass field
2546 return TypePtr::NULL_PTR;
2547 }
2548 // Fold up the load of the hidden field
2549 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2550 }
2551 // non-constant mirror, so we can't tell what's going on
2552 }
2553 if (!tinst->is_loaded())
2554 return _type; // Bail out if not loaded
2555 if (offset == oopDesc::klass_offset_in_bytes()) {
2556 return tinst->as_klass_type(true);
2557 }
2558 }
2559
2560 // Check for loading klass from an array
2561 const TypeAryPtr *tary = tp->isa_aryptr();
2562 if (tary != nullptr &&
2563 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2564 return tary->as_klass_type(true);
2565 }
2566
2567 // Check for loading klass from an array klass
2568 const TypeKlassPtr *tkls = tp->isa_klassptr();
2569 if (tkls != nullptr && !StressReflectiveCode) {
2570 if (!tkls->is_loaded())
2571 return _type; // Bail out if not loaded
2572 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2573 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2574 // // Always returning precise element type is incorrect,
2575 // // e.g., element type could be object and array may contain strings
2576 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2577
2578 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2579 // according to the element type's subclassing.
2580 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2581 }
2582 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2583 tkls->offset() == in_bytes(Klass::super_offset())) {
2584 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2585 // The field is Klass::_super. Return its (constant) value.
2586 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2587 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2588 }
2589 }
2590
2591 if (tkls != nullptr && !UseSecondarySupersCache
2592 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2593 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2594 return TypePtr::NULL_PTR;
2595 }
2596
2597 // Bailout case
2598 return LoadNode::Value(phase);
2599 }
2600
2601 //------------------------------Identity---------------------------------------
2624 base = bs->step_over_gc_barrier(base);
2625 }
2626
2627 // We can fetch the klass directly through an AllocateNode.
2628 // This works even if the klass is not constant (clone or newArray).
2629 if (offset == oopDesc::klass_offset_in_bytes()) {
2630 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2631 if (allocated_klass != nullptr) {
2632 return allocated_klass;
2633 }
2634 }
2635
2636 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2637 // See inline_native_Class_query for occurrences of these patterns.
2638 // Java Example: x.getClass().isAssignableFrom(y)
2639 //
2640 // This improves reflective code, often making the Class
2641 // mirror go completely dead. (Current exception: Class
2642 // mirrors may appear in debug info, but we could clean them out by
2643 // introducing a new debug info operator for Klass.java_mirror).
2644
2645 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2646 && offset == java_lang_Class::klass_offset()) {
2647 if (base->is_Load()) {
2648 Node* base2 = base->in(MemNode::Address);
2649 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2650 Node* adr2 = base2->in(MemNode::Address);
2651 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2652 if (tkls != nullptr && !tkls->empty()
2653 && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2654 && adr2->is_AddP()
2655 ) {
2656 int mirror_field = in_bytes(Klass::java_mirror_offset());
2657 if (tkls->offset() == mirror_field) {
2658 #ifdef ASSERT
2659 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2660 assert(tkls2->offset() == 0, "not a load of java_mirror");
2661 #endif
2662 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2663 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2664 return adr2->in(AddPNode::Address);
2665 }
2666 }
2667 }
2668 }
2669 }
2670
2671 return this;
2672 }
2673
2674 LoadNode* LoadNode::clone_pinned() const {
2675 LoadNode* ld = clone()->as_Load();
2802 // Polymorphic factory method:
2803 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2804 assert((mo == unordered || mo == release), "unexpected");
2805 Compile* C = gvn.C;
2806 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
2807 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2808 ctl != nullptr, "raw memory operations should have control edge");
2809
2810 switch (bt) {
2811 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2812 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2813 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2814 case T_CHAR:
2815 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2816 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2817 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2818 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2819 case T_METADATA:
2820 case T_ADDRESS:
2821 case T_OBJECT:
2822 #ifdef _LP64
2823 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2824 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2825 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2826 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2827 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
2828 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2829 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2830 }
2831 #endif
2832 {
2833 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2834 }
2835 default:
2836 ShouldNotReachHere();
2837 return (StoreNode*)nullptr;
2838 }
2839 }
2840
2841 //--------------------------bottom_type----------------------------------------
2842 const Type *StoreNode::bottom_type() const {
2843 return Type::MEMORY;
2844 }
2845
2846 //------------------------------hash-------------------------------------------
2847 uint StoreNode::hash() const {
2848 // unroll addition of interesting fields
2849 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2850
2851 // Since they are not commoned, do not hash them:
2852 return NO_HASH;
2853 }
2854
2855 // Link together multiple stores (B/S/C/I) into a longer one.
2856 //
3478 }
3479 ss.print_cr("[TraceMergeStores]: with");
3480 merged_input_value->dump("\n", false, &ss);
3481 merged_store->dump("\n", false, &ss);
3482 tty->print("%s", ss.as_string());
3483 }
3484 #endif
3485
3486 //------------------------------Ideal------------------------------------------
3487 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3488 // When a store immediately follows a relevant allocation/initialization,
3489 // try to capture it into the initialization, or hoist it above.
3490 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3491 Node* p = MemNode::Ideal_common(phase, can_reshape);
3492 if (p) return (p == NodeSentinel) ? nullptr : p;
3493
3494 Node* mem = in(MemNode::Memory);
3495 Node* address = in(MemNode::Address);
3496 Node* value = in(MemNode::ValueIn);
3497 // Back-to-back stores to same address? Fold em up. Generally
3498 // unsafe if I have intervening uses.
3499 {
3500 Node* st = mem;
3501 // If Store 'st' has more than one use, we cannot fold 'st' away.
3502 // For example, 'st' might be the final state at a conditional
3503 // return. Or, 'st' might be used by some node which is live at
3504 // the same time 'st' is live, which might be unschedulable. So,
3505 // require exactly ONE user until such time as we clone 'mem' for
3506 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3507 // true).
3508 while (st->is_Store() && st->outcnt() == 1) {
3509 // Looking at a dead closed cycle of memory?
3510 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3511 assert(Opcode() == st->Opcode() ||
3512 st->Opcode() == Op_StoreVector ||
3513 Opcode() == Op_StoreVector ||
3514 st->Opcode() == Op_StoreVectorScatter ||
3515 Opcode() == Op_StoreVectorScatter ||
3516 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3517 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3518 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3519 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3520 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3521
3522 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3523 st->as_Store()->memory_size() <= this->memory_size()) {
3524 Node* use = st->raw_out(0);
3525 if (phase->is_IterGVN()) {
3526 phase->is_IterGVN()->rehash_node_delayed(use);
3527 }
3528 // It's OK to do this in the parser, since DU info is always accurate,
3529 // and the parser always refers to nodes via SafePointNode maps.
3530 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3531 return this;
3532 }
3533 st = st->in(MemNode::Memory);
3534 }
3535 }
3536
3537
3538 // Capture an unaliased, unconditional, simple store into an initializer.
3639 const StoreVectorNode* store_vector = as_StoreVector();
3640 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3641 const Node* store_indices = store_vector->indices();
3642 const Node* mem_indices = mem_vector->indices();
3643 const Node* store_mask = store_vector->mask();
3644 const Node* mem_mask = mem_vector->mask();
3645 // Ensure types, indices, and masks match
3646 if (store_vector->vect_type() == mem_vector->vect_type() &&
3647 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3648 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3649 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3650 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3651 result = mem;
3652 }
3653 }
3654 }
3655
3656 // Store of zero anywhere into a freshly-allocated object?
3657 // Then the store is useless.
3658 // (It must already have been captured by the InitializeNode.)
3659 if (result == this &&
3660 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3661 // a newly allocated object is already all-zeroes everywhere
3662 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3663 result = mem;
3664 }
3665
3666 if (result == this) {
3667 // the store may also apply to zero-bits in an earlier object
3668 Node* prev_mem = find_previous_store(phase);
3669 // Steps (a), (b): Walk past independent stores to find an exact match.
3670 if (prev_mem != nullptr) {
3671 if (prev_mem->is_top()) {
3672 // find_previous_store returns top when the access is dead
3673 return prev_mem;
3674 }
3675 Node* prev_val = can_see_stored_value(prev_mem, phase);
3676 if (prev_val != nullptr && prev_val == val) {
3677 // prev_val and val might differ by a cast; it would be good
3678 // to keep the more informative of the two.
3679 result = mem;
3680 }
3681 }
3682 }
3683 }
3684
3685 PhaseIterGVN* igvn = phase->is_IterGVN();
3686 if (result != this && igvn != nullptr) {
4159 // Clearing a short array is faster with stores
4160 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4161 // Already know this is a large node, do not try to ideal it
4162 if (_is_large) return nullptr;
4163
4164 const int unit = BytesPerLong;
4165 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4166 if (!t) return nullptr;
4167 if (!t->is_con()) return nullptr;
4168 intptr_t raw_count = t->get_con();
4169 intptr_t size = raw_count;
4170 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4171 // Clearing nothing uses the Identity call.
4172 // Negative clears are possible on dead ClearArrays
4173 // (see jck test stmt114.stmt11402.val).
4174 if (size <= 0 || size % unit != 0) return nullptr;
4175 intptr_t count = size / unit;
4176 // Length too long; communicate this to matchers and assemblers.
4177 // Assemblers are responsible to produce fast hardware clears for it.
4178 if (size > InitArrayShortSize) {
4179 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4180 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4181 return nullptr;
4182 }
4183 if (!IdealizeClearArrayNode) return nullptr;
4184 Node *mem = in(1);
4185 if( phase->type(mem)==Type::TOP ) return nullptr;
4186 Node *adr = in(3);
4187 const Type* at = phase->type(adr);
4188 if( at==Type::TOP ) return nullptr;
4189 const TypePtr* atp = at->isa_ptr();
4190 // adjust atp to be the correct array element address type
4191 if (atp == nullptr) atp = TypePtr::BOTTOM;
4192 else atp = atp->add_offset(Type::OffsetBot);
4193 // Get base for derived pointer purposes
4194 if( adr->Opcode() != Op_AddP ) Unimplemented();
4195 Node *base = adr->in(1);
4196
4197 Node *zero = phase->makecon(TypeLong::ZERO);
4198 Node *off = phase->MakeConX(BytesPerLong);
4199 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4200 count--;
4201 while (count--) {
4202 mem = phase->transform(mem);
4203 adr = phase->transform(AddPNode::make_with_base(base, adr, off));
4204 mem = new StoreLNode(in(0), mem, adr, atp, zero, MemNode::unordered, false);
4205 }
4206 return mem;
4207 }
4208
4209 //----------------------------step_through----------------------------------
4210 // Return allocation input memory edge if it is different instance
4211 // or itself if it is the one we are looking for.
4212 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4213 Node* n = *np;
4214 assert(n->is_ClearArray(), "sanity");
4215 intptr_t offset;
4216 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4217 // This method is called only before Allocate nodes are expanded
4218 // during macro nodes expansion. Before that ClearArray nodes are
4219 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4220 // Allocate nodes are expanded) which follows allocations.
4221 assert(alloc != nullptr, "should have allocation");
4222 if (alloc->_idx == instance_id) {
4223 // Can not bypass initialization of the instance we are looking for.
4224 return false;
4227 InitializeNode* init = alloc->initialization();
4228 if (init != nullptr)
4229 *np = init->in(TypeFunc::Memory);
4230 else
4231 *np = alloc->in(TypeFunc::Memory);
4232 return true;
4233 }
4234
4235 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4236 Node* base = dest;
4237 if (raw_base) {
4238 // May be called as part of the initialization of a just allocated object
4239 base = phase->C->top();
4240 }
4241 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4242 }
4243
4244 //----------------------------clear_memory-------------------------------------
4245 // Generate code to initialize object storage to zero.
4246 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4247 intptr_t start_offset,
4248 Node* end_offset,
4249 bool raw_base,
4250 PhaseGVN* phase) {
4251 intptr_t offset = start_offset;
4252
4253 int unit = BytesPerLong;
4254 if ((offset % unit) != 0) {
4255 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4256 const TypePtr* atp = TypeRawPtr::BOTTOM;
4257 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4258 mem = phase->transform(mem);
4259 offset += BytesPerInt;
4260 }
4261 assert((offset % unit) == 0, "");
4262
4263 // Initialize the remaining stuff, if any, with a ClearArray.
4264 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, raw_base, phase);
4265 }
4266
4267 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4268 Node* start_offset,
4269 Node* end_offset,
4270 bool raw_base,
4271 PhaseGVN* phase) {
4272 if (start_offset == end_offset) {
4273 // nothing to do
4274 return mem;
4275 }
4276
4277 int unit = BytesPerLong;
4278 Node* zbase = start_offset;
4279 Node* zend = end_offset;
4280
4281 // Scale to the unit required by the CPU:
4282 if (!Matcher::init_array_count_is_in_bytes) {
4283 Node* shift = phase->intcon(exact_log2(unit));
4284 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4285 zend = phase->transform(new URShiftXNode(zend, shift) );
4286 }
4287
4288 // Bulk clear double-words
4289 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4290 Node* adr = make_address(dest, start_offset, raw_base, phase);
4291 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4292 return phase->transform(mem);
4293 }
4294
4295 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4296 intptr_t start_offset,
4297 intptr_t end_offset,
4298 bool raw_base,
4299 PhaseGVN* phase) {
4300 if (start_offset == end_offset) {
4301 // nothing to do
4302 return mem;
4303 }
4304
4305 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4306 intptr_t done_offset = end_offset;
4307 if ((done_offset % BytesPerLong) != 0) {
4308 done_offset -= BytesPerInt;
4309 }
4310 if (done_offset > start_offset) {
4311 mem = clear_memory(ctl, mem, dest,
4312 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4313 }
4314 if (done_offset < end_offset) { // emit the final 32-bit store
4315 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4316 const TypePtr* atp = TypeRawPtr::BOTTOM;
4317 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4318 mem = phase->transform(mem);
4319 done_offset += BytesPerInt;
4320 }
4321 assert(done_offset == end_offset, "");
4322 return mem;
4323 }
4324
4325 //=============================================================================
4326 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4327 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4328 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4329 #ifdef ASSERT
4330 , _pair_idx(0)
4331 #endif
4332 {
4333 init_class_id(Class_MemBar);
4334 Node* top = C->top();
4335 init_req(TypeFunc::I_O,top);
4336 init_req(TypeFunc::FramePtr,top);
4337 init_req(TypeFunc::ReturnAdr,top);
4446 PhaseIterGVN* igvn = phase->is_IterGVN();
4447 remove(igvn);
4448 // Must return either the original node (now dead) or a new node
4449 // (Do not return a top here, since that would break the uniqueness of top.)
4450 return new ConINode(TypeInt::ZERO);
4451 }
4452 }
4453 return progress ? this : nullptr;
4454 }
4455
4456 //------------------------------Value------------------------------------------
4457 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4458 if( !in(0) ) return Type::TOP;
4459 if( phase->type(in(0)) == Type::TOP )
4460 return Type::TOP;
4461 return TypeTuple::MEMBAR;
4462 }
4463
4464 //------------------------------match------------------------------------------
4465 // Construct projections for memory.
4466 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4467 switch (proj->_con) {
4468 case TypeFunc::Control:
4469 case TypeFunc::Memory:
4470 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4471 }
4472 ShouldNotReachHere();
4473 return nullptr;
4474 }
4475
4476 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4477 trailing->_kind = TrailingStore;
4478 leading->_kind = LeadingStore;
4479 #ifdef ASSERT
4480 trailing->_pair_idx = leading->_idx;
4481 leading->_pair_idx = leading->_idx;
4482 #endif
4483 }
4484
4485 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4486 trailing->_kind = TrailingLoadStore;
4733 return (req() > RawStores);
4734 }
4735
4736 void InitializeNode::set_complete(PhaseGVN* phase) {
4737 assert(!is_complete(), "caller responsibility");
4738 _is_complete = Complete;
4739
4740 // After this node is complete, it contains a bunch of
4741 // raw-memory initializations. There is no need for
4742 // it to have anything to do with non-raw memory effects.
4743 // Therefore, tell all non-raw users to re-optimize themselves,
4744 // after skipping the memory effects of this initialization.
4745 PhaseIterGVN* igvn = phase->is_IterGVN();
4746 if (igvn) igvn->add_users_to_worklist(this);
4747 }
4748
4749 // convenience function
4750 // return false if the init contains any stores already
4751 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4752 InitializeNode* init = initialization();
4753 if (init == nullptr || init->is_complete()) return false;
4754 init->remove_extra_zeroes();
4755 // for now, if this allocation has already collected any inits, bail:
4756 if (init->is_non_zero()) return false;
4757 init->set_complete(phase);
4758 return true;
4759 }
4760
4761 void InitializeNode::remove_extra_zeroes() {
4762 if (req() == RawStores) return;
4763 Node* zmem = zero_memory();
4764 uint fill = RawStores;
4765 for (uint i = fill; i < req(); i++) {
4766 Node* n = in(i);
4767 if (n->is_top() || n == zmem) continue; // skip
4768 if (fill < i) set_req(fill, n); // compact
4769 ++fill;
4770 }
4771 // delete any empty spaces created:
4772 while (fill < req()) {
4773 del_req(fill);
4917 // store node that we'd like to capture. We need to check
4918 // the uses of the MergeMemNode.
4919 mems.push(n);
4920 }
4921 } else if (n->is_Mem()) {
4922 Node* other_adr = n->in(MemNode::Address);
4923 if (other_adr == adr) {
4924 failed = true;
4925 break;
4926 } else {
4927 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4928 if (other_t_adr != nullptr) {
4929 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4930 if (other_alias_idx == alias_idx) {
4931 // A load from the same memory slice as the store right
4932 // after the InitializeNode. We check the control of the
4933 // object/array that is loaded from. If it's the same as
4934 // the store control then we cannot capture the store.
4935 assert(!n->is_Store(), "2 stores to same slice on same control?");
4936 Node* base = other_adr;
4937 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4938 base = base->in(AddPNode::Base);
4939 if (base != nullptr) {
4940 base = base->uncast();
4941 if (base->is_Proj() && base->in(0) == alloc) {
4942 failed = true;
4943 break;
4944 }
4945 }
4946 }
4947 }
4948 }
4949 } else {
4950 failed = true;
4951 break;
4952 }
4953 }
4954 }
4955 }
4956 if (failed) {
5502 // z's_done 12 16 16 16 12 16 12
5503 // z's_needed 12 16 16 16 16 16 16
5504 // zsize 0 0 0 0 4 0 4
5505 if (next_full_store < 0) {
5506 // Conservative tack: Zero to end of current word.
5507 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5508 } else {
5509 // Zero to beginning of next fully initialized word.
5510 // Or, don't zero at all, if we are already in that word.
5511 assert(next_full_store >= zeroes_needed, "must go forward");
5512 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5513 zeroes_needed = next_full_store;
5514 }
5515 }
5516
5517 if (zeroes_needed > zeroes_done) {
5518 intptr_t zsize = zeroes_needed - zeroes_done;
5519 // Do some incremental zeroing on rawmem, in parallel with inits.
5520 zeroes_done = align_down(zeroes_done, BytesPerInt);
5521 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5522 zeroes_done, zeroes_needed,
5523 true,
5524 phase);
5525 zeroes_done = zeroes_needed;
5526 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5527 do_zeroing = false; // leave the hole, next time
5528 }
5529 }
5530
5531 // Collect the store and move on:
5532 phase->replace_input_of(st, MemNode::Memory, inits);
5533 inits = st; // put it on the linearized chain
5534 set_req(i, zmem); // unhook from previous position
5535
5536 if (zeroes_done == st_off)
5537 zeroes_done = next_init_off;
5538
5539 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5540
5541 #ifdef ASSERT
5562 remove_extra_zeroes(); // clear out all the zmems left over
5563 add_req(inits);
5564
5565 if (!(UseTLAB && ZeroTLAB)) {
5566 // If anything remains to be zeroed, zero it all now.
5567 zeroes_done = align_down(zeroes_done, BytesPerInt);
5568 // if it is the last unused 4 bytes of an instance, forget about it
5569 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5570 if (zeroes_done + BytesPerLong >= size_limit) {
5571 AllocateNode* alloc = allocation();
5572 assert(alloc != nullptr, "must be present");
5573 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5574 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5575 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5576 if (zeroes_done == k->layout_helper())
5577 zeroes_done = size_limit;
5578 }
5579 }
5580 if (zeroes_done < size_limit) {
5581 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5582 zeroes_done, size_in_bytes, true, phase);
5583 }
5584 }
5585
5586 set_complete(phase);
5587 return rawmem;
5588 }
5589
5590 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5591 auto replace_proj = [&](ProjNode* proj) {
5592 C->gvn_replace_by(proj, mem);
5593 return CONTINUE;
5594 };
5595 apply_to_projs(replace_proj, TypeFunc::Memory);
5596 }
5597
5598 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5599 DUIterator_Fast imax, i = fast_outs(imax);
5600 auto replace_proj = [&](ProjNode* proj) {
5601 igvn->replace_node(proj, mem);
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "ci/ciInlineKlass.hpp"
28 #include "classfile/javaClasses.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "classfile/vmIntrinsics.hpp"
31 #include "compiler/compileLog.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/c2/barrierSetC2.hpp"
34 #include "gc/shared/tlab_globals.hpp"
35 #include "memory/allocation.inline.hpp"
36 #include "memory/resourceArea.hpp"
37 #include "oops/flatArrayKlass.hpp"
38 #include "oops/objArrayKlass.hpp"
39 #include "opto/addnode.hpp"
40 #include "opto/arraycopynode.hpp"
41 #include "opto/callnode.hpp"
42 #include "opto/cfgnode.hpp"
43 #include "opto/compile.hpp"
44 #include "opto/connode.hpp"
45 #include "opto/convertnode.hpp"
46 #include "opto/inlinetypenode.hpp"
47 #include "opto/loopnode.hpp"
48 #include "opto/machnode.hpp"
49 #include "opto/matcher.hpp"
50 #include "opto/memnode.hpp"
51 #include "opto/mempointer.hpp"
52 #include "opto/mulnode.hpp"
53 #include "opto/narrowptrnode.hpp"
54 #include "opto/opcodes.hpp"
55 #include "opto/phaseX.hpp"
56 #include "opto/regalloc.hpp"
57 #include "opto/regmask.hpp"
58 #include "opto/rootnode.hpp"
59 #include "opto/traceMergeStoresTag.hpp"
60 #include "opto/vectornode.hpp"
61 #include "runtime/arguments.hpp"
62 #include "utilities/align.hpp"
63 #include "utilities/copy.hpp"
64 #include "utilities/globalDefinitions.hpp"
65 #include "utilities/macros.hpp"
66 #include "utilities/powerOfTwo.hpp"
67 #include "utilities/vmError.hpp"
68
69 // Portions of code courtesy of Clifford Click
70
71 // Optimization - Graph Style
72
73 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
74
75 //=============================================================================
76 uint MemNode::size_of() const { return sizeof(*this); }
77
78 const TypePtr *MemNode::adr_type() const {
79 Node* adr = in(Address);
80 if (adr == nullptr) return nullptr; // node is dead
81 const TypePtr* cross_check = nullptr;
82 DEBUG_ONLY(cross_check = _adr_type);
83 return calculate_adr_type(adr->bottom_type(), cross_check);
84 }
133 st->print(", idx=Bot;");
134 else if (atp->index() == Compile::AliasIdxTop)
135 st->print(", idx=Top;");
136 else if (atp->index() == Compile::AliasIdxRaw)
137 st->print(", idx=Raw;");
138 else {
139 ciField* field = atp->field();
140 if (field) {
141 st->print(", name=");
142 field->print_name_on(st);
143 }
144 st->print(", idx=%d;", atp->index());
145 }
146 }
147 }
148
149 extern void print_alias_types();
150
151 #endif
152
153 // Find the memory output corresponding to the fall-through path of a call
154 static Node* find_call_fallthrough_mem_output(CallNode* call) {
155 ResourceMark rm;
156 CallProjections* projs = call->extract_projections(false, false);
157 Node* res = projs->fallthrough_memproj;
158 assert(res != nullptr, "must have a fallthrough mem output");
159 return res;
160 }
161
162 // Try to find a better memory input for a load from a strict final field
163 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
164 intptr_t offset = 0;
165 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
166 if (base == nullptr) {
167 return nullptr;
168 }
169
170 Node* base_uncasted = base->uncast();
171 if (base_uncasted->is_Proj()) {
172 Node* multi = base_uncasted->in(0);
173 if (multi->is_top()) {
174 // The pointer dies, make the memory die, too
175 return multi;
176 } else if (multi->is_Allocate()) {
177 base_local = base_uncasted->as_Proj();
178 return nullptr;
179 } else if (multi->is_Call()) {
180 if (!multi->is_CallJava() || multi->as_CallJava()->method() == nullptr || !multi->as_CallJava()->method()->return_value_is_larval()) {
181 // The oop is returned from a call, the memory can be the fallthrough output of the call
182 return find_call_fallthrough_mem_output(multi->as_Call());
183 }
184 } else if (multi->is_Start()) {
185 // The oop is a parameter
186 if (base_uncasted->as_Proj()->_con == TypeFunc::Parms && phase->C->method()->receiver_maybe_larval()) {
187 // The receiver of a constructor is similar to the result of an AllocateNode
188 base_local = base_uncasted->as_Proj();
189 return nullptr;
190 } else {
191 // Use the start memory otherwise
192 return multi->as_Start()->proj_out(TypeFunc::Memory);
193 }
194 }
195 }
196
197 return nullptr;
198 }
199
200 // Whether a call can modify a strict final field, given that the object is allocated inside the
201 // current compilation unit, or is the first parameter when the compilation root is a constructor.
202 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
203 // the target method is a subclass of the class declaring 'field'.
204 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
205 if (!call->is_CallJava()) {
206 return false;
207 }
208
209 ciMethod* target = call->as_CallJava()->method();
210 if (target == nullptr) {
211 return false;
212 } else if (target->intrinsic_id() == vmIntrinsicID::_linkToSpecial) {
213 // linkToSpecial can be used to call a constructor, used in the construction of objects in the
214 // reflection API
215 return true;
216 } else if (!target->is_object_constructor()) {
217 return false;
218 }
219
220 // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
221 // then the field is set inside the constructor, else the field must be set before the
222 // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
223 // while a field Sub.y must be set before Super::<init> is invoked.
224 // We can try to be more heroic and decide if the receiver of the constructor invocation is the
225 // object from which we are loading from. This, however, may be problematic as deciding if 2
226 // nodes are definitely different may not be trivial, especially if the graph is not canonical.
227 // As a result, it is made more conservative for now.
228 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
229 return target->holder()->is_subclass_of(field->holder());
230 }
231
232 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
233 assert(t_oop != nullptr, "sanity");
234 bool is_known_instance = t_oop->is_known_instance_field();
235 bool is_strict_final_load = false;
236
237 // After macro expansion, an allocation may become a call, changing the memory input to the
238 // memory output of that call would be illegal. As a result, disallow this transformation after
239 // macro expansion.
240 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
241 is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
242 #ifdef ASSERT
243 if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
244 assert(is_strict_final_load, "sanity check for basic cases");
245 }
246 #endif // ASSERT
247 }
248
249 if (!is_known_instance && !is_strict_final_load) {
250 return mchain;
251 }
252
253 Node* result = mchain;
254 ProjNode* base_local = nullptr;
255
256 ciField* field = nullptr;
257 if (is_strict_final_load) {
258 field = phase->C->alias_type(t_oop)->field();
259 assert(field != nullptr, "must point to a field");
260
261 Node* adr = load->in(MemNode::Address);
262 assert(phase->type(adr) == t_oop, "inconsistent type");
263 Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
264 if (tmp != nullptr) {
265 result = tmp;
266 }
267 }
268
269 uint instance_id = t_oop->instance_id();
270 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
271 Node* prev = nullptr;
272 while (prev != result) {
273 prev = result;
274 if (result == start_mem) {
275 // start_mem is the earliest memory possible
276 break;
277 }
278
279 // skip over a call which does not affect this memory slice
280 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
281 Node* proj_in = result->in(0);
282 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
283 // This is the allocation that creates the object from which we are loading from
284 break;
285 } else if (proj_in->is_Call()) {
286 // ArrayCopyNodes processed here as well
287 CallNode* call = proj_in->as_Call();
288 if (!call->may_modify(t_oop, phase)) {
289 result = call->in(TypeFunc::Memory);
290 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
291 result = call->in(TypeFunc::Memory);
292 }
293 } else if (proj_in->Opcode() == Op_Tuple) {
294 // The call will be folded, skip over it.
295 break;
296 } else if (proj_in->is_Initialize()) {
297 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
298 // Stop if this is the initialization for the object instance which
299 // contains this memory slice, otherwise skip over it.
300 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
301 break;
302 }
303 if (is_known_instance) {
304 result = proj_in->in(TypeFunc::Memory);
305 } else if (is_strict_final_load) {
306 Node* klass = alloc->in(AllocateNode::KlassNode);
307 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
308 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
309 // Allocation of another type, must be another object
310 result = proj_in->in(TypeFunc::Memory);
311 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
312 // Allocation of another object
313 result = proj_in->in(TypeFunc::Memory);
314 }
315 }
316 } else if (proj_in->is_MemBar()) {
317 ArrayCopyNode* ac = nullptr;
318 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
319 break;
320 }
321 result = proj_in->in(TypeFunc::Memory);
322 } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
323 if (is_strict_final_load) {
324 // LoadFlat and StoreFlat cannot happen to strict final fields
325 result = proj_in->in(TypeFunc::Memory);
326 }
327 } else if (proj_in->is_top()) {
328 break; // dead code
329 } else {
330 assert(false, "unexpected projection of %s", proj_in->Name());
331 }
332 } else if (result->is_ClearArray()) {
333 if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
334 // Can not bypass initialization of the instance
335 // we are looking for.
336 break;
337 }
338 // Otherwise skip it (the call updated 'result' value).
339 } else if (result->is_MergeMem()) {
340 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
341 }
342 }
343 return result;
344 }
345
346 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
347 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
348 if (t_oop == nullptr)
349 return mchain; // don't try to optimize non-oop types
350 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
351 bool is_instance = t_oop->is_known_instance_field();
352 PhaseIterGVN *igvn = phase->is_IterGVN();
353 if (is_instance && igvn != nullptr && result->is_Phi()) {
354 PhiNode *mphi = result->as_Phi();
355 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
356 const TypePtr *t = mphi->adr_type();
357 bool do_split = false;
358 // In the following cases, Load memory input can be further optimized based on
359 // its precise address type
360 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
361 do_split = true;
362 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
363 const TypeOopPtr* mem_t =
364 t->is_oopptr()->cast_to_exactness(true)
365 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
366 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
367 if (t_oop->isa_aryptr()) {
368 mem_t = mem_t->is_aryptr()
369 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
370 ->cast_to_size(t_oop->is_aryptr()->size())
371 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
372 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
373 ->with_offset(t_oop->is_aryptr()->offset())
374 ->is_aryptr();
375 }
376 do_split = mem_t == t_oop;
377 }
378 if (do_split) {
379 // clone the Phi with our address type
380 result = mphi->split_out_instance(t_adr, igvn);
381 } else {
382 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
383 }
384 }
385 return result;
386 }
387
388 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
389 uint alias_idx = phase->C->get_alias_index(tp);
390 Node *mem = mmem;
391 #ifdef ASSERT
392 {
393 // Check that current type is consistent with the alias index used during graph construction
394 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
395 bool consistent = adr_check == nullptr || adr_check->empty() ||
396 phase->C->must_alias(adr_check, alias_idx );
397 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
398 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
399 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
400 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
401 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
402 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
403 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
404 // don't assert if it is dead code.
405 consistent = true;
406 }
407 if( !consistent ) {
408 st->print("alias_idx==%d, adr_check==", alias_idx);
409 if( adr_check == nullptr ) {
410 st->print("null");
411 } else {
412 adr_check->dump();
413 }
414 st->cr();
415 print_alias_types();
416 assert(consistent, "adr_check must match alias idx");
417 }
418 }
419 #endif
1129 in_bytes(JavaThread::vthread_offset()),
1130 in_bytes(JavaThread::scopedValueCache_offset()),
1131 };
1132
1133 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1134 if (offset == offsets[i]) {
1135 return true;
1136 }
1137 }
1138 }
1139
1140 return false;
1141 }
1142 #endif
1143
1144 //----------------------------LoadNode::make-----------------------------------
1145 // Polymorphic factory method:
1146 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1147 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1148 Compile* C = gvn.C;
1149 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr(), true) == C->get_alias_index(adr_type, true), "adr and adr_type must agree");
1150
1151 // sanity check the alias category against the created node type
1152 assert(!(adr_type->isa_oopptr() &&
1153 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1154 "use LoadKlassNode instead");
1155 assert(!(adr_type->isa_aryptr() &&
1156 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1157 "use LoadRangeNode instead");
1158 // Check control edge of raw loads
1159 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1160 // oop will be recorded in oop map if load crosses safepoint
1161 rt->isa_oopptr() || is_immutable_value(adr),
1162 "raw memory operations should have control edge");
1163 LoadNode* load = nullptr;
1164 switch (bt) {
1165 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1166 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1167 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1168 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1169 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1170 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1171 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1172 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1173 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1174 case T_ARRAY:
1175 case T_OBJECT:
1176 case T_NARROWOOP:
1177 #ifdef _LP64
1178 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1179 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1180 } else
1181 #endif
1182 {
1183 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1184 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1185 }
1186 break;
1187 default:
1188 guarantee(false, "unexpected basic type %s", type2name(bt));
1189 break;
1190 }
1191 assert(load != nullptr, "LoadNode should have been created");
1192 if (unaligned) {
1193 load->set_unaligned_access();
1194 }
1195 if (mismatched) {
1196 load->set_mismatched_access();
1197 }
1198 if (unsafe) {
1199 load->set_unsafe_access();
1200 }
1201 load->set_barrier_data(barrier_data);
1202 if (load->Opcode() == Op_LoadN) {
1203 Node* ld = gvn.transform(load);
1204 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1205 }
1206
1207 return load;
1208 }
1209
1210 //------------------------------hash-------------------------------------------
1211 uint LoadNode::hash() const {
1212 // unroll addition of interesting fields
1213 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1214 }
1215
1216 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1217 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1218 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1219 bool is_stable_ary = FoldStableValues &&
1220 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1221 tp->isa_aryptr()->is_stable();
1222
1223 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1224 }
1225
1226 return false;
1227 }
1228
1229 // Is the value loaded previously stored by an arraycopy? If so return
1230 // a load node that reads from the source array so we may be able to
1231 // optimize out the ArrayCopy node later.
1232 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1233 Node* ld_adr = in(MemNode::Address);
1234 intptr_t ld_off = 0;
1235 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1236 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1237 if (ac != nullptr) {
1238 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1239
1240 Node* mem = ac->in(TypeFunc::Memory);
1241 Node* ctl = ac->in(0);
1242 Node* src = ac->in(ArrayCopyNode::Src);
1243
1251 if (ac->as_ArrayCopy()->is_clonebasic()) {
1252 assert(ld_alloc != nullptr, "need an alloc");
1253 assert(addp->is_AddP(), "address must be addp");
1254 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1255 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1256 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1257 addp->set_req(AddPNode::Base, src);
1258 addp->set_req(AddPNode::Address, src);
1259 } else {
1260 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1261 ac->as_ArrayCopy()->is_copyof_validated() ||
1262 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1263 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1264 addp->set_req(AddPNode::Base, src);
1265 addp->set_req(AddPNode::Address, src);
1266
1267 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1268 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1269 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1270
1271 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1272
1273 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1274 #ifdef _LP64
1275 diff = phase->transform(new ConvI2LNode(diff));
1276 #endif
1277 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1278
1279 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1280 addp->set_req(AddPNode::Offset, offset);
1281 }
1282 addp = phase->transform(addp);
1283 #ifdef ASSERT
1284 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1285 ld->_adr_type = adr_type;
1286 #endif
1287 ld->set_req(MemNode::Address, addp);
1288 ld->set_req(0, ctl);
1289 ld->set_req(MemNode::Memory, mem);
1290 return ld;
1291 }
1292 return nullptr;
1293 }
1294
1295 static Node* see_through_inline_type(PhaseValues* phase, const MemNode* load, Node* base, int offset) {
1296 if (!load->is_mismatched_access() && base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1297 InlineTypeNode* vt = base->as_InlineType();
1298 Node* value = vt->field_value_by_offset(offset, true);
1299 assert(value != nullptr, "must see some value");
1300 return value;
1301 }
1302
1303 return nullptr;
1304 }
1305
1306 // This routine exists to make sure this set of tests is done the same
1307 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1308 // will change the graph shape in a way which makes memory alive twice at the
1309 // same time (uses the Oracle model of aliasing), then some
1310 // LoadXNode::Identity will fold things back to the equivalence-class model
1311 // of aliasing.
1312 // This method may find an unencoded node instead of the corresponding encoded one.
1313 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1314 Node* ld_adr = in(MemNode::Address);
1315 intptr_t ld_off = 0;
1316 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1317 // Try to see through an InlineTypeNode
1318 // LoadN is special because the input is not compressed
1319 if (Opcode() != Op_LoadN) {
1320 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1321 if (value != nullptr) {
1322 return value;
1323 }
1324 }
1325
1326 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1327 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1328
1329 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1330 uint alias_idx = atp->index();
1331 Node* result = nullptr;
1332 Node* current = st;
1333 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1334 // this load. Stop once any other kind of node is encountered.
1335 //
1336 // In principle, folding a load is moving it up until it meets a matching store.
1337 //
1338 // store(ptr, v); store(ptr, v); store(ptr, v);
1339 // membar1; -> membar1; -> load(ptr);
1340 // membar2; load(ptr); membar1;
1341 // load(ptr); membar2; membar2;
1342 //
1343 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1344 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1345 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1425
1426 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1427 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1428 if (in_vt != out_vt) {
1429 return nullptr;
1430 }
1431 }
1432 return st->in(MemNode::ValueIn);
1433 }
1434
1435 // A load from a freshly-created object always returns zero.
1436 // (This can happen after LoadNode::Ideal resets the load's memory input
1437 // to find_captured_store, which returned InitializeNode::zero_memory.)
1438 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1439 (st->in(0) == ld_alloc) &&
1440 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1441 // return a zero value for the load's basic type
1442 // (This is one of the few places where a generic PhaseTransform
1443 // can create new nodes. Think of it as lazily manifesting
1444 // virtually pre-existing constants.)
1445 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1446 if (init_value != nullptr) {
1447 const TypeAryPtr* ld_adr_type = phase->type(ld_adr)->isa_aryptr();
1448 if (ld_adr_type == nullptr) {
1449 return nullptr;
1450 }
1451
1452 // We know that this is not a flat array, the load should return the whole oop
1453 if (ld_adr_type->is_not_flat()) {
1454 return init_value;
1455 }
1456
1457 // If this is a flat array, try to see through init_value
1458 if (init_value->is_EncodeP()) {
1459 init_value = init_value->in(1);
1460 }
1461 if (!init_value->is_InlineType() || ld_adr_type->field_offset() == Type::Offset::bottom) {
1462 return nullptr;
1463 }
1464
1465 ciInlineKlass* vk = phase->type(init_value)->inline_klass();
1466 int field_offset_in_payload = ld_adr_type->field_offset().get();
1467 if (field_offset_in_payload == vk->null_marker_offset_in_payload()) {
1468 return init_value->as_InlineType()->get_null_marker();
1469 } else {
1470 return init_value->as_InlineType()->field_value_by_offset(field_offset_in_payload + vk->payload_offset(), true);
1471 }
1472 }
1473 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1474 if (value_basic_type() != T_VOID) {
1475 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1476 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1477 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1478 // by the ArrayCopyNode.
1479 return phase->zerocon(value_basic_type());
1480 }
1481 } else {
1482 // TODO: materialize all-zero vector constant
1483 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1484 }
1485 }
1486
1487 // A load from an initialization barrier can match a captured store.
1488 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1489 InitializeNode* init = st->in(0)->as_Initialize();
1490 AllocateNode* alloc = init->allocation();
1491 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1492 // examine a captured store value
1493 st = init->find_captured_store(ld_off, memory_size(), phase);
1506 base = bs->step_over_gc_barrier(base);
1507 if (base != nullptr && base->is_Proj() &&
1508 base->as_Proj()->_con == TypeFunc::Parms &&
1509 base->in(0)->is_CallStaticJava() &&
1510 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1511 return base->in(0)->in(TypeFunc::Parms);
1512 }
1513 }
1514
1515 break;
1516 }
1517
1518 return nullptr;
1519 }
1520
1521 //----------------------is_instance_field_load_with_local_phi------------------
1522 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1523 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1524 in(Address)->is_AddP() ) {
1525 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1526 // Only known instances and immutable fields
1527 if( t_oop != nullptr &&
1528 (t_oop->is_ptr_to_strict_final_field() ||
1529 t_oop->is_known_instance_field()) &&
1530 t_oop->offset() != Type::OffsetBot &&
1531 t_oop->offset() != Type::OffsetTop) {
1532 return true;
1533 }
1534 }
1535 return false;
1536 }
1537
1538 //------------------------------Identity---------------------------------------
1539 // Loads are identity if previous store is to same address
1540 Node* LoadNode::Identity(PhaseGVN* phase) {
1541 // If the previous store-maker is the right kind of Store, and the store is
1542 // to the same address, then we are equal to the value stored.
1543 Node* mem = in(Memory);
1544 Node* value = can_see_stored_value_through_membars(mem, phase);
1545 if( value ) {
1546 // byte, short & char stores truncate naturally.
1547 // A load has to load the truncated value which requires
1548 // some sort of masking operation and that requires an
1549 // Ideal call instead of an Identity call.
1550 if (memory_size() < BytesPerInt) {
1551 // If the input to the store does not fit with the load's result type,
1552 // it must be truncated via an Ideal call.
1553 if (!phase->type(value)->higher_equal(phase->type(this)))
1554 return this;
1555 }
1556
1557 if (phase->type(value)->isa_ptr() && phase->type(this)->isa_narrowoop()) {
1558 return this;
1559 }
1560 // (This works even when value is a Con, but LoadNode::Value
1561 // usually runs first, producing the singleton type of the Con.)
1562 if (!has_pinned_control_dependency() || value->is_Con()) {
1563 return value;
1564 } else {
1565 return this;
1566 }
1567 }
1568
1569 if (has_pinned_control_dependency()) {
1570 return this;
1571 }
1572 // Search for an existing data phi which was generated before for the same
1573 // instance's field to avoid infinite generation of phis in a loop.
1574 Node *region = mem->in(0);
1575 if (is_instance_field_load_with_local_phi(region)) {
1576 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1577 int this_index = phase->C->get_alias_index(addr_t);
1578 int this_offset = addr_t->offset();
1579 int this_iid = addr_t->instance_id();
1580 if (!addr_t->is_known_instance() &&
1581 addr_t->is_ptr_to_strict_final_field()) {
1582 // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1583 intptr_t ignore = 0;
1584 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1585 if (base == nullptr) {
1586 return this;
1587 }
1588 this_iid = base->_idx;
1589 }
1590 const Type* this_type = bottom_type();
1591 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1592 Node* phi = region->fast_out(i);
1593 if (phi->is_Phi() && phi != mem &&
1594 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1595 return phi;
1596 }
1597 }
1598 }
1599
1600 return this;
1601 }
1602
1846 Node* in = phi->in(i);
1847 if (in == nullptr || phase->type(in) == Type::TOP)
1848 return false; // Wait stable graph
1849 }
1850 return true;
1851 }
1852
1853 //------------------------------split_through_phi------------------------------
1854 // Check whether a call to 'split_through_phi' would split this load through the
1855 // Phi *base*. This method is essentially a copy of the validations performed
1856 // by 'split_through_phi'. The first use of this method was in EA code as part
1857 // of simplification of allocation merges.
1858 // Some differences from original method (split_through_phi):
1859 // - If base->is_CastPP(): base = base->in(1)
1860 bool LoadNode::can_split_through_phi_base(PhaseGVN* phase) {
1861 Node* mem = in(Memory);
1862 Node* address = in(Address);
1863 intptr_t ignore = 0;
1864 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1865
1866 if (base != nullptr && base->is_CastPP()) {
1867 base = base->in(1);
1868 }
1869
1870 if (req() > 3 || base == nullptr || !base->is_Phi()) {
1871 return false;
1872 }
1873
1874 if (!mem->is_Phi()) {
1875 if (!MemNode::all_controls_dominate(mem, base->in(0))) {
1876 return false;
1877 }
1878 } else if (base->in(0) != mem->in(0)) {
1879 if (!MemNode::all_controls_dominate(mem, base->in(0))) {
1880 return false;
1881 }
1882 }
1883
1884 return true;
1885 }
1886
2133 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2134 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2135
2136 // Skip up past a SafePoint control. Cannot do this for Stores because
2137 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2138 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2139 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2140 !addr_mark &&
2141 (depends_only_on_test() || has_unknown_control_dependency())) {
2142 ctrl = ctrl->in(0);
2143 set_req(MemNode::Control,ctrl);
2144 return this;
2145 }
2146
2147 intptr_t ignore = 0;
2148 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2149 if (base != nullptr
2150 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2151 // Check for useless control edge in some common special cases
2152 if (in(MemNode::Control) != nullptr
2153 // TODO 8350865 Can we re-enable this?
2154 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2155 && can_remove_control()
2156 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2157 && all_controls_dominate(base, phase->C->start())) {
2158 // A method-invariant, non-null address (constant or 'this' argument).
2159 set_req(MemNode::Control, nullptr);
2160 return this;
2161 }
2162 }
2163
2164 Node* mem = in(MemNode::Memory);
2165 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2166
2167 if (can_reshape && (addr_t != nullptr)) {
2168 // try to optimize our memory input
2169 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2170 if (opt_mem != mem) {
2171 set_req_X(MemNode::Memory, opt_mem, phase);
2172 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2173 return this;
2174 }
2287 // No match.
2288 return nullptr;
2289 }
2290
2291 //------------------------------Value-----------------------------------------
2292 const Type* LoadNode::Value(PhaseGVN* phase) const {
2293 // Either input is TOP ==> the result is TOP
2294 Node* mem = in(MemNode::Memory);
2295 const Type *t1 = phase->type(mem);
2296 if (t1 == Type::TOP) return Type::TOP;
2297 Node* adr = in(MemNode::Address);
2298 const TypePtr* tp = phase->type(adr)->isa_ptr();
2299 if (tp == nullptr || tp->empty()) return Type::TOP;
2300 int off = tp->offset();
2301 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2302 Compile* C = phase->C;
2303
2304 // If load can see a previous constant store, use that.
2305 Node* value = can_see_stored_value_through_membars(mem, phase);
2306 if (value != nullptr && value->is_Con()) {
2307 if (phase->type(value)->isa_ptr() && _type->isa_narrowoop()) {
2308 return phase->type(value)->make_narrowoop();
2309 } else {
2310 assert(value->bottom_type()->higher_equal(_type), "sanity");
2311 return phase->type(value);
2312 }
2313 }
2314 // Try to guess loaded type from pointer type
2315 if (tp->isa_aryptr()) {
2316 const TypeAryPtr* ary = tp->is_aryptr();
2317 const Type* t = ary->elem();
2318
2319 // Determine whether the reference is beyond the header or not, by comparing
2320 // the offset against the offset of the start of the array's data.
2321 // Different array types begin at slightly different offsets (12 vs. 16).
2322 // We choose T_BYTE as an example base type that is least restrictive
2323 // as to alignment, which will therefore produce the smallest
2324 // possible base offset.
2325 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2326 const bool off_beyond_header = (off >= min_base_off);
2327
2328 // Try to constant-fold a stable array element.
2329 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2330 // Make sure the reference is not into the header and the offset is constant
2331 ciObject* aobj = ary->const_oop();
2332 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2333 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2334 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off, ary->field_offset().get(),
2335 stable_dimension,
2336 value_basic_type(), is_unsigned());
2337 if (con_type != nullptr) {
2338 return con_type;
2339 }
2340 }
2341 }
2342
2343 // Don't do this for integer types. There is only potential profit if
2344 // the element type t is lower than _type; that is, for int types, if _type is
2345 // more restrictive than t. This only happens here if one is short and the other
2346 // char (both 16 bits), and in those cases we've made an intentional decision
2347 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2348 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2349 //
2350 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2351 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2352 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2353 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2354 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2355 // In fact, that could have been the original type of p1, and p1 could have
2356 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2357 // expression (LShiftL quux 3) independently optimized to the constant 8.
2358 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2359 && (_type->isa_vect() == nullptr)
2360 && !ary->is_flat()
2361 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2362 // t might actually be lower than _type, if _type is a unique
2363 // concrete subclass of abstract class t.
2364 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2365 const Type* jt = t->join_speculative(_type);
2366 // In any case, do not allow the join, per se, to empty out the type.
2367 if (jt->empty() && !t->empty()) {
2368 // This can happen if a interface-typed array narrows to a class type.
2369 jt = _type;
2370 }
2371 #ifdef ASSERT
2372 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2373 // The pointers in the autobox arrays are always non-null
2374 Node* base = adr->in(AddPNode::Base);
2375 if ((base != nullptr) && base->is_DecodeN()) {
2376 // Get LoadN node which loads IntegerCache.cache field
2377 base = base->in(1);
2378 }
2379 if ((base != nullptr) && base->is_Con()) {
2380 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2381 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2382 // It could be narrow oop
2383 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2384 }
2385 }
2386 }
2387 #endif
2388 return jt;
2389 }
2390 }
2391 } else if (tp->base() == Type::InstPtr) {
2392 assert( off != Type::OffsetBot ||
2393 // arrays can be cast to Objects
2394 !tp->isa_instptr() ||
2395 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2396 // Default value load
2397 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2398 // unsafe field access may not have a constant offset
2399 C->has_unsafe_access(),
2400 "Field accesses must be precise" );
2401 // For oop loads, we expect the _type to be precise.
2402
2403 const TypeInstPtr* tinst = tp->is_instptr();
2404 BasicType bt = value_basic_type();
2405
2406 // Fold loads of the field map
2407 if (tinst != nullptr) {
2408 ciInstanceKlass* ik = tinst->instance_klass();
2409 int offset = tinst->offset();
2410 if (ik == phase->C->env()->Class_klass()) {
2411 ciType* t = tinst->java_mirror_type();
2412 if (t != nullptr && t->is_inlinetype() && offset == t->as_inline_klass()->field_map_offset()) {
2413 ciConstant map = t->as_inline_klass()->get_field_map();
2414 bool is_narrow_oop = (bt == T_NARROWOOP);
2415 return Type::make_from_constant(map, true, 1, is_narrow_oop);
2416 }
2417 }
2418 }
2419
2420 // Optimize loads from constant fields.
2421 ciObject* const_oop = tinst->const_oop();
2422 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2423 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2424 if (con_type != nullptr) {
2425 return con_type;
2426 }
2427 }
2428 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2429 assert(off != Type::OffsetBot ||
2430 !tp->isa_instklassptr() ||
2431 // arrays can be cast to Objects
2432 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2433 // also allow array-loading from the primary supertype
2434 // array during subtype checks
2435 Opcode() == Op_LoadKlass,
2436 "Field accesses must be precise");
2437 // For klass/static loads, we expect the _type to be precise
2438 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2439 /* With mirrors being an indirect in the Klass*
2440 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2441 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2442 *
2443 * So check the type and klass of the node before the LoadP.
2450 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2451 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2452 return TypeInstPtr::make(klass->java_mirror());
2453 }
2454 }
2455 }
2456
2457 const TypeKlassPtr *tkls = tp->isa_klassptr();
2458 if (tkls != nullptr) {
2459 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2460 ciKlass* klass = tkls->exact_klass();
2461 // We are loading a field from a Klass metaobject whose identity
2462 // is known at compile time (the type is "exact" or "precise").
2463 // Check for fields we know are maintained as constants by the VM.
2464 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2465 // The field is Klass::_super_check_offset. Return its (constant) value.
2466 // (Folds up type checking code.)
2467 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2468 return TypeInt::make(klass->super_check_offset());
2469 }
2470 if (klass->is_inlinetype() && tkls->offset() == in_bytes(InstanceKlass::acmp_maps_offset_offset())) {
2471 return TypeInt::make(klass->as_inline_klass()->field_map_offset());
2472 }
2473 if (klass->is_obj_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset())) {
2474 // Fold loads from LibraryCallKit::load_default_refined_array_klass
2475 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr();
2476 }
2477 if (klass->is_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::properties_offset())) {
2478 assert(klass->is_type_array_klass() || tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2479 return TypeInt::make((jint)klass->as_array_klass()->properties().value());
2480 }
2481 if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2482 assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2483 return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2484 }
2485 if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2486 // The field is Klass::_prototype_header. Return its (constant) value.
2487 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2488 return TypeX::make(klass->prototype_header());
2489 }
2490 // Compute index into primary_supers array
2491 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2492 // Check for overflowing; use unsigned compare to handle the negative case.
2493 if( depth < ciKlass::primary_super_limit() ) {
2494 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2495 // (Folds up type checking code.)
2496 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2497 ciKlass *ss = klass->super_of_depth(depth);
2498 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2499 }
2500 const Type* aift = load_array_final_field(tkls, klass);
2501 if (aift != nullptr) return aift;
2502 }
2503
2504 // We can still check if we are loading from the primary_supers array at a
2505 // shallow enough depth. Even though the klass is not exact, entries less
2506 // than or equal to its super depth are correct.
2507 if (tkls->is_loaded()) {
2508 ciKlass* klass = nullptr;
2542 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2543 // The key property of this type is that it folds up tests
2544 // for array-ness, since it proves that the layout_helper is positive.
2545 // Thus, a generic value like the basic object layout helper works fine.
2546 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2547 }
2548 }
2549
2550 // If we are loading from a freshly-allocated object/array, produce a zero.
2551 // Things to check:
2552 // 1. Load is beyond the header: headers are not guaranteed to be zero
2553 // 2. Load is not vectorized: vectors have no zero constant
2554 // 3. Load has no matching store, i.e. the input is the initial memory state
2555 const TypeOopPtr* tinst = tp->isa_oopptr();
2556 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2557 bool is_not_vect = (_type->isa_vect() == nullptr);
2558 if (is_not_header && is_not_vect) {
2559 Node* mem = in(MemNode::Memory);
2560 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2561 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2562 // TODO 8350865 Scalar replacement does not work well for flat arrays.
2563 // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2564 // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2565 // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2566 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2567 intptr_t offset = 0;
2568 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2569 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2570 if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2571 return _type;
2572 }
2573 }
2574 return Type::get_zero_type(_type->basic_type());
2575 }
2576 }
2577 if (!UseCompactObjectHeaders) {
2578 Node* alloc = is_new_object_mark_load();
2579 if (alloc != nullptr) {
2580 if (Arguments::is_valhalla_enabled()) {
2581 // The mark word may contain property bits (inline, flat, null-free)
2582 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2583 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2584 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2585 return TypeX::make(tkls->exact_klass()->prototype_header());
2586 }
2587 } else {
2588 return TypeX::make(markWord::prototype().value());
2589 }
2590 }
2591 }
2592
2593 return _type;
2594 }
2595
2596 //------------------------------match_edge-------------------------------------
2597 // Do we Match on this edge index or not? Match only the address.
2598 uint LoadNode::match_edge(uint idx) const {
2599 return idx == MemNode::Address;
2600 }
2601
2602 //--------------------------LoadBNode::Ideal--------------------------------------
2603 //
2604 // If the previous store is to the same address as this load,
2605 // and the value stored was larger than a byte, replace this load
2606 // with the value stored truncated to a byte. If no truncation is
2607 // needed, the replacement is done in LoadNode::Identity().
2608 //
2609 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2718 }
2719 }
2720 // Identity call will handle the case where truncation is not needed.
2721 return LoadNode::Ideal(phase, can_reshape);
2722 }
2723
2724 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2725 Node* mem = in(MemNode::Memory);
2726 Node* value = can_see_stored_value_through_membars(mem, phase);
2727 if (value != nullptr && value->is_Con() &&
2728 !value->bottom_type()->higher_equal(_type)) {
2729 // If the input to the store does not fit with the load's result type,
2730 // it must be truncated. We can't delay until Ideal call since
2731 // a singleton Value is needed for split_thru_phi optimization.
2732 int con = value->get_int();
2733 return TypeInt::make((con << 16) >> 16);
2734 }
2735 return LoadNode::Value(phase);
2736 }
2737
2738 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2739 // Loading from an InlineType, find the input and make an EncodeP
2740 Node* addr = in(Address);
2741 intptr_t offset;
2742 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
2743 Node* value = see_through_inline_type(phase, this, base, offset);
2744 if (value != nullptr) {
2745 return new EncodePNode(value, type());
2746 }
2747
2748 // Can see the corresponding value, may need to add an EncodeP
2749 value = can_see_stored_value(in(Memory), phase);
2750 if (value != nullptr && phase->type(value)->isa_ptr() && type()->isa_narrowoop()) {
2751 return new EncodePNode(value, type());
2752 }
2753
2754 // Identity call will handle the case where EncodeP is unnecessary
2755 return LoadNode::Ideal(phase, can_reshape);
2756 }
2757
2758 //=============================================================================
2759 //----------------------------LoadKlassNode::make------------------------------
2760 // Polymorphic factory method:
2761 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2762 // sanity check the alias category against the created node type
2763 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2764 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2765 #ifdef _LP64
2766 if (adr_type->is_ptr_to_narrowklass()) {
2767 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2768 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2769 }
2770 #endif
2771 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2772 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2773 }
2774
2775 //------------------------------Value------------------------------------------
2776 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2777 return klass_value_common(phase);
2810 }
2811 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2812 }
2813 if (!t->is_klass()) {
2814 // a primitive Class (e.g., int.class) has null for a klass field
2815 return TypePtr::NULL_PTR;
2816 }
2817 // Fold up the load of the hidden field
2818 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2819 }
2820 // non-constant mirror, so we can't tell what's going on
2821 }
2822 if (!tinst->is_loaded())
2823 return _type; // Bail out if not loaded
2824 if (offset == oopDesc::klass_offset_in_bytes()) {
2825 return tinst->as_klass_type(true);
2826 }
2827 }
2828
2829 // Check for loading klass from an array
2830 const TypeAryPtr* tary = tp->isa_aryptr();
2831 if (tary != nullptr &&
2832 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2833 return tary->as_klass_type(true)->is_aryklassptr();
2834 }
2835
2836 // Check for loading klass from an array klass
2837 const TypeKlassPtr *tkls = tp->isa_klassptr();
2838 if (tkls != nullptr && !StressReflectiveCode) {
2839 if (!tkls->is_loaded())
2840 return _type; // Bail out if not loaded
2841 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2842 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2843 // // Always returning precise element type is incorrect,
2844 // // e.g., element type could be object and array may contain strings
2845 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2846
2847 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2848 // according to the element type's subclassing.
2849 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2850 }
2851 if (tkls->isa_aryklassptr() != nullptr && tkls->klass_is_exact() &&
2852 !tkls->exact_klass()->is_type_array_klass() &&
2853 tkls->offset() == in_bytes(Klass::super_offset())) {
2854 // We are loading the super klass of a refined array klass, return the non-refined klass pointer
2855 assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2856 return tkls->is_aryklassptr()->with_offset(0)->cast_to_non_refined();
2857 }
2858 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2859 tkls->offset() == in_bytes(Klass::super_offset())) {
2860 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2861 // The field is Klass::_super. Return its (constant) value.
2862 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2863 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2864 }
2865 }
2866
2867 if (tkls != nullptr && !UseSecondarySupersCache
2868 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2869 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2870 return TypePtr::NULL_PTR;
2871 }
2872
2873 // Bailout case
2874 return LoadNode::Value(phase);
2875 }
2876
2877 //------------------------------Identity---------------------------------------
2900 base = bs->step_over_gc_barrier(base);
2901 }
2902
2903 // We can fetch the klass directly through an AllocateNode.
2904 // This works even if the klass is not constant (clone or newArray).
2905 if (offset == oopDesc::klass_offset_in_bytes()) {
2906 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2907 if (allocated_klass != nullptr) {
2908 return allocated_klass;
2909 }
2910 }
2911
2912 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2913 // See inline_native_Class_query for occurrences of these patterns.
2914 // Java Example: x.getClass().isAssignableFrom(y)
2915 //
2916 // This improves reflective code, often making the Class
2917 // mirror go completely dead. (Current exception: Class
2918 // mirrors may appear in debug info, but we could clean them out by
2919 // introducing a new debug info operator for Klass.java_mirror).
2920 //
2921 // This optimization does not apply to arrays because if k is not a
2922 // constant, it was obtained via load_klass which returns the VM type
2923 // and '.java_mirror.as_klass' should return the Java type instead.
2924
2925 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2926 && offset == java_lang_Class::klass_offset()) {
2927 if (base->is_Load()) {
2928 Node* base2 = base->in(MemNode::Address);
2929 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2930 Node* adr2 = base2->in(MemNode::Address);
2931 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2932 if (tkls != nullptr && !tkls->empty()
2933 && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2934 && adr2->is_AddP()) {
2935 int mirror_field = in_bytes(Klass::java_mirror_offset());
2936 if (tkls->offset() == mirror_field) {
2937 #ifdef ASSERT
2938 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2939 assert(tkls2->offset() == 0, "not a load of java_mirror");
2940 #endif
2941 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2942 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2943 return adr2->in(AddPNode::Address);
2944 }
2945 }
2946 }
2947 }
2948 }
2949
2950 return this;
2951 }
2952
2953 LoadNode* LoadNode::clone_pinned() const {
2954 LoadNode* ld = clone()->as_Load();
3081 // Polymorphic factory method:
3082 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
3083 assert((mo == unordered || mo == release), "unexpected");
3084 Compile* C = gvn.C;
3085 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
3086 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
3087 ctl != nullptr, "raw memory operations should have control edge");
3088
3089 switch (bt) {
3090 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
3091 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
3092 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
3093 case T_CHAR:
3094 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
3095 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3096 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
3097 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3098 case T_METADATA:
3099 case T_ADDRESS:
3100 case T_OBJECT:
3101 case T_ARRAY:
3102 #ifdef _LP64
3103 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3104 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
3105 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
3106 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
3107 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
3108 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
3109 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
3110 }
3111 #endif
3112 {
3113 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
3114 }
3115 default:
3116 guarantee(false, "unexpected basic type %s", type2name(bt));
3117 return (StoreNode*)nullptr;
3118 }
3119 }
3120
3121 //--------------------------bottom_type----------------------------------------
3122 const Type *StoreNode::bottom_type() const {
3123 return Type::MEMORY;
3124 }
3125
3126 //------------------------------hash-------------------------------------------
3127 uint StoreNode::hash() const {
3128 // unroll addition of interesting fields
3129 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
3130
3131 // Since they are not commoned, do not hash them:
3132 return NO_HASH;
3133 }
3134
3135 // Link together multiple stores (B/S/C/I) into a longer one.
3136 //
3758 }
3759 ss.print_cr("[TraceMergeStores]: with");
3760 merged_input_value->dump("\n", false, &ss);
3761 merged_store->dump("\n", false, &ss);
3762 tty->print("%s", ss.as_string());
3763 }
3764 #endif
3765
3766 //------------------------------Ideal------------------------------------------
3767 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3768 // When a store immediately follows a relevant allocation/initialization,
3769 // try to capture it into the initialization, or hoist it above.
3770 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3771 Node* p = MemNode::Ideal_common(phase, can_reshape);
3772 if (p) return (p == NodeSentinel) ? nullptr : p;
3773
3774 Node* mem = in(MemNode::Memory);
3775 Node* address = in(MemNode::Address);
3776 Node* value = in(MemNode::ValueIn);
3777 // Back-to-back stores to same address? Fold em up. Generally
3778 // unsafe if I have intervening uses...
3779 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3780 Node* st = mem;
3781 // If Store 'st' has more than one use, we cannot fold 'st' away.
3782 // For example, 'st' might be the final state at a conditional
3783 // return. Or, 'st' might be used by some node which is live at
3784 // the same time 'st' is live, which might be unschedulable. So,
3785 // require exactly ONE user until such time as we clone 'mem' for
3786 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3787 // true).
3788 while (st->is_Store() && st->outcnt() == 1) {
3789 // Looking at a dead closed cycle of memory?
3790 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3791 assert(Opcode() == st->Opcode() ||
3792 st->Opcode() == Op_StoreVector ||
3793 Opcode() == Op_StoreVector ||
3794 st->Opcode() == Op_StoreVectorScatter ||
3795 Opcode() == Op_StoreVectorScatter ||
3796 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3797 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3798 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3799 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3800 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3801 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3802
3803 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3804 st->as_Store()->memory_size() <= this->memory_size()) {
3805 Node* use = st->raw_out(0);
3806 if (phase->is_IterGVN()) {
3807 phase->is_IterGVN()->rehash_node_delayed(use);
3808 }
3809 // It's OK to do this in the parser, since DU info is always accurate,
3810 // and the parser always refers to nodes via SafePointNode maps.
3811 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3812 return this;
3813 }
3814 st = st->in(MemNode::Memory);
3815 }
3816 }
3817
3818
3819 // Capture an unaliased, unconditional, simple store into an initializer.
3920 const StoreVectorNode* store_vector = as_StoreVector();
3921 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3922 const Node* store_indices = store_vector->indices();
3923 const Node* mem_indices = mem_vector->indices();
3924 const Node* store_mask = store_vector->mask();
3925 const Node* mem_mask = mem_vector->mask();
3926 // Ensure types, indices, and masks match
3927 if (store_vector->vect_type() == mem_vector->vect_type() &&
3928 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3929 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3930 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3931 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3932 result = mem;
3933 }
3934 }
3935 }
3936
3937 // Store of zero anywhere into a freshly-allocated object?
3938 // Then the store is useless.
3939 // (It must already have been captured by the InitializeNode.)
3940 if (result == this && ReduceFieldZeroing) {
3941 // a newly allocated object is already all-zeroes everywhere
3942 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3943 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3944 result = mem;
3945 }
3946
3947 if (result == this && phase->type(val)->is_zero_type()) {
3948 // the store may also apply to zero-bits in an earlier object
3949 Node* prev_mem = find_previous_store(phase);
3950 // Steps (a), (b): Walk past independent stores to find an exact match.
3951 if (prev_mem != nullptr) {
3952 if (prev_mem->is_top()) {
3953 // find_previous_store returns top when the access is dead
3954 return prev_mem;
3955 }
3956 Node* prev_val = can_see_stored_value(prev_mem, phase);
3957 if (prev_val != nullptr && prev_val == val) {
3958 // prev_val and val might differ by a cast; it would be good
3959 // to keep the more informative of the two.
3960 result = mem;
3961 }
3962 }
3963 }
3964 }
3965
3966 PhaseIterGVN* igvn = phase->is_IterGVN();
3967 if (result != this && igvn != nullptr) {
4440 // Clearing a short array is faster with stores
4441 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4442 // Already know this is a large node, do not try to ideal it
4443 if (_is_large) return nullptr;
4444
4445 const int unit = BytesPerLong;
4446 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4447 if (!t) return nullptr;
4448 if (!t->is_con()) return nullptr;
4449 intptr_t raw_count = t->get_con();
4450 intptr_t size = raw_count;
4451 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4452 // Clearing nothing uses the Identity call.
4453 // Negative clears are possible on dead ClearArrays
4454 // (see jck test stmt114.stmt11402.val).
4455 if (size <= 0 || size % unit != 0) return nullptr;
4456 intptr_t count = size / unit;
4457 // Length too long; communicate this to matchers and assemblers.
4458 // Assemblers are responsible to produce fast hardware clears for it.
4459 if (size > InitArrayShortSize) {
4460 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4461 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4462 return nullptr;
4463 }
4464 if (!IdealizeClearArrayNode) return nullptr;
4465 Node *mem = in(1);
4466 if( phase->type(mem)==Type::TOP ) return nullptr;
4467 Node *adr = in(3);
4468 const Type* at = phase->type(adr);
4469 if( at==Type::TOP ) return nullptr;
4470 const TypePtr* atp = at->isa_ptr();
4471 // adjust atp to be the correct array element address type
4472 if (atp == nullptr) atp = TypePtr::BOTTOM;
4473 else atp = atp->add_offset(Type::OffsetBot);
4474 // Get base for derived pointer purposes
4475 if( adr->Opcode() != Op_AddP ) Unimplemented();
4476 Node *base = adr->in(1);
4477
4478 Node *val = in(4);
4479 Node *off = phase->MakeConX(BytesPerLong);
4480 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4481 count--;
4482 while (count--) {
4483 mem = phase->transform(mem);
4484 adr = phase->transform(AddPNode::make_with_base(base,adr,off));
4485 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4486 }
4487 return mem;
4488 }
4489
4490 //----------------------------step_through----------------------------------
4491 // Return allocation input memory edge if it is different instance
4492 // or itself if it is the one we are looking for.
4493 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4494 Node* n = *np;
4495 assert(n->is_ClearArray(), "sanity");
4496 intptr_t offset;
4497 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4498 // This method is called only before Allocate nodes are expanded
4499 // during macro nodes expansion. Before that ClearArray nodes are
4500 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4501 // Allocate nodes are expanded) which follows allocations.
4502 assert(alloc != nullptr, "should have allocation");
4503 if (alloc->_idx == instance_id) {
4504 // Can not bypass initialization of the instance we are looking for.
4505 return false;
4508 InitializeNode* init = alloc->initialization();
4509 if (init != nullptr)
4510 *np = init->in(TypeFunc::Memory);
4511 else
4512 *np = alloc->in(TypeFunc::Memory);
4513 return true;
4514 }
4515
4516 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4517 Node* base = dest;
4518 if (raw_base) {
4519 // May be called as part of the initialization of a just allocated object
4520 base = phase->C->top();
4521 }
4522 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4523 }
4524
4525 //----------------------------clear_memory-------------------------------------
4526 // Generate code to initialize object storage to zero.
4527 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4528 Node* val,
4529 Node* raw_val,
4530 intptr_t start_offset,
4531 Node* end_offset,
4532 bool raw_base,
4533 PhaseGVN* phase) {
4534 intptr_t offset = start_offset;
4535
4536 int unit = BytesPerLong;
4537 if ((offset % unit) != 0) {
4538 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4539 const TypePtr* atp = TypeRawPtr::BOTTOM;
4540 if (val != nullptr) {
4541 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4542 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4543 } else {
4544 assert(raw_val == nullptr, "val may not be null");
4545 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4546 }
4547 mem = phase->transform(mem);
4548 offset += BytesPerInt;
4549 }
4550 assert((offset % unit) == 0, "");
4551
4552 // Initialize the remaining stuff, if any, with a ClearArray.
4553 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, raw_base, phase);
4554 }
4555
4556 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4557 Node* raw_val,
4558 Node* start_offset,
4559 Node* end_offset,
4560 bool raw_base,
4561 PhaseGVN* phase) {
4562 if (start_offset == end_offset) {
4563 // nothing to do
4564 return mem;
4565 }
4566
4567 int unit = BytesPerLong;
4568 Node* zbase = start_offset;
4569 Node* zend = end_offset;
4570
4571 // Scale to the unit required by the CPU:
4572 if (!Matcher::init_array_count_is_in_bytes) {
4573 Node* shift = phase->intcon(exact_log2(unit));
4574 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4575 zend = phase->transform(new URShiftXNode(zend, shift) );
4576 }
4577
4578 // Bulk clear double-words
4579 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4580 Node* adr = make_address(dest, start_offset, raw_base, phase);
4581 if (raw_val == nullptr) {
4582 raw_val = phase->MakeConX(0);
4583 }
4584 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4585 return phase->transform(mem);
4586 }
4587
4588 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4589 Node* val,
4590 Node* raw_val,
4591 intptr_t start_offset,
4592 intptr_t end_offset,
4593 bool raw_base,
4594 PhaseGVN* phase) {
4595 if (start_offset == end_offset) {
4596 // nothing to do
4597 return mem;
4598 }
4599
4600 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4601 intptr_t done_offset = end_offset;
4602 if ((done_offset % BytesPerLong) != 0) {
4603 done_offset -= BytesPerInt;
4604 }
4605 if (done_offset > start_offset) {
4606 mem = clear_memory(ctl, mem, dest, val, raw_val,
4607 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4608 }
4609 if (done_offset < end_offset) { // emit the final 32-bit store
4610 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4611 const TypePtr* atp = TypeRawPtr::BOTTOM;
4612 if (val != nullptr) {
4613 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4614 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4615 } else {
4616 assert(raw_val == nullptr, "val may not be null");
4617 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4618 }
4619 mem = phase->transform(mem);
4620 done_offset += BytesPerInt;
4621 }
4622 assert(done_offset == end_offset, "");
4623 return mem;
4624 }
4625
4626 //=============================================================================
4627 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4628 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4629 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4630 #ifdef ASSERT
4631 , _pair_idx(0)
4632 #endif
4633 {
4634 init_class_id(Class_MemBar);
4635 Node* top = C->top();
4636 init_req(TypeFunc::I_O,top);
4637 init_req(TypeFunc::FramePtr,top);
4638 init_req(TypeFunc::ReturnAdr,top);
4747 PhaseIterGVN* igvn = phase->is_IterGVN();
4748 remove(igvn);
4749 // Must return either the original node (now dead) or a new node
4750 // (Do not return a top here, since that would break the uniqueness of top.)
4751 return new ConINode(TypeInt::ZERO);
4752 }
4753 }
4754 return progress ? this : nullptr;
4755 }
4756
4757 //------------------------------Value------------------------------------------
4758 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4759 if( !in(0) ) return Type::TOP;
4760 if( phase->type(in(0)) == Type::TOP )
4761 return Type::TOP;
4762 return TypeTuple::MEMBAR;
4763 }
4764
4765 //------------------------------match------------------------------------------
4766 // Construct projections for memory.
4767 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4768 switch (proj->_con) {
4769 case TypeFunc::Control:
4770 case TypeFunc::Memory:
4771 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4772 }
4773 ShouldNotReachHere();
4774 return nullptr;
4775 }
4776
4777 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4778 trailing->_kind = TrailingStore;
4779 leading->_kind = LeadingStore;
4780 #ifdef ASSERT
4781 trailing->_pair_idx = leading->_idx;
4782 leading->_pair_idx = leading->_idx;
4783 #endif
4784 }
4785
4786 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4787 trailing->_kind = TrailingLoadStore;
5034 return (req() > RawStores);
5035 }
5036
5037 void InitializeNode::set_complete(PhaseGVN* phase) {
5038 assert(!is_complete(), "caller responsibility");
5039 _is_complete = Complete;
5040
5041 // After this node is complete, it contains a bunch of
5042 // raw-memory initializations. There is no need for
5043 // it to have anything to do with non-raw memory effects.
5044 // Therefore, tell all non-raw users to re-optimize themselves,
5045 // after skipping the memory effects of this initialization.
5046 PhaseIterGVN* igvn = phase->is_IterGVN();
5047 if (igvn) igvn->add_users_to_worklist(this);
5048 }
5049
5050 // convenience function
5051 // return false if the init contains any stores already
5052 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
5053 InitializeNode* init = initialization();
5054 if (init == nullptr || init->is_complete()) {
5055 return false;
5056 }
5057 init->remove_extra_zeroes();
5058 // for now, if this allocation has already collected any inits, bail:
5059 if (init->is_non_zero()) return false;
5060 init->set_complete(phase);
5061 return true;
5062 }
5063
5064 void InitializeNode::remove_extra_zeroes() {
5065 if (req() == RawStores) return;
5066 Node* zmem = zero_memory();
5067 uint fill = RawStores;
5068 for (uint i = fill; i < req(); i++) {
5069 Node* n = in(i);
5070 if (n->is_top() || n == zmem) continue; // skip
5071 if (fill < i) set_req(fill, n); // compact
5072 ++fill;
5073 }
5074 // delete any empty spaces created:
5075 while (fill < req()) {
5076 del_req(fill);
5220 // store node that we'd like to capture. We need to check
5221 // the uses of the MergeMemNode.
5222 mems.push(n);
5223 }
5224 } else if (n->is_Mem()) {
5225 Node* other_adr = n->in(MemNode::Address);
5226 if (other_adr == adr) {
5227 failed = true;
5228 break;
5229 } else {
5230 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5231 if (other_t_adr != nullptr) {
5232 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5233 if (other_alias_idx == alias_idx) {
5234 // A load from the same memory slice as the store right
5235 // after the InitializeNode. We check the control of the
5236 // object/array that is loaded from. If it's the same as
5237 // the store control then we cannot capture the store.
5238 assert(!n->is_Store(), "2 stores to same slice on same control?");
5239 Node* base = other_adr;
5240 if (base->is_Phi()) {
5241 // In rare case, base may be a PhiNode and it may read
5242 // the same memory slice between InitializeNode and store.
5243 failed = true;
5244 break;
5245 }
5246 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5247 base = base->in(AddPNode::Base);
5248 if (base != nullptr) {
5249 base = base->uncast();
5250 if (base->is_Proj() && base->in(0) == alloc) {
5251 failed = true;
5252 break;
5253 }
5254 }
5255 }
5256 }
5257 }
5258 } else {
5259 failed = true;
5260 break;
5261 }
5262 }
5263 }
5264 }
5265 if (failed) {
5811 // z's_done 12 16 16 16 12 16 12
5812 // z's_needed 12 16 16 16 16 16 16
5813 // zsize 0 0 0 0 4 0 4
5814 if (next_full_store < 0) {
5815 // Conservative tack: Zero to end of current word.
5816 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5817 } else {
5818 // Zero to beginning of next fully initialized word.
5819 // Or, don't zero at all, if we are already in that word.
5820 assert(next_full_store >= zeroes_needed, "must go forward");
5821 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5822 zeroes_needed = next_full_store;
5823 }
5824 }
5825
5826 if (zeroes_needed > zeroes_done) {
5827 intptr_t zsize = zeroes_needed - zeroes_done;
5828 // Do some incremental zeroing on rawmem, in parallel with inits.
5829 zeroes_done = align_down(zeroes_done, BytesPerInt);
5830 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5831 allocation()->in(AllocateNode::InitValue),
5832 allocation()->in(AllocateNode::RawInitValue),
5833 zeroes_done, zeroes_needed,
5834 true,
5835 phase);
5836 zeroes_done = zeroes_needed;
5837 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5838 do_zeroing = false; // leave the hole, next time
5839 }
5840 }
5841
5842 // Collect the store and move on:
5843 phase->replace_input_of(st, MemNode::Memory, inits);
5844 inits = st; // put it on the linearized chain
5845 set_req(i, zmem); // unhook from previous position
5846
5847 if (zeroes_done == st_off)
5848 zeroes_done = next_init_off;
5849
5850 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5851
5852 #ifdef ASSERT
5873 remove_extra_zeroes(); // clear out all the zmems left over
5874 add_req(inits);
5875
5876 if (!(UseTLAB && ZeroTLAB)) {
5877 // If anything remains to be zeroed, zero it all now.
5878 zeroes_done = align_down(zeroes_done, BytesPerInt);
5879 // if it is the last unused 4 bytes of an instance, forget about it
5880 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5881 if (zeroes_done + BytesPerLong >= size_limit) {
5882 AllocateNode* alloc = allocation();
5883 assert(alloc != nullptr, "must be present");
5884 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5885 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5886 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5887 if (zeroes_done == k->layout_helper())
5888 zeroes_done = size_limit;
5889 }
5890 }
5891 if (zeroes_done < size_limit) {
5892 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5893 allocation()->in(AllocateNode::InitValue),
5894 allocation()->in(AllocateNode::RawInitValue),
5895 zeroes_done, size_in_bytes, true, phase);
5896 }
5897 }
5898
5899 set_complete(phase);
5900 return rawmem;
5901 }
5902
5903 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5904 auto replace_proj = [&](ProjNode* proj) {
5905 C->gvn_replace_by(proj, mem);
5906 return CONTINUE;
5907 };
5908 apply_to_projs(replace_proj, TypeFunc::Memory);
5909 }
5910
5911 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5912 DUIterator_Fast imax, i = fast_outs(imax);
5913 auto replace_proj = [&](ProjNode* proj) {
5914 igvn->replace_node(proj, mem);
|