6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/compile.hpp"
38 #include "opto/connode.hpp"
39 #include "opto/convertnode.hpp"
40 #include "opto/loopnode.hpp"
41 #include "opto/machnode.hpp"
42 #include "opto/matcher.hpp"
43 #include "opto/memnode.hpp"
44 #include "opto/mempointer.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/opcodes.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regalloc.hpp"
50 #include "opto/regmask.hpp"
51 #include "opto/rootnode.hpp"
52 #include "opto/traceMergeStoresTag.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/macros.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #include "utilities/vmError.hpp"
59
60 // Portions of code courtesy of Clifford Click
61
62 // Optimization - Graph Style
63
64 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
65
66 //=============================================================================
67 uint MemNode::size_of() const { return sizeof(*this); }
68
69 const TypePtr *MemNode::adr_type() const {
70 Node* adr = in(Address);
71 if (adr == nullptr) return nullptr; // node is dead
72 const TypePtr* cross_check = nullptr;
73 DEBUG_ONLY(cross_check = _adr_type);
74 return calculate_adr_type(adr->bottom_type(), cross_check);
75 }
124 st->print(", idx=Bot;");
125 else if (atp->index() == Compile::AliasIdxTop)
126 st->print(", idx=Top;");
127 else if (atp->index() == Compile::AliasIdxRaw)
128 st->print(", idx=Raw;");
129 else {
130 ciField* field = atp->field();
131 if (field) {
132 st->print(", name=");
133 field->print_name_on(st);
134 }
135 st->print(", idx=%d;", atp->index());
136 }
137 }
138 }
139
140 extern void print_alias_types();
141
142 #endif
143
144 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
145 assert((t_oop != nullptr), "sanity");
146 bool is_instance = t_oop->is_known_instance_field();
147 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
148 (load != nullptr) && load->is_Load() &&
149 (phase->is_IterGVN() != nullptr);
150 if (!(is_instance || is_boxed_value_load))
151 return mchain; // don't try to optimize non-instance types
152 uint instance_id = t_oop->instance_id();
153 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
154 Node *prev = nullptr;
155 Node *result = mchain;
156 while (prev != result) {
157 prev = result;
158 if (result == start_mem)
159 break; // hit one of our sentinels
160 // skip over a call which does not affect this memory slice
161 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
162 Node *proj_in = result->in(0);
163 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
164 break; // hit one of our sentinels
165 } else if (proj_in->is_Call()) {
166 // ArrayCopyNodes processed here as well
167 CallNode *call = proj_in->as_Call();
168 if (!call->may_modify(t_oop, phase)) { // returns false for instances
169 result = call->in(TypeFunc::Memory);
170 }
171 } else if (proj_in->is_Initialize()) {
172 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
173 // Stop if this is the initialization for the object instance which
174 // contains this memory slice, otherwise skip over it.
175 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
176 break;
177 }
178 if (is_instance) {
179 result = proj_in->in(TypeFunc::Memory);
180 } else if (is_boxed_value_load) {
181 Node* klass = alloc->in(AllocateNode::KlassNode);
182 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
183 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
184 result = proj_in->in(TypeFunc::Memory); // not related allocation
185 }
186 }
187 } else if (proj_in->is_MemBar()) {
188 ArrayCopyNode* ac = nullptr;
189 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
190 break;
191 }
192 result = proj_in->in(TypeFunc::Memory);
193 } else if (proj_in->is_top()) {
194 break; // dead code
195 } else {
196 assert(false, "unexpected projection");
197 }
198 } else if (result->is_ClearArray()) {
199 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
200 // Can not bypass initialization of the instance
201 // we are looking for.
202 break;
203 }
204 // Otherwise skip it (the call updated 'result' value).
205 } else if (result->is_MergeMem()) {
206 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
207 }
208 }
209 return result;
210 }
211
212 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
213 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
214 if (t_oop == nullptr)
215 return mchain; // don't try to optimize non-oop types
216 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
217 bool is_instance = t_oop->is_known_instance_field();
218 PhaseIterGVN *igvn = phase->is_IterGVN();
219 if (is_instance && igvn != nullptr && result->is_Phi()) {
220 PhiNode *mphi = result->as_Phi();
221 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
222 const TypePtr *t = mphi->adr_type();
223 bool do_split = false;
224 // In the following cases, Load memory input can be further optimized based on
225 // its precise address type
226 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
227 do_split = true;
228 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
229 const TypeOopPtr* mem_t =
230 t->is_oopptr()->cast_to_exactness(true)
231 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
232 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
233 if (t_oop->isa_aryptr()) {
234 mem_t = mem_t->is_aryptr()
235 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
236 ->cast_to_size(t_oop->is_aryptr()->size())
237 ->with_offset(t_oop->is_aryptr()->offset())
238 ->is_aryptr();
239 }
240 do_split = mem_t == t_oop;
241 }
242 if (do_split) {
243 // clone the Phi with our address type
244 result = mphi->split_out_instance(t_adr, igvn);
245 } else {
246 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
247 }
248 }
249 return result;
250 }
251
252 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
253 uint alias_idx = phase->C->get_alias_index(tp);
254 Node *mem = mmem;
255 #ifdef ASSERT
256 {
257 // Check that current type is consistent with the alias index used during graph construction
258 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
259 bool consistent = adr_check == nullptr || adr_check->empty() ||
260 phase->C->must_alias(adr_check, alias_idx );
261 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
262 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
263 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
264 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
265 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
266 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
267 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
268 // don't assert if it is dead code.
269 consistent = true;
270 }
271 if( !consistent ) {
272 st->print("alias_idx==%d, adr_check==", alias_idx);
273 if( adr_check == nullptr ) {
274 st->print("null");
275 } else {
276 adr_check->dump();
277 }
278 st->cr();
279 print_alias_types();
280 assert(consistent, "adr_check must match alias idx");
281 }
282 }
283 #endif
996 in_bytes(JavaThread::vthread_offset()),
997 in_bytes(JavaThread::scopedValueCache_offset()),
998 };
999
1000 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1001 if (offset == offsets[i]) {
1002 return true;
1003 }
1004 }
1005 }
1006
1007 return false;
1008 }
1009 #endif
1010
1011 //----------------------------LoadNode::make-----------------------------------
1012 // Polymorphic factory method:
1013 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1014 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1015 Compile* C = gvn.C;
1016 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
1017
1018 // sanity check the alias category against the created node type
1019 assert(!(adr_type->isa_oopptr() &&
1020 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1021 "use LoadKlassNode instead");
1022 assert(!(adr_type->isa_aryptr() &&
1023 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1024 "use LoadRangeNode instead");
1025 // Check control edge of raw loads
1026 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1027 // oop will be recorded in oop map if load crosses safepoint
1028 rt->isa_oopptr() || is_immutable_value(adr),
1029 "raw memory operations should have control edge");
1030 LoadNode* load = nullptr;
1031 switch (bt) {
1032 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1033 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1034 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1035 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1036 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1037 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1038 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1039 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1040 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1041 case T_OBJECT:
1042 case T_NARROWOOP:
1043 #ifdef _LP64
1044 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1045 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1046 } else
1047 #endif
1048 {
1049 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1050 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1051 }
1052 break;
1053 default:
1054 ShouldNotReachHere();
1055 break;
1056 }
1057 assert(load != nullptr, "LoadNode should have been created");
1058 if (unaligned) {
1059 load->set_unaligned_access();
1060 }
1061 if (mismatched) {
1062 load->set_mismatched_access();
1063 }
1064 if (unsafe) {
1065 load->set_unsafe_access();
1066 }
1067 load->set_barrier_data(barrier_data);
1068 if (load->Opcode() == Op_LoadN) {
1069 Node* ld = gvn.transform(load);
1070 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1071 }
1072
1073 return load;
1074 }
1075
1076 //------------------------------hash-------------------------------------------
1077 uint LoadNode::hash() const {
1078 // unroll addition of interesting fields
1079 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1080 }
1081
1082 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1083 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1084 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1085 bool is_stable_ary = FoldStableValues &&
1086 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1087 tp->isa_aryptr()->is_stable();
1088
1089 return (eliminate_boxing && non_volatile) || is_stable_ary;
1090 }
1091
1092 return false;
1093 }
1094
1095 // Is the value loaded previously stored by an arraycopy? If so return
1096 // a load node that reads from the source array so we may be able to
1097 // optimize out the ArrayCopy node later.
1098 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1099 Node* ld_adr = in(MemNode::Address);
1100 intptr_t ld_off = 0;
1101 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1102 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1103 if (ac != nullptr) {
1104 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1105
1106 Node* mem = ac->in(TypeFunc::Memory);
1107 Node* ctl = ac->in(0);
1108 Node* src = ac->in(ArrayCopyNode::Src);
1109
1117 if (ac->as_ArrayCopy()->is_clonebasic()) {
1118 assert(ld_alloc != nullptr, "need an alloc");
1119 assert(addp->is_AddP(), "address must be addp");
1120 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1121 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1122 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1123 addp->set_req(AddPNode::Base, src);
1124 addp->set_req(AddPNode::Address, src);
1125 } else {
1126 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1127 ac->as_ArrayCopy()->is_copyof_validated() ||
1128 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1129 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1130 addp->set_req(AddPNode::Base, src);
1131 addp->set_req(AddPNode::Address, src);
1132
1133 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1134 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1135 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1136
1137 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1138 uint shift = exact_log2(type2aelembytes(ary_elem));
1139
1140 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1141 #ifdef _LP64
1142 diff = phase->transform(new ConvI2LNode(diff));
1143 #endif
1144 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1145
1146 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1147 addp->set_req(AddPNode::Offset, offset);
1148 }
1149 addp = phase->transform(addp);
1150 #ifdef ASSERT
1151 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1152 ld->_adr_type = adr_type;
1153 #endif
1154 ld->set_req(MemNode::Address, addp);
1155 ld->set_req(0, ctl);
1156 ld->set_req(MemNode::Memory, mem);
1157 return ld;
1158 }
1159 return nullptr;
1160 }
1161
1162 // This routine exists to make sure this set of tests is done the same
1163 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1164 // will change the graph shape in a way which makes memory alive twice at the
1165 // same time (uses the Oracle model of aliasing), then some
1166 // LoadXNode::Identity will fold things back to the equivalence-class model
1167 // of aliasing.
1168 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1169 Node* ld_adr = in(MemNode::Address);
1170 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1171 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1172
1173 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1174 uint alias_idx = atp->index();
1175 Node* result = nullptr;
1176 Node* current = st;
1177 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1178 // this load. Stop once any other kind of node is encountered.
1179 //
1180 // In principle, folding a load is moving it up until it meets a matching store.
1181 //
1182 // store(ptr, v); store(ptr, v); store(ptr, v);
1183 // membar1; -> membar1; -> load(ptr);
1184 // membar2; load(ptr); membar1;
1185 // load(ptr); membar2; membar2;
1186 //
1187 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1188 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1189 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1269
1270 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1271 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1272 if (in_vt != out_vt) {
1273 return nullptr;
1274 }
1275 }
1276 return st->in(MemNode::ValueIn);
1277 }
1278
1279 // A load from a freshly-created object always returns zero.
1280 // (This can happen after LoadNode::Ideal resets the load's memory input
1281 // to find_captured_store, which returned InitializeNode::zero_memory.)
1282 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1283 (st->in(0) == ld_alloc) &&
1284 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1285 // return a zero value for the load's basic type
1286 // (This is one of the few places where a generic PhaseTransform
1287 // can create new nodes. Think of it as lazily manifesting
1288 // virtually pre-existing constants.)
1289 if (value_basic_type() != T_VOID) {
1290 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1291 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1292 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1293 // by the ArrayCopyNode.
1294 return phase->zerocon(value_basic_type());
1295 }
1296 } else {
1297 // TODO: materialize all-zero vector constant
1298 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1299 }
1300 }
1301
1302 // A load from an initialization barrier can match a captured store.
1303 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1304 InitializeNode* init = st->in(0)->as_Initialize();
1305 AllocateNode* alloc = init->allocation();
1306 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1307 // examine a captured store value
1308 st = init->find_captured_store(ld_off, memory_size(), phase);
1321 base = bs->step_over_gc_barrier(base);
1322 if (base != nullptr && base->is_Proj() &&
1323 base->as_Proj()->_con == TypeFunc::Parms &&
1324 base->in(0)->is_CallStaticJava() &&
1325 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1326 return base->in(0)->in(TypeFunc::Parms);
1327 }
1328 }
1329
1330 break;
1331 }
1332
1333 return nullptr;
1334 }
1335
1336 //----------------------is_instance_field_load_with_local_phi------------------
1337 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1338 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1339 in(Address)->is_AddP() ) {
1340 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1341 // Only instances and boxed values.
1342 if( t_oop != nullptr &&
1343 (t_oop->is_ptr_to_boxed_value() ||
1344 t_oop->is_known_instance_field()) &&
1345 t_oop->offset() != Type::OffsetBot &&
1346 t_oop->offset() != Type::OffsetTop) {
1347 return true;
1348 }
1349 }
1350 return false;
1351 }
1352
1353 //------------------------------Identity---------------------------------------
1354 // Loads are identity if previous store is to same address
1355 Node* LoadNode::Identity(PhaseGVN* phase) {
1356 // If the previous store-maker is the right kind of Store, and the store is
1357 // to the same address, then we are equal to the value stored.
1358 Node* mem = in(Memory);
1359 Node* value = can_see_stored_value_through_membars(mem, phase);
1360 if( value ) {
1361 // byte, short & char stores truncate naturally.
1362 // A load has to load the truncated value which requires
1363 // some sort of masking operation and that requires an
1364 // Ideal call instead of an Identity call.
1365 if (memory_size() < BytesPerInt) {
1366 // If the input to the store does not fit with the load's result type,
1367 // it must be truncated via an Ideal call.
1368 if (!phase->type(value)->higher_equal(phase->type(this)))
1369 return this;
1370 }
1371 // (This works even when value is a Con, but LoadNode::Value
1372 // usually runs first, producing the singleton type of the Con.)
1373 if (!has_pinned_control_dependency() || value->is_Con()) {
1374 return value;
1375 } else {
1376 return this;
1377 }
1378 }
1379
1380 if (has_pinned_control_dependency()) {
1381 return this;
1382 }
1383 // Search for an existing data phi which was generated before for the same
1384 // instance's field to avoid infinite generation of phis in a loop.
1385 Node *region = mem->in(0);
1386 if (is_instance_field_load_with_local_phi(region)) {
1387 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1388 int this_index = phase->C->get_alias_index(addr_t);
1389 int this_offset = addr_t->offset();
1390 int this_iid = addr_t->instance_id();
1391 if (!addr_t->is_known_instance() &&
1392 addr_t->is_ptr_to_boxed_value()) {
1393 // Use _idx of address base (could be Phi node) for boxed values.
1394 intptr_t ignore = 0;
1395 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1396 if (base == nullptr) {
1397 return this;
1398 }
1399 this_iid = base->_idx;
1400 }
1401 const Type* this_type = bottom_type();
1402 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1403 Node* phi = region->fast_out(i);
1404 if (phi->is_Phi() && phi != mem &&
1405 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1406 return phi;
1407 }
1408 }
1409 }
1410
1411 return this;
1412 }
1413
1948 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1949 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1950
1951 // Skip up past a SafePoint control. Cannot do this for Stores because
1952 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1953 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1954 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1955 !addr_mark &&
1956 (depends_only_on_test() || has_unknown_control_dependency())) {
1957 ctrl = ctrl->in(0);
1958 set_req(MemNode::Control,ctrl);
1959 return this;
1960 }
1961
1962 intptr_t ignore = 0;
1963 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1964 if (base != nullptr
1965 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1966 // Check for useless control edge in some common special cases
1967 if (in(MemNode::Control) != nullptr
1968 && can_remove_control()
1969 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1970 && all_controls_dominate(base, phase->C->start(), phase)) {
1971 // A method-invariant, non-null address (constant or 'this' argument).
1972 set_req(MemNode::Control, nullptr);
1973 return this;
1974 }
1975 }
1976
1977 Node* mem = in(MemNode::Memory);
1978 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1979
1980 if (can_reshape && (addr_t != nullptr)) {
1981 // try to optimize our memory input
1982 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1983 if (opt_mem != mem) {
1984 set_req_X(MemNode::Memory, opt_mem, phase);
1985 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1986 return this;
1987 }
2100 // No match.
2101 return nullptr;
2102 }
2103
2104 //------------------------------Value-----------------------------------------
2105 const Type* LoadNode::Value(PhaseGVN* phase) const {
2106 // Either input is TOP ==> the result is TOP
2107 Node* mem = in(MemNode::Memory);
2108 const Type *t1 = phase->type(mem);
2109 if (t1 == Type::TOP) return Type::TOP;
2110 Node* adr = in(MemNode::Address);
2111 const TypePtr* tp = phase->type(adr)->isa_ptr();
2112 if (tp == nullptr || tp->empty()) return Type::TOP;
2113 int off = tp->offset();
2114 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2115 Compile* C = phase->C;
2116
2117 // If load can see a previous constant store, use that.
2118 Node* value = can_see_stored_value_through_membars(mem, phase);
2119 if (value != nullptr && value->is_Con()) {
2120 assert(value->bottom_type()->higher_equal(_type), "sanity");
2121 return value->bottom_type();
2122 }
2123
2124 // Try to guess loaded type from pointer type
2125 if (tp->isa_aryptr()) {
2126 const TypeAryPtr* ary = tp->is_aryptr();
2127 const Type* t = ary->elem();
2128
2129 // Determine whether the reference is beyond the header or not, by comparing
2130 // the offset against the offset of the start of the array's data.
2131 // Different array types begin at slightly different offsets (12 vs. 16).
2132 // We choose T_BYTE as an example base type that is least restrictive
2133 // as to alignment, which will therefore produce the smallest
2134 // possible base offset.
2135 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2136 const bool off_beyond_header = (off >= min_base_off);
2137
2138 // Try to constant-fold a stable array element.
2139 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2140 // Make sure the reference is not into the header and the offset is constant
2141 ciObject* aobj = ary->const_oop();
2142 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2143 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2144 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off,
2145 stable_dimension,
2146 value_basic_type(), is_unsigned());
2147 if (con_type != nullptr) {
2148 return con_type;
2149 }
2150 }
2151 }
2152
2153 // Don't do this for integer types. There is only potential profit if
2154 // the element type t is lower than _type; that is, for int types, if _type is
2155 // more restrictive than t. This only happens here if one is short and the other
2156 // char (both 16 bits), and in those cases we've made an intentional decision
2157 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2158 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2159 //
2160 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2161 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2162 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2163 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2164 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2165 // In fact, that could have been the original type of p1, and p1 could have
2166 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2167 // expression (LShiftL quux 3) independently optimized to the constant 8.
2168 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2169 && (_type->isa_vect() == nullptr)
2170 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2171 // t might actually be lower than _type, if _type is a unique
2172 // concrete subclass of abstract class t.
2173 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2174 const Type* jt = t->join_speculative(_type);
2175 // In any case, do not allow the join, per se, to empty out the type.
2176 if (jt->empty() && !t->empty()) {
2177 // This can happen if a interface-typed array narrows to a class type.
2178 jt = _type;
2179 }
2180 #ifdef ASSERT
2181 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2182 // The pointers in the autobox arrays are always non-null
2183 Node* base = adr->in(AddPNode::Base);
2184 if ((base != nullptr) && base->is_DecodeN()) {
2185 // Get LoadN node which loads IntegerCache.cache field
2186 base = base->in(1);
2187 }
2188 if ((base != nullptr) && base->is_Con()) {
2189 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2190 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2191 // It could be narrow oop
2192 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2193 }
2194 }
2195 }
2196 #endif
2197 return jt;
2198 }
2199 }
2200 } else if (tp->base() == Type::InstPtr) {
2201 assert( off != Type::OffsetBot ||
2202 // arrays can be cast to Objects
2203 !tp->isa_instptr() ||
2204 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2205 // unsafe field access may not have a constant offset
2206 C->has_unsafe_access(),
2207 "Field accesses must be precise" );
2208 // For oop loads, we expect the _type to be precise.
2209
2210 // Optimize loads from constant fields.
2211 const TypeInstPtr* tinst = tp->is_instptr();
2212 ciObject* const_oop = tinst->const_oop();
2213 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2214 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2215 if (con_type != nullptr) {
2216 return con_type;
2217 }
2218 }
2219 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2220 assert(off != Type::OffsetBot ||
2221 !tp->isa_instklassptr() ||
2222 // arrays can be cast to Objects
2223 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2224 // also allow array-loading from the primary supertype
2225 // array during subtype checks
2226 Opcode() == Op_LoadKlass,
2227 "Field accesses must be precise");
2228 // For klass/static loads, we expect the _type to be precise
2229 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2230 /* With mirrors being an indirect in the Klass*
2231 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2232 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2233 *
2234 * So check the type and klass of the node before the LoadP.
2241 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2242 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2243 return TypeInstPtr::make(klass->java_mirror());
2244 }
2245 }
2246 }
2247
2248 const TypeKlassPtr *tkls = tp->isa_klassptr();
2249 if (tkls != nullptr) {
2250 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2251 ciKlass* klass = tkls->exact_klass();
2252 // We are loading a field from a Klass metaobject whose identity
2253 // is known at compile time (the type is "exact" or "precise").
2254 // Check for fields we know are maintained as constants by the VM.
2255 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2256 // The field is Klass::_super_check_offset. Return its (constant) value.
2257 // (Folds up type checking code.)
2258 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2259 return TypeInt::make(klass->super_check_offset());
2260 }
2261 if (UseCompactObjectHeaders) {
2262 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2263 // The field is Klass::_prototype_header. Return its (constant) value.
2264 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2265 return TypeX::make(klass->prototype_header());
2266 }
2267 }
2268 // Compute index into primary_supers array
2269 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2270 // Check for overflowing; use unsigned compare to handle the negative case.
2271 if( depth < ciKlass::primary_super_limit() ) {
2272 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2273 // (Folds up type checking code.)
2274 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2275 ciKlass *ss = klass->super_of_depth(depth);
2276 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2277 }
2278 const Type* aift = load_array_final_field(tkls, klass);
2279 if (aift != nullptr) return aift;
2280 }
2281
2282 // We can still check if we are loading from the primary_supers array at a
2283 // shallow enough depth. Even though the klass is not exact, entries less
2284 // than or equal to its super depth are correct.
2285 if (tkls->is_loaded()) {
2286 ciKlass* klass = nullptr;
2320 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2321 // The key property of this type is that it folds up tests
2322 // for array-ness, since it proves that the layout_helper is positive.
2323 // Thus, a generic value like the basic object layout helper works fine.
2324 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2325 }
2326 }
2327
2328 // If we are loading from a freshly-allocated object/array, produce a zero.
2329 // Things to check:
2330 // 1. Load is beyond the header: headers are not guaranteed to be zero
2331 // 2. Load is not vectorized: vectors have no zero constant
2332 // 3. Load has no matching store, i.e. the input is the initial memory state
2333 const TypeOopPtr* tinst = tp->isa_oopptr();
2334 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2335 bool is_not_vect = (_type->isa_vect() == nullptr);
2336 if (is_not_header && is_not_vect) {
2337 Node* mem = in(MemNode::Memory);
2338 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2339 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2340 return Type::get_zero_type(_type->basic_type());
2341 }
2342 }
2343
2344 if (!UseCompactObjectHeaders) {
2345 Node* alloc = is_new_object_mark_load();
2346 if (alloc != nullptr) {
2347 return TypeX::make(markWord::prototype().value());
2348 }
2349 }
2350
2351 return _type;
2352 }
2353
2354 //------------------------------match_edge-------------------------------------
2355 // Do we Match on this edge index or not? Match only the address.
2356 uint LoadNode::match_edge(uint idx) const {
2357 return idx == MemNode::Address;
2358 }
2359
2360 //--------------------------LoadBNode::Ideal--------------------------------------
2361 //
2362 // If the previous store is to the same address as this load,
2363 // and the value stored was larger than a byte, replace this load
2364 // with the value stored truncated to a byte. If no truncation is
2365 // needed, the replacement is done in LoadNode::Identity().
2366 //
2367 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2476 }
2477 }
2478 // Identity call will handle the case where truncation is not needed.
2479 return LoadNode::Ideal(phase, can_reshape);
2480 }
2481
2482 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2483 Node* mem = in(MemNode::Memory);
2484 Node* value = can_see_stored_value_through_membars(mem, phase);
2485 if (value != nullptr && value->is_Con() &&
2486 !value->bottom_type()->higher_equal(_type)) {
2487 // If the input to the store does not fit with the load's result type,
2488 // it must be truncated. We can't delay until Ideal call since
2489 // a singleton Value is needed for split_thru_phi optimization.
2490 int con = value->get_int();
2491 return TypeInt::make((con << 16) >> 16);
2492 }
2493 return LoadNode::Value(phase);
2494 }
2495
2496 //=============================================================================
2497 //----------------------------LoadKlassNode::make------------------------------
2498 // Polymorphic factory method:
2499 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2500 // sanity check the alias category against the created node type
2501 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2502 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2503 #ifdef _LP64
2504 if (adr_type->is_ptr_to_narrowklass()) {
2505 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2506 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2507 }
2508 #endif
2509 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2510 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2511 }
2512
2513 //------------------------------Value------------------------------------------
2514 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2515 return klass_value_common(phase);
2548 }
2549 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2550 }
2551 if (!t->is_klass()) {
2552 // a primitive Class (e.g., int.class) has null for a klass field
2553 return TypePtr::NULL_PTR;
2554 }
2555 // Fold up the load of the hidden field
2556 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2557 }
2558 // non-constant mirror, so we can't tell what's going on
2559 }
2560 if (!tinst->is_loaded())
2561 return _type; // Bail out if not loaded
2562 if (offset == oopDesc::klass_offset_in_bytes()) {
2563 return tinst->as_klass_type(true);
2564 }
2565 }
2566
2567 // Check for loading klass from an array
2568 const TypeAryPtr *tary = tp->isa_aryptr();
2569 if (tary != nullptr &&
2570 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2571 return tary->as_klass_type(true);
2572 }
2573
2574 // Check for loading klass from an array klass
2575 const TypeKlassPtr *tkls = tp->isa_klassptr();
2576 if (tkls != nullptr && !StressReflectiveCode) {
2577 if (!tkls->is_loaded())
2578 return _type; // Bail out if not loaded
2579 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2580 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2581 // // Always returning precise element type is incorrect,
2582 // // e.g., element type could be object and array may contain strings
2583 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2584
2585 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2586 // according to the element type's subclassing.
2587 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2588 }
2589 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2590 tkls->offset() == in_bytes(Klass::super_offset())) {
2591 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2592 // The field is Klass::_super. Return its (constant) value.
2593 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2594 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2595 }
2596 }
2597
2598 if (tkls != nullptr && !UseSecondarySupersCache
2599 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2600 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2601 return TypePtr::NULL_PTR;
2602 }
2603
2604 // Bailout case
2605 return LoadNode::Value(phase);
2606 }
2607
2608 //------------------------------Identity---------------------------------------
2631 base = bs->step_over_gc_barrier(base);
2632 }
2633
2634 // We can fetch the klass directly through an AllocateNode.
2635 // This works even if the klass is not constant (clone or newArray).
2636 if (offset == oopDesc::klass_offset_in_bytes()) {
2637 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2638 if (allocated_klass != nullptr) {
2639 return allocated_klass;
2640 }
2641 }
2642
2643 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2644 // See inline_native_Class_query for occurrences of these patterns.
2645 // Java Example: x.getClass().isAssignableFrom(y)
2646 //
2647 // This improves reflective code, often making the Class
2648 // mirror go completely dead. (Current exception: Class
2649 // mirrors may appear in debug info, but we could clean them out by
2650 // introducing a new debug info operator for Klass.java_mirror).
2651
2652 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2653 && offset == java_lang_Class::klass_offset()) {
2654 if (base->is_Load()) {
2655 Node* base2 = base->in(MemNode::Address);
2656 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2657 Node* adr2 = base2->in(MemNode::Address);
2658 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2659 if (tkls != nullptr && !tkls->empty()
2660 && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2661 && adr2->is_AddP()
2662 ) {
2663 int mirror_field = in_bytes(Klass::java_mirror_offset());
2664 if (tkls->offset() == mirror_field) {
2665 #ifdef ASSERT
2666 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2667 assert(tkls2->offset() == 0, "not a load of java_mirror");
2668 #endif
2669 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2670 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2671 return adr2->in(AddPNode::Address);
2672 }
2673 }
2674 }
2675 }
2676 }
2677
2678 return this;
2679 }
2680
2681 LoadNode* LoadNode::clone_pinned() const {
2682 LoadNode* ld = clone()->as_Load();
2809 // Polymorphic factory method:
2810 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2811 assert((mo == unordered || mo == release), "unexpected");
2812 Compile* C = gvn.C;
2813 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
2814 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2815 ctl != nullptr, "raw memory operations should have control edge");
2816
2817 switch (bt) {
2818 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2819 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2820 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2821 case T_CHAR:
2822 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2823 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2824 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2825 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2826 case T_METADATA:
2827 case T_ADDRESS:
2828 case T_OBJECT:
2829 #ifdef _LP64
2830 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2831 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2832 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2833 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2834 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
2835 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2836 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2837 }
2838 #endif
2839 {
2840 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2841 }
2842 default:
2843 ShouldNotReachHere();
2844 return (StoreNode*)nullptr;
2845 }
2846 }
2847
2848 //--------------------------bottom_type----------------------------------------
2849 const Type *StoreNode::bottom_type() const {
2850 return Type::MEMORY;
2851 }
2852
2853 //------------------------------hash-------------------------------------------
2854 uint StoreNode::hash() const {
2855 // unroll addition of interesting fields
2856 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2857
2858 // Since they are not commoned, do not hash them:
2859 return NO_HASH;
2860 }
2861
2862 // Link together multiple stores (B/S/C/I) into a longer one.
2863 //
3485 }
3486 ss.print_cr("[TraceMergeStores]: with");
3487 merged_input_value->dump("\n", false, &ss);
3488 merged_store->dump("\n", false, &ss);
3489 tty->print("%s", ss.as_string());
3490 }
3491 #endif
3492
3493 //------------------------------Ideal------------------------------------------
3494 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3495 // When a store immediately follows a relevant allocation/initialization,
3496 // try to capture it into the initialization, or hoist it above.
3497 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3498 Node* p = MemNode::Ideal_common(phase, can_reshape);
3499 if (p) return (p == NodeSentinel) ? nullptr : p;
3500
3501 Node* mem = in(MemNode::Memory);
3502 Node* address = in(MemNode::Address);
3503 Node* value = in(MemNode::ValueIn);
3504 // Back-to-back stores to same address? Fold em up. Generally
3505 // unsafe if I have intervening uses.
3506 {
3507 Node* st = mem;
3508 // If Store 'st' has more than one use, we cannot fold 'st' away.
3509 // For example, 'st' might be the final state at a conditional
3510 // return. Or, 'st' might be used by some node which is live at
3511 // the same time 'st' is live, which might be unschedulable. So,
3512 // require exactly ONE user until such time as we clone 'mem' for
3513 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3514 // true).
3515 while (st->is_Store() && st->outcnt() == 1) {
3516 // Looking at a dead closed cycle of memory?
3517 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3518 assert(Opcode() == st->Opcode() ||
3519 st->Opcode() == Op_StoreVector ||
3520 Opcode() == Op_StoreVector ||
3521 st->Opcode() == Op_StoreVectorScatter ||
3522 Opcode() == Op_StoreVectorScatter ||
3523 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3524 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3525 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3526 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3527 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3528
3529 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3530 st->as_Store()->memory_size() <= this->memory_size()) {
3531 Node* use = st->raw_out(0);
3532 if (phase->is_IterGVN()) {
3533 phase->is_IterGVN()->rehash_node_delayed(use);
3534 }
3535 // It's OK to do this in the parser, since DU info is always accurate,
3536 // and the parser always refers to nodes via SafePointNode maps.
3537 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3538 return this;
3539 }
3540 st = st->in(MemNode::Memory);
3541 }
3542 }
3543
3544
3545 // Capture an unaliased, unconditional, simple store into an initializer.
3646 const StoreVectorNode* store_vector = as_StoreVector();
3647 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3648 const Node* store_indices = store_vector->indices();
3649 const Node* mem_indices = mem_vector->indices();
3650 const Node* store_mask = store_vector->mask();
3651 const Node* mem_mask = mem_vector->mask();
3652 // Ensure types, indices, and masks match
3653 if (store_vector->vect_type() == mem_vector->vect_type() &&
3654 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3655 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3656 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3657 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3658 result = mem;
3659 }
3660 }
3661 }
3662
3663 // Store of zero anywhere into a freshly-allocated object?
3664 // Then the store is useless.
3665 // (It must already have been captured by the InitializeNode.)
3666 if (result == this &&
3667 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3668 // a newly allocated object is already all-zeroes everywhere
3669 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3670 result = mem;
3671 }
3672
3673 if (result == this) {
3674 // the store may also apply to zero-bits in an earlier object
3675 Node* prev_mem = find_previous_store(phase);
3676 // Steps (a), (b): Walk past independent stores to find an exact match.
3677 if (prev_mem != nullptr) {
3678 if (prev_mem->is_top()) {
3679 // find_previous_store returns top when the access is dead
3680 return prev_mem;
3681 }
3682 Node* prev_val = can_see_stored_value(prev_mem, phase);
3683 if (prev_val != nullptr && prev_val == val) {
3684 // prev_val and val might differ by a cast; it would be good
3685 // to keep the more informative of the two.
3686 result = mem;
3687 }
3688 }
3689 }
3690 }
3691
3692 PhaseIterGVN* igvn = phase->is_IterGVN();
3693 if (result != this && igvn != nullptr) {
4166 // Clearing a short array is faster with stores
4167 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4168 // Already know this is a large node, do not try to ideal it
4169 if (_is_large) return nullptr;
4170
4171 const int unit = BytesPerLong;
4172 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4173 if (!t) return nullptr;
4174 if (!t->is_con()) return nullptr;
4175 intptr_t raw_count = t->get_con();
4176 intptr_t size = raw_count;
4177 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4178 // Clearing nothing uses the Identity call.
4179 // Negative clears are possible on dead ClearArrays
4180 // (see jck test stmt114.stmt11402.val).
4181 if (size <= 0 || size % unit != 0) return nullptr;
4182 intptr_t count = size / unit;
4183 // Length too long; communicate this to matchers and assemblers.
4184 // Assemblers are responsible to produce fast hardware clears for it.
4185 if (size > InitArrayShortSize) {
4186 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4187 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4188 return nullptr;
4189 }
4190 if (!IdealizeClearArrayNode) return nullptr;
4191 Node *mem = in(1);
4192 if( phase->type(mem)==Type::TOP ) return nullptr;
4193 Node *adr = in(3);
4194 const Type* at = phase->type(adr);
4195 if( at==Type::TOP ) return nullptr;
4196 const TypePtr* atp = at->isa_ptr();
4197 // adjust atp to be the correct array element address type
4198 if (atp == nullptr) atp = TypePtr::BOTTOM;
4199 else atp = atp->add_offset(Type::OffsetBot);
4200 // Get base for derived pointer purposes
4201 if( adr->Opcode() != Op_AddP ) Unimplemented();
4202 Node *base = adr->in(1);
4203
4204 Node *zero = phase->makecon(TypeLong::ZERO);
4205 Node *off = phase->MakeConX(BytesPerLong);
4206 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4207 count--;
4208 while (count--) {
4209 mem = phase->transform(mem);
4210 adr = phase->transform(AddPNode::make_with_base(base, adr, off));
4211 mem = new StoreLNode(in(0), mem, adr, atp, zero, MemNode::unordered, false);
4212 }
4213 return mem;
4214 }
4215
4216 //----------------------------step_through----------------------------------
4217 // Return allocation input memory edge if it is different instance
4218 // or itself if it is the one we are looking for.
4219 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4220 Node* n = *np;
4221 assert(n->is_ClearArray(), "sanity");
4222 intptr_t offset;
4223 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4224 // This method is called only before Allocate nodes are expanded
4225 // during macro nodes expansion. Before that ClearArray nodes are
4226 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4227 // Allocate nodes are expanded) which follows allocations.
4228 assert(alloc != nullptr, "should have allocation");
4229 if (alloc->_idx == instance_id) {
4230 // Can not bypass initialization of the instance we are looking for.
4231 return false;
4234 InitializeNode* init = alloc->initialization();
4235 if (init != nullptr)
4236 *np = init->in(TypeFunc::Memory);
4237 else
4238 *np = alloc->in(TypeFunc::Memory);
4239 return true;
4240 }
4241
4242 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4243 Node* base = dest;
4244 if (raw_base) {
4245 // May be called as part of the initialization of a just allocated object
4246 base = phase->C->top();
4247 }
4248 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4249 }
4250
4251 //----------------------------clear_memory-------------------------------------
4252 // Generate code to initialize object storage to zero.
4253 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4254 intptr_t start_offset,
4255 Node* end_offset,
4256 bool raw_base,
4257 PhaseGVN* phase) {
4258 intptr_t offset = start_offset;
4259
4260 int unit = BytesPerLong;
4261 if ((offset % unit) != 0) {
4262 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4263 const TypePtr* atp = TypeRawPtr::BOTTOM;
4264 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4265 mem = phase->transform(mem);
4266 offset += BytesPerInt;
4267 }
4268 assert((offset % unit) == 0, "");
4269
4270 // Initialize the remaining stuff, if any, with a ClearArray.
4271 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, raw_base, phase);
4272 }
4273
4274 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4275 Node* start_offset,
4276 Node* end_offset,
4277 bool raw_base,
4278 PhaseGVN* phase) {
4279 if (start_offset == end_offset) {
4280 // nothing to do
4281 return mem;
4282 }
4283
4284 int unit = BytesPerLong;
4285 Node* zbase = start_offset;
4286 Node* zend = end_offset;
4287
4288 // Scale to the unit required by the CPU:
4289 if (!Matcher::init_array_count_is_in_bytes) {
4290 Node* shift = phase->intcon(exact_log2(unit));
4291 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4292 zend = phase->transform(new URShiftXNode(zend, shift) );
4293 }
4294
4295 // Bulk clear double-words
4296 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4297 Node* adr = make_address(dest, start_offset, raw_base, phase);
4298 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4299 return phase->transform(mem);
4300 }
4301
4302 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4303 intptr_t start_offset,
4304 intptr_t end_offset,
4305 bool raw_base,
4306 PhaseGVN* phase) {
4307 if (start_offset == end_offset) {
4308 // nothing to do
4309 return mem;
4310 }
4311
4312 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4313 intptr_t done_offset = end_offset;
4314 if ((done_offset % BytesPerLong) != 0) {
4315 done_offset -= BytesPerInt;
4316 }
4317 if (done_offset > start_offset) {
4318 mem = clear_memory(ctl, mem, dest,
4319 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4320 }
4321 if (done_offset < end_offset) { // emit the final 32-bit store
4322 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4323 const TypePtr* atp = TypeRawPtr::BOTTOM;
4324 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4325 mem = phase->transform(mem);
4326 done_offset += BytesPerInt;
4327 }
4328 assert(done_offset == end_offset, "");
4329 return mem;
4330 }
4331
4332 //=============================================================================
4333 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4334 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4335 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4336 #ifdef ASSERT
4337 , _pair_idx(0)
4338 #endif
4339 {
4340 init_class_id(Class_MemBar);
4341 Node* top = C->top();
4342 init_req(TypeFunc::I_O,top);
4343 init_req(TypeFunc::FramePtr,top);
4344 init_req(TypeFunc::ReturnAdr,top);
4453 PhaseIterGVN* igvn = phase->is_IterGVN();
4454 remove(igvn);
4455 // Must return either the original node (now dead) or a new node
4456 // (Do not return a top here, since that would break the uniqueness of top.)
4457 return new ConINode(TypeInt::ZERO);
4458 }
4459 }
4460 return progress ? this : nullptr;
4461 }
4462
4463 //------------------------------Value------------------------------------------
4464 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4465 if( !in(0) ) return Type::TOP;
4466 if( phase->type(in(0)) == Type::TOP )
4467 return Type::TOP;
4468 return TypeTuple::MEMBAR;
4469 }
4470
4471 //------------------------------match------------------------------------------
4472 // Construct projections for memory.
4473 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4474 switch (proj->_con) {
4475 case TypeFunc::Control:
4476 case TypeFunc::Memory:
4477 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4478 }
4479 ShouldNotReachHere();
4480 return nullptr;
4481 }
4482
4483 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4484 trailing->_kind = TrailingStore;
4485 leading->_kind = LeadingStore;
4486 #ifdef ASSERT
4487 trailing->_pair_idx = leading->_idx;
4488 leading->_pair_idx = leading->_idx;
4489 #endif
4490 }
4491
4492 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4493 trailing->_kind = TrailingLoadStore;
4740 return (req() > RawStores);
4741 }
4742
4743 void InitializeNode::set_complete(PhaseGVN* phase) {
4744 assert(!is_complete(), "caller responsibility");
4745 _is_complete = Complete;
4746
4747 // After this node is complete, it contains a bunch of
4748 // raw-memory initializations. There is no need for
4749 // it to have anything to do with non-raw memory effects.
4750 // Therefore, tell all non-raw users to re-optimize themselves,
4751 // after skipping the memory effects of this initialization.
4752 PhaseIterGVN* igvn = phase->is_IterGVN();
4753 if (igvn) igvn->add_users_to_worklist(this);
4754 }
4755
4756 // convenience function
4757 // return false if the init contains any stores already
4758 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4759 InitializeNode* init = initialization();
4760 if (init == nullptr || init->is_complete()) return false;
4761 init->remove_extra_zeroes();
4762 // for now, if this allocation has already collected any inits, bail:
4763 if (init->is_non_zero()) return false;
4764 init->set_complete(phase);
4765 return true;
4766 }
4767
4768 void InitializeNode::remove_extra_zeroes() {
4769 if (req() == RawStores) return;
4770 Node* zmem = zero_memory();
4771 uint fill = RawStores;
4772 for (uint i = fill; i < req(); i++) {
4773 Node* n = in(i);
4774 if (n->is_top() || n == zmem) continue; // skip
4775 if (fill < i) set_req(fill, n); // compact
4776 ++fill;
4777 }
4778 // delete any empty spaces created:
4779 while (fill < req()) {
4780 del_req(fill);
4924 // store node that we'd like to capture. We need to check
4925 // the uses of the MergeMemNode.
4926 mems.push(n);
4927 }
4928 } else if (n->is_Mem()) {
4929 Node* other_adr = n->in(MemNode::Address);
4930 if (other_adr == adr) {
4931 failed = true;
4932 break;
4933 } else {
4934 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4935 if (other_t_adr != nullptr) {
4936 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4937 if (other_alias_idx == alias_idx) {
4938 // A load from the same memory slice as the store right
4939 // after the InitializeNode. We check the control of the
4940 // object/array that is loaded from. If it's the same as
4941 // the store control then we cannot capture the store.
4942 assert(!n->is_Store(), "2 stores to same slice on same control?");
4943 Node* base = other_adr;
4944 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4945 base = base->in(AddPNode::Base);
4946 if (base != nullptr) {
4947 base = base->uncast();
4948 if (base->is_Proj() && base->in(0) == alloc) {
4949 failed = true;
4950 break;
4951 }
4952 }
4953 }
4954 }
4955 }
4956 } else {
4957 failed = true;
4958 break;
4959 }
4960 }
4961 }
4962 }
4963 if (failed) {
5509 // z's_done 12 16 16 16 12 16 12
5510 // z's_needed 12 16 16 16 16 16 16
5511 // zsize 0 0 0 0 4 0 4
5512 if (next_full_store < 0) {
5513 // Conservative tack: Zero to end of current word.
5514 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5515 } else {
5516 // Zero to beginning of next fully initialized word.
5517 // Or, don't zero at all, if we are already in that word.
5518 assert(next_full_store >= zeroes_needed, "must go forward");
5519 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5520 zeroes_needed = next_full_store;
5521 }
5522 }
5523
5524 if (zeroes_needed > zeroes_done) {
5525 intptr_t zsize = zeroes_needed - zeroes_done;
5526 // Do some incremental zeroing on rawmem, in parallel with inits.
5527 zeroes_done = align_down(zeroes_done, BytesPerInt);
5528 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5529 zeroes_done, zeroes_needed,
5530 true,
5531 phase);
5532 zeroes_done = zeroes_needed;
5533 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5534 do_zeroing = false; // leave the hole, next time
5535 }
5536 }
5537
5538 // Collect the store and move on:
5539 phase->replace_input_of(st, MemNode::Memory, inits);
5540 inits = st; // put it on the linearized chain
5541 set_req(i, zmem); // unhook from previous position
5542
5543 if (zeroes_done == st_off)
5544 zeroes_done = next_init_off;
5545
5546 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5547
5548 #ifdef ASSERT
5569 remove_extra_zeroes(); // clear out all the zmems left over
5570 add_req(inits);
5571
5572 if (!(UseTLAB && ZeroTLAB)) {
5573 // If anything remains to be zeroed, zero it all now.
5574 zeroes_done = align_down(zeroes_done, BytesPerInt);
5575 // if it is the last unused 4 bytes of an instance, forget about it
5576 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5577 if (zeroes_done + BytesPerLong >= size_limit) {
5578 AllocateNode* alloc = allocation();
5579 assert(alloc != nullptr, "must be present");
5580 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5581 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5582 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5583 if (zeroes_done == k->layout_helper())
5584 zeroes_done = size_limit;
5585 }
5586 }
5587 if (zeroes_done < size_limit) {
5588 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5589 zeroes_done, size_in_bytes, true, phase);
5590 }
5591 }
5592
5593 set_complete(phase);
5594 return rawmem;
5595 }
5596
5597 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5598 auto replace_proj = [&](ProjNode* proj) {
5599 C->gvn_replace_by(proj, mem);
5600 return CONTINUE;
5601 };
5602 apply_to_projs(replace_proj, TypeFunc::Memory);
5603 }
5604
5605 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5606 DUIterator_Fast imax, i = fast_outs(imax);
5607 auto replace_proj = [&](ProjNode* proj) {
5608 igvn->replace_node(proj, mem);
5806 //------------------------------Identity---------------------------------------
5807 Node* MergeMemNode::Identity(PhaseGVN* phase) {
5808 // Identity if this merge point does not record any interesting memory
5809 // disambiguations.
5810 Node* base_mem = base_memory();
5811 Node* empty_mem = empty_memory();
5812 if (base_mem != empty_mem) { // Memory path is not dead?
5813 for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
5814 Node* mem = in(i);
5815 if (mem != empty_mem && mem != base_mem) {
5816 return this; // Many memory splits; no change
5817 }
5818 }
5819 }
5820 return base_mem; // No memory splits; ID on the one true input
5821 }
5822
5823 //------------------------------Ideal------------------------------------------
5824 // This method is invoked recursively on chains of MergeMem nodes
5825 Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) {
5826 // Remove chain'd MergeMems
5827 //
5828 // This is delicate, because the each "in(i)" (i >= Raw) is interpreted
5829 // relative to the "in(Bot)". Since we are patching both at the same time,
5830 // we have to be careful to read each "in(i)" relative to the old "in(Bot)",
5831 // but rewrite each "in(i)" relative to the new "in(Bot)".
5832 Node *progress = nullptr;
5833
5834
5835 Node* old_base = base_memory();
5836 Node* empty_mem = empty_memory();
5837 if (old_base == empty_mem)
5838 return nullptr; // Dead memory path.
5839
5840 MergeMemNode* old_mbase;
5841 if (old_base != nullptr && old_base->is_MergeMem())
5842 old_mbase = old_base->as_MergeMem();
5843 else
5844 old_mbase = nullptr;
5845 Node* new_base = old_base;
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "ci/ciInlineKlass.hpp"
28 #include "ci/ciInstanceKlass.hpp"
29 #include "classfile/javaClasses.hpp"
30 #include "classfile/systemDictionary.hpp"
31 #include "classfile/vmIntrinsics.hpp"
32 #include "compiler/compileLog.hpp"
33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/c2/barrierSetC2.hpp"
35 #include "gc/shared/tlab_globals.hpp"
36 #include "memory/allocation.inline.hpp"
37 #include "memory/resourceArea.hpp"
38 #include "oops/flatArrayKlass.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "opto/addnode.hpp"
41 #include "opto/arraycopynode.hpp"
42 #include "opto/callnode.hpp"
43 #include "opto/cfgnode.hpp"
44 #include "opto/compile.hpp"
45 #include "opto/connode.hpp"
46 #include "opto/convertnode.hpp"
47 #include "opto/inlinetypenode.hpp"
48 #include "opto/loopnode.hpp"
49 #include "opto/machnode.hpp"
50 #include "opto/matcher.hpp"
51 #include "opto/memnode.hpp"
52 #include "opto/mempointer.hpp"
53 #include "opto/mulnode.hpp"
54 #include "opto/narrowptrnode.hpp"
55 #include "opto/opcodes.hpp"
56 #include "opto/phaseX.hpp"
57 #include "opto/regalloc.hpp"
58 #include "opto/regmask.hpp"
59 #include "opto/rootnode.hpp"
60 #include "opto/traceMergeStoresTag.hpp"
61 #include "opto/vectornode.hpp"
62 #include "runtime/arguments.hpp"
63 #include "utilities/align.hpp"
64 #include "utilities/copy.hpp"
65 #include "utilities/globalDefinitions.hpp"
66 #include "utilities/macros.hpp"
67 #include "utilities/powerOfTwo.hpp"
68 #include "utilities/vmError.hpp"
69
70 // Portions of code courtesy of Clifford Click
71
72 // Optimization - Graph Style
73
74 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
75
76 //=============================================================================
77 uint MemNode::size_of() const { return sizeof(*this); }
78
79 const TypePtr *MemNode::adr_type() const {
80 Node* adr = in(Address);
81 if (adr == nullptr) return nullptr; // node is dead
82 const TypePtr* cross_check = nullptr;
83 DEBUG_ONLY(cross_check = _adr_type);
84 return calculate_adr_type(adr->bottom_type(), cross_check);
85 }
134 st->print(", idx=Bot;");
135 else if (atp->index() == Compile::AliasIdxTop)
136 st->print(", idx=Top;");
137 else if (atp->index() == Compile::AliasIdxRaw)
138 st->print(", idx=Raw;");
139 else {
140 ciField* field = atp->field();
141 if (field) {
142 st->print(", name=");
143 field->print_name_on(st);
144 }
145 st->print(", idx=%d;", atp->index());
146 }
147 }
148 }
149
150 extern void print_alias_types();
151
152 #endif
153
154 // Find the memory output corresponding to the fall-through path of a call
155 static Node* find_call_fallthrough_mem_output(CallNode* call) {
156 ResourceMark rm;
157 CallProjections* projs = call->extract_projections(false, false);
158 Node* res = projs->fallthrough_memproj;
159 assert(res != nullptr, "must have a fallthrough mem output");
160 return res;
161 }
162
163 // Try to find a better memory input for a load from a strict final field
164 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
165 intptr_t offset = 0;
166 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
167 if (base == nullptr) {
168 return nullptr;
169 }
170
171 Node* base_uncasted = base->uncast();
172 if (base_uncasted->is_Proj()) {
173 Node* multi = base_uncasted->in(0);
174 if (multi->is_top()) {
175 // The pointer dies, make the memory die, too
176 return multi;
177 } else if (multi->is_Allocate()) {
178 base_local = base_uncasted->as_Proj();
179 return nullptr;
180 } else if (multi->is_Call()) {
181 if (!multi->is_CallJava() || multi->as_CallJava()->method() == nullptr || !multi->as_CallJava()->method()->return_value_is_larval()) {
182 // The oop is returned from a call, the memory can be the fallthrough output of the call
183 return find_call_fallthrough_mem_output(multi->as_Call());
184 }
185 } else if (multi->is_Start()) {
186 // The oop is a parameter
187 if (base_uncasted->as_Proj()->_con == TypeFunc::Parms && phase->C->method()->receiver_maybe_larval()) {
188 // The receiver of a constructor is similar to the result of an AllocateNode
189 base_local = base_uncasted->as_Proj();
190 return nullptr;
191 } else {
192 // Use the start memory otherwise
193 return multi->as_Start()->proj_out(TypeFunc::Memory);
194 }
195 }
196 }
197
198 return nullptr;
199 }
200
201 // Whether a call can modify a strict final field, given that the object is allocated inside the
202 // current compilation unit, or is the first parameter when the compilation root is a constructor.
203 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
204 // the target method is a subclass of the class declaring 'field'.
205 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
206 if (!call->is_CallJava()) {
207 return false;
208 }
209
210 ciMethod* target = call->as_CallJava()->method();
211 if (target == nullptr) {
212 return false;
213 } else if (target->intrinsic_id() == vmIntrinsicID::_linkToSpecial) {
214 // linkToSpecial can be used to call a constructor, used in the construction of objects in the
215 // reflection API
216 return true;
217 } else if (!target->is_object_constructor()) {
218 return false;
219 }
220
221 // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
222 // then the field is set inside the constructor, else the field must be set before the
223 // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
224 // while a field Sub.y must be set before Super::<init> is invoked.
225 // We can try to be more heroic and decide if the receiver of the constructor invocation is the
226 // object from which we are loading from. This, however, may be problematic as deciding if 2
227 // nodes are definitely different may not be trivial, especially if the graph is not canonical.
228 // As a result, it is made more conservative for now.
229 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
230 return target->holder()->is_subclass_of(field->holder());
231 }
232
233 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
234 assert(t_oop != nullptr, "sanity");
235 bool is_known_instance = t_oop->is_known_instance_field();
236 bool is_strict_final_load = false;
237
238 // After macro expansion, an allocation may become a call, changing the memory input to the
239 // memory output of that call would be illegal. As a result, disallow this transformation after
240 // macro expansion.
241 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
242 is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
243 #ifdef ASSERT
244 if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
245 assert(is_strict_final_load, "sanity check for basic cases");
246 }
247 #endif // ASSERT
248 }
249
250 if (!is_known_instance && !is_strict_final_load) {
251 return mchain;
252 }
253
254 Node* result = mchain;
255 ProjNode* base_local = nullptr;
256
257 ciField* field = nullptr;
258 if (is_strict_final_load) {
259 field = phase->C->alias_type(t_oop)->field();
260 assert(field != nullptr, "must point to a field");
261
262 Node* adr = load->in(MemNode::Address);
263 assert(phase->type(adr) == t_oop, "inconsistent type");
264 Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
265 if (tmp != nullptr) {
266 result = tmp;
267 }
268 }
269
270 uint instance_id = t_oop->instance_id();
271 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
272 Node* prev = nullptr;
273 while (prev != result) {
274 prev = result;
275 if (result == start_mem) {
276 // start_mem is the earliest memory possible
277 break;
278 }
279
280 // skip over a call which does not affect this memory slice
281 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
282 Node* proj_in = result->in(0);
283 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
284 // This is the allocation that creates the object from which we are loading from
285 break;
286 } else if (proj_in->is_Call()) {
287 // ArrayCopyNodes processed here as well
288 CallNode* call = proj_in->as_Call();
289 if (!call->may_modify(t_oop, phase)) {
290 result = call->in(TypeFunc::Memory);
291 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
292 result = call->in(TypeFunc::Memory);
293 }
294 } else if (proj_in->Opcode() == Op_Tuple) {
295 // The call will be folded, skip over it.
296 break;
297 } else if (proj_in->is_Initialize()) {
298 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
299 // Stop if this is the initialization for the object instance which
300 // contains this memory slice, otherwise skip over it.
301 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
302 break;
303 }
304 if (is_known_instance) {
305 result = proj_in->in(TypeFunc::Memory);
306 } else if (is_strict_final_load) {
307 Node* klass = alloc->in(AllocateNode::KlassNode);
308 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
309 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
310 // Allocation of another type, must be another object
311 result = proj_in->in(TypeFunc::Memory);
312 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
313 // Allocation of another object
314 result = proj_in->in(TypeFunc::Memory);
315 }
316 }
317 } else if (proj_in->is_MemBar()) {
318 ArrayCopyNode* ac = nullptr;
319 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
320 break;
321 }
322 result = proj_in->in(TypeFunc::Memory);
323 } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
324 if (is_strict_final_load) {
325 // LoadFlat and StoreFlat cannot happen to strict final fields
326 result = proj_in->in(TypeFunc::Memory);
327 }
328 } else if (proj_in->is_top()) {
329 break; // dead code
330 } else {
331 assert(false, "unexpected projection of %s", proj_in->Name());
332 }
333 } else if (result->is_ClearArray()) {
334 if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
335 // Can not bypass initialization of the instance
336 // we are looking for.
337 break;
338 }
339 // Otherwise skip it (the call updated 'result' value).
340 } else if (result->is_MergeMem()) {
341 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
342 }
343 }
344 return result;
345 }
346
347 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
348 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
349 if (t_oop == nullptr)
350 return mchain; // don't try to optimize non-oop types
351 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
352 bool is_instance = t_oop->is_known_instance_field();
353 PhaseIterGVN *igvn = phase->is_IterGVN();
354 if (is_instance && igvn != nullptr && result->is_Phi()) {
355 PhiNode *mphi = result->as_Phi();
356 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
357 const TypePtr *t = mphi->adr_type();
358 bool do_split = false;
359 // In the following cases, Load memory input can be further optimized based on
360 // its precise address type
361 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
362 do_split = true;
363 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
364 const TypeOopPtr* mem_t =
365 t->is_oopptr()->cast_to_exactness(true)
366 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
367 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
368 if (t_oop->isa_aryptr()) {
369 mem_t = mem_t->is_aryptr()
370 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
371 ->cast_to_size(t_oop->is_aryptr()->size())
372 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
373 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
374 ->with_offset(t_oop->is_aryptr()->offset())
375 ->is_aryptr();
376 }
377 do_split = mem_t == t_oop;
378 }
379 if (do_split) {
380 // clone the Phi with our address type
381 result = mphi->split_out_instance(t_adr, igvn);
382 } else {
383 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
384 }
385 }
386 return result;
387 }
388
389 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
390 uint alias_idx = phase->C->get_alias_index(tp);
391 Node *mem = mmem;
392 #ifdef ASSERT
393 {
394 // Check that current type is consistent with the alias index used during graph construction
395 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
396 bool consistent = adr_check == nullptr || adr_check->empty() ||
397 phase->C->must_alias(adr_check, alias_idx );
398 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
399 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
400 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
401 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
402 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
403 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
404 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
405 // don't assert if it is dead code.
406 consistent = true;
407 }
408 if( !consistent ) {
409 st->print("alias_idx==%d, adr_check==", alias_idx);
410 if( adr_check == nullptr ) {
411 st->print("null");
412 } else {
413 adr_check->dump();
414 }
415 st->cr();
416 print_alias_types();
417 assert(consistent, "adr_check must match alias idx");
418 }
419 }
420 #endif
1133 in_bytes(JavaThread::vthread_offset()),
1134 in_bytes(JavaThread::scopedValueCache_offset()),
1135 };
1136
1137 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1138 if (offset == offsets[i]) {
1139 return true;
1140 }
1141 }
1142 }
1143
1144 return false;
1145 }
1146 #endif
1147
1148 //----------------------------LoadNode::make-----------------------------------
1149 // Polymorphic factory method:
1150 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1151 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1152 Compile* C = gvn.C;
1153 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr(), true) == C->get_alias_index(adr_type, true), "adr and adr_type must agree");
1154
1155 // sanity check the alias category against the created node type
1156 assert(!(adr_type->isa_oopptr() &&
1157 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1158 "use LoadKlassNode instead");
1159 assert(!(adr_type->isa_aryptr() &&
1160 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1161 "use LoadRangeNode instead");
1162 // Check control edge of raw loads
1163 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1164 // oop will be recorded in oop map if load crosses safepoint
1165 rt->isa_oopptr() || is_immutable_value(adr),
1166 "raw memory operations should have control edge");
1167 LoadNode* load = nullptr;
1168 switch (bt) {
1169 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1170 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1171 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1172 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1173 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1174 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1175 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1176 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1177 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1178 case T_ARRAY:
1179 case T_OBJECT:
1180 case T_NARROWOOP:
1181 #ifdef _LP64
1182 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1183 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1184 } else
1185 #endif
1186 {
1187 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1188 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1189 }
1190 break;
1191 default:
1192 guarantee(false, "unexpected basic type %s", type2name(bt));
1193 break;
1194 }
1195 assert(load != nullptr, "LoadNode should have been created");
1196 if (unaligned) {
1197 load->set_unaligned_access();
1198 }
1199 if (mismatched) {
1200 load->set_mismatched_access();
1201 }
1202 if (unsafe) {
1203 load->set_unsafe_access();
1204 }
1205 load->set_barrier_data(barrier_data);
1206 if (load->Opcode() == Op_LoadN) {
1207 Node* ld = gvn.transform(load);
1208 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1209 }
1210
1211 return load;
1212 }
1213
1214 //------------------------------hash-------------------------------------------
1215 uint LoadNode::hash() const {
1216 // unroll addition of interesting fields
1217 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1218 }
1219
1220 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1221 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1222 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1223 bool is_stable_ary = FoldStableValues &&
1224 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1225 tp->isa_aryptr()->is_stable();
1226
1227 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1228 }
1229
1230 return false;
1231 }
1232
1233 // Is the value loaded previously stored by an arraycopy? If so return
1234 // a load node that reads from the source array so we may be able to
1235 // optimize out the ArrayCopy node later.
1236 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1237 Node* ld_adr = in(MemNode::Address);
1238 intptr_t ld_off = 0;
1239 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1240 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1241 if (ac != nullptr) {
1242 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1243
1244 Node* mem = ac->in(TypeFunc::Memory);
1245 Node* ctl = ac->in(0);
1246 Node* src = ac->in(ArrayCopyNode::Src);
1247
1255 if (ac->as_ArrayCopy()->is_clonebasic()) {
1256 assert(ld_alloc != nullptr, "need an alloc");
1257 assert(addp->is_AddP(), "address must be addp");
1258 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1259 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1260 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1261 addp->set_req(AddPNode::Base, src);
1262 addp->set_req(AddPNode::Address, src);
1263 } else {
1264 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1265 ac->as_ArrayCopy()->is_copyof_validated() ||
1266 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1267 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1268 addp->set_req(AddPNode::Base, src);
1269 addp->set_req(AddPNode::Address, src);
1270
1271 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1272 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1273 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1274
1275 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1276
1277 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1278 #ifdef _LP64
1279 diff = phase->transform(new ConvI2LNode(diff));
1280 #endif
1281 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1282
1283 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1284 addp->set_req(AddPNode::Offset, offset);
1285 }
1286 addp = phase->transform(addp);
1287 #ifdef ASSERT
1288 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1289 ld->_adr_type = adr_type;
1290 #endif
1291 ld->set_req(MemNode::Address, addp);
1292 ld->set_req(0, ctl);
1293 ld->set_req(MemNode::Memory, mem);
1294 return ld;
1295 }
1296 return nullptr;
1297 }
1298
1299 static Node* see_through_inline_type(PhaseValues* phase, const LoadNode* load, Node* base, int offset) {
1300 if (load->is_mismatched_access() || base == nullptr) {
1301 return nullptr;
1302 }
1303
1304 InlineTypeNode* vt = base->uncast()->isa_InlineType();
1305 if (vt == nullptr) {
1306 return nullptr;
1307 }
1308
1309 const TypeInstPtr* base_type = phase->type(base)->isa_instptr();
1310 if (base_type == nullptr) {
1311 return nullptr;
1312 }
1313
1314 // There can be cases when an InlineTypeNode is casted to some unrelated type, the graph is dead
1315 // but we should not recklessly fold the load
1316 ciInstanceKlass* expected_type = base_type->instance_klass();
1317 ciInlineKlass* actual_type = vt->type()->inline_klass();
1318 if (expected_type == actual_type && actual_type->get_field_by_offset(offset, false) != nullptr) {
1319 Node* value = vt->field_value_by_offset(offset, true);
1320 assert(value != nullptr, "must see some value");
1321 return value;
1322 }
1323
1324 return nullptr;
1325 }
1326
1327 // This routine exists to make sure this set of tests is done the same
1328 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1329 // will change the graph shape in a way which makes memory alive twice at the
1330 // same time (uses the Oracle model of aliasing), then some
1331 // LoadXNode::Identity will fold things back to the equivalence-class model
1332 // of aliasing.
1333 // This method may find an unencoded node instead of the corresponding encoded one.
1334 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1335 Node* ld_adr = in(MemNode::Address);
1336 intptr_t ld_off = 0;
1337 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1338 // Try to see through an InlineTypeNode
1339 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1340 if (value != nullptr) {
1341 return value;
1342 }
1343
1344 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1345 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1346
1347 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1348 uint alias_idx = atp->index();
1349 Node* result = nullptr;
1350 Node* current = st;
1351 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1352 // this load. Stop once any other kind of node is encountered.
1353 //
1354 // In principle, folding a load is moving it up until it meets a matching store.
1355 //
1356 // store(ptr, v); store(ptr, v); store(ptr, v);
1357 // membar1; -> membar1; -> load(ptr);
1358 // membar2; load(ptr); membar1;
1359 // load(ptr); membar2; membar2;
1360 //
1361 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1362 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1363 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1443
1444 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1445 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1446 if (in_vt != out_vt) {
1447 return nullptr;
1448 }
1449 }
1450 return st->in(MemNode::ValueIn);
1451 }
1452
1453 // A load from a freshly-created object always returns zero.
1454 // (This can happen after LoadNode::Ideal resets the load's memory input
1455 // to find_captured_store, which returned InitializeNode::zero_memory.)
1456 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1457 (st->in(0) == ld_alloc) &&
1458 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1459 // return a zero value for the load's basic type
1460 // (This is one of the few places where a generic PhaseTransform
1461 // can create new nodes. Think of it as lazily manifesting
1462 // virtually pre-existing constants.)
1463 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1464 if (init_value != nullptr) {
1465 const TypeAryPtr* ld_adr_type = phase->type(ld_adr)->isa_aryptr();
1466 if (ld_adr_type == nullptr) {
1467 return nullptr;
1468 }
1469
1470 // We know that this is not a flat array, the load should return the whole oop
1471 if (ld_adr_type->is_not_flat()) {
1472 return init_value;
1473 }
1474
1475 // If this is a flat array, try to see through init_value
1476 if (init_value->is_EncodeP()) {
1477 init_value = init_value->in(1);
1478 }
1479 if (!init_value->is_InlineType() || ld_adr_type->field_offset() == Type::Offset::bottom) {
1480 return nullptr;
1481 }
1482
1483 ciInlineKlass* vk = phase->type(init_value)->inline_klass();
1484 int field_offset_in_payload = ld_adr_type->field_offset().get();
1485 if (field_offset_in_payload == vk->null_marker_offset_in_payload()) {
1486 return init_value->as_InlineType()->get_null_marker();
1487 } else {
1488 return init_value->as_InlineType()->field_value_by_offset(field_offset_in_payload + vk->payload_offset(), true);
1489 }
1490 }
1491 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1492 if (value_basic_type() != T_VOID) {
1493 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1494 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1495 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1496 // by the ArrayCopyNode.
1497 return phase->zerocon(value_basic_type());
1498 }
1499 } else {
1500 // TODO: materialize all-zero vector constant
1501 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1502 }
1503 }
1504
1505 // A load from an initialization barrier can match a captured store.
1506 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1507 InitializeNode* init = st->in(0)->as_Initialize();
1508 AllocateNode* alloc = init->allocation();
1509 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1510 // examine a captured store value
1511 st = init->find_captured_store(ld_off, memory_size(), phase);
1524 base = bs->step_over_gc_barrier(base);
1525 if (base != nullptr && base->is_Proj() &&
1526 base->as_Proj()->_con == TypeFunc::Parms &&
1527 base->in(0)->is_CallStaticJava() &&
1528 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1529 return base->in(0)->in(TypeFunc::Parms);
1530 }
1531 }
1532
1533 break;
1534 }
1535
1536 return nullptr;
1537 }
1538
1539 //----------------------is_instance_field_load_with_local_phi------------------
1540 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1541 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1542 in(Address)->is_AddP() ) {
1543 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1544 // Only known instances and immutable fields
1545 if( t_oop != nullptr &&
1546 (t_oop->is_ptr_to_strict_final_field() ||
1547 t_oop->is_known_instance_field()) &&
1548 t_oop->offset() != Type::OffsetBot &&
1549 t_oop->offset() != Type::OffsetTop) {
1550 return true;
1551 }
1552 }
1553 return false;
1554 }
1555
1556 //------------------------------Identity---------------------------------------
1557 // Loads are identity if previous store is to same address
1558 Node* LoadNode::Identity(PhaseGVN* phase) {
1559 // If the previous store-maker is the right kind of Store, and the store is
1560 // to the same address, then we are equal to the value stored.
1561 Node* mem = in(Memory);
1562 Node* value = can_see_stored_value_through_membars(mem, phase);
1563 if( value ) {
1564 // byte, short & char stores truncate naturally.
1565 // A load has to load the truncated value which requires
1566 // some sort of masking operation and that requires an
1567 // Ideal call instead of an Identity call.
1568 if (memory_size() < BytesPerInt) {
1569 // If the input to the store does not fit with the load's result type,
1570 // it must be truncated via an Ideal call.
1571 if (!phase->type(value)->higher_equal(phase->type(this)))
1572 return this;
1573 }
1574
1575 if (phase->type(value)->isa_ptr() && phase->type(this)->isa_narrowoop()) {
1576 return this;
1577 }
1578 // (This works even when value is a Con, but LoadNode::Value
1579 // usually runs first, producing the singleton type of the Con.)
1580 if (!has_pinned_control_dependency() || value->is_Con()) {
1581 return value;
1582 } else {
1583 return this;
1584 }
1585 }
1586
1587 if (has_pinned_control_dependency()) {
1588 return this;
1589 }
1590 // Search for an existing data phi which was generated before for the same
1591 // instance's field to avoid infinite generation of phis in a loop.
1592 Node *region = mem->in(0);
1593 if (is_instance_field_load_with_local_phi(region)) {
1594 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1595 int this_index = phase->C->get_alias_index(addr_t);
1596 int this_offset = addr_t->offset();
1597 int this_iid = addr_t->instance_id();
1598 if (!addr_t->is_known_instance() &&
1599 addr_t->is_ptr_to_strict_final_field()) {
1600 // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1601 intptr_t ignore = 0;
1602 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1603 if (base == nullptr) {
1604 return this;
1605 }
1606 this_iid = base->_idx;
1607 }
1608 const Type* this_type = bottom_type();
1609 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1610 Node* phi = region->fast_out(i);
1611 if (phi->is_Phi() && phi != mem &&
1612 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1613 return phi;
1614 }
1615 }
1616 }
1617
1618 return this;
1619 }
1620
2155 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2156 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2157
2158 // Skip up past a SafePoint control. Cannot do this for Stores because
2159 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2160 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2161 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2162 !addr_mark &&
2163 (depends_only_on_test() || has_unknown_control_dependency())) {
2164 ctrl = ctrl->in(0);
2165 set_req(MemNode::Control,ctrl);
2166 return this;
2167 }
2168
2169 intptr_t ignore = 0;
2170 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2171 if (base != nullptr
2172 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2173 // Check for useless control edge in some common special cases
2174 if (in(MemNode::Control) != nullptr
2175 // TODO 8350865 Can we re-enable this?
2176 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2177 && can_remove_control()
2178 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2179 && all_controls_dominate(base, phase->C->start(), phase)) {
2180 // A method-invariant, non-null address (constant or 'this' argument).
2181 set_req(MemNode::Control, nullptr);
2182 return this;
2183 }
2184 }
2185
2186 Node* mem = in(MemNode::Memory);
2187 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2188
2189 if (can_reshape && (addr_t != nullptr)) {
2190 // try to optimize our memory input
2191 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2192 if (opt_mem != mem) {
2193 set_req_X(MemNode::Memory, opt_mem, phase);
2194 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2195 return this;
2196 }
2309 // No match.
2310 return nullptr;
2311 }
2312
2313 //------------------------------Value-----------------------------------------
2314 const Type* LoadNode::Value(PhaseGVN* phase) const {
2315 // Either input is TOP ==> the result is TOP
2316 Node* mem = in(MemNode::Memory);
2317 const Type *t1 = phase->type(mem);
2318 if (t1 == Type::TOP) return Type::TOP;
2319 Node* adr = in(MemNode::Address);
2320 const TypePtr* tp = phase->type(adr)->isa_ptr();
2321 if (tp == nullptr || tp->empty()) return Type::TOP;
2322 int off = tp->offset();
2323 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2324 Compile* C = phase->C;
2325
2326 // If load can see a previous constant store, use that.
2327 Node* value = can_see_stored_value_through_membars(mem, phase);
2328 if (value != nullptr && value->is_Con()) {
2329 if (phase->type(value)->isa_ptr() && _type->isa_narrowoop()) {
2330 return phase->type(value)->make_narrowoop();
2331 } else {
2332 assert(value->bottom_type()->higher_equal(_type), "sanity");
2333 return phase->type(value);
2334 }
2335 }
2336 // Try to guess loaded type from pointer type
2337 if (tp->isa_aryptr()) {
2338 const TypeAryPtr* ary = tp->is_aryptr();
2339 const Type* t = ary->elem();
2340
2341 // Determine whether the reference is beyond the header or not, by comparing
2342 // the offset against the offset of the start of the array's data.
2343 // Different array types begin at slightly different offsets (12 vs. 16).
2344 // We choose T_BYTE as an example base type that is least restrictive
2345 // as to alignment, which will therefore produce the smallest
2346 // possible base offset.
2347 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2348 const bool off_beyond_header = (off >= min_base_off);
2349
2350 // Try to constant-fold a stable array element.
2351 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2352 // Make sure the reference is not into the header and the offset is constant
2353 ciObject* aobj = ary->const_oop();
2354 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2355 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2356 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off, ary->field_offset().get(),
2357 stable_dimension,
2358 value_basic_type(), is_unsigned());
2359 if (con_type != nullptr) {
2360 return con_type;
2361 }
2362 }
2363 }
2364
2365 // Don't do this for integer types. There is only potential profit if
2366 // the element type t is lower than _type; that is, for int types, if _type is
2367 // more restrictive than t. This only happens here if one is short and the other
2368 // char (both 16 bits), and in those cases we've made an intentional decision
2369 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2370 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2371 //
2372 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2373 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2374 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2375 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2376 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2377 // In fact, that could have been the original type of p1, and p1 could have
2378 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2379 // expression (LShiftL quux 3) independently optimized to the constant 8.
2380 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2381 && (_type->isa_vect() == nullptr)
2382 && !ary->is_flat()
2383 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2384 // t might actually be lower than _type, if _type is a unique
2385 // concrete subclass of abstract class t.
2386 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2387 const Type* jt = t->join_speculative(_type);
2388 // In any case, do not allow the join, per se, to empty out the type.
2389 if (jt->empty() && !t->empty()) {
2390 // This can happen if a interface-typed array narrows to a class type.
2391 jt = _type;
2392 }
2393 #ifdef ASSERT
2394 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2395 // The pointers in the autobox arrays are always non-null
2396 Node* base = adr->in(AddPNode::Base);
2397 if ((base != nullptr) && base->is_DecodeN()) {
2398 // Get LoadN node which loads IntegerCache.cache field
2399 base = base->in(1);
2400 }
2401 if ((base != nullptr) && base->is_Con()) {
2402 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2403 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2404 // It could be narrow oop
2405 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2406 }
2407 }
2408 }
2409 #endif
2410 return jt;
2411 }
2412 }
2413 } else if (tp->base() == Type::InstPtr) {
2414 assert( off != Type::OffsetBot ||
2415 // arrays can be cast to Objects
2416 !tp->isa_instptr() ||
2417 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2418 // Default value load
2419 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2420 // unsafe field access may not have a constant offset
2421 is_unsafe_access(),
2422 "Field accesses must be precise" );
2423 // For oop loads, we expect the _type to be precise.
2424
2425 const TypeInstPtr* tinst = tp->is_instptr();
2426 BasicType bt = value_basic_type();
2427
2428 // Fold loads of the field map
2429 if (tinst != nullptr) {
2430 ciInstanceKlass* ik = tinst->instance_klass();
2431 int offset = tinst->offset();
2432 if (ik == phase->C->env()->Class_klass()) {
2433 ciType* t = tinst->java_mirror_type();
2434 if (t != nullptr && t->is_inlinetype() && offset == t->as_inline_klass()->field_map_offset()) {
2435 ciConstant map = t->as_inline_klass()->get_field_map();
2436 bool is_narrow_oop = (bt == T_NARROWOOP);
2437 return Type::make_from_constant(map, true, 1, is_narrow_oop);
2438 }
2439 }
2440 }
2441
2442 // Optimize loads from constant fields.
2443 ciObject* const_oop = tinst->const_oop();
2444 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2445 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2446 if (con_type != nullptr) {
2447 return con_type;
2448 }
2449 }
2450 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2451 assert(off != Type::OffsetBot ||
2452 !tp->isa_instklassptr() ||
2453 // arrays can be cast to Objects
2454 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2455 // also allow array-loading from the primary supertype
2456 // array during subtype checks
2457 Opcode() == Op_LoadKlass,
2458 "Field accesses must be precise");
2459 // For klass/static loads, we expect the _type to be precise
2460 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2461 /* With mirrors being an indirect in the Klass*
2462 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2463 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2464 *
2465 * So check the type and klass of the node before the LoadP.
2472 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2473 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2474 return TypeInstPtr::make(klass->java_mirror());
2475 }
2476 }
2477 }
2478
2479 const TypeKlassPtr *tkls = tp->isa_klassptr();
2480 if (tkls != nullptr) {
2481 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2482 ciKlass* klass = tkls->exact_klass();
2483 // We are loading a field from a Klass metaobject whose identity
2484 // is known at compile time (the type is "exact" or "precise").
2485 // Check for fields we know are maintained as constants by the VM.
2486 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2487 // The field is Klass::_super_check_offset. Return its (constant) value.
2488 // (Folds up type checking code.)
2489 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2490 return TypeInt::make(klass->super_check_offset());
2491 }
2492 if (klass->is_inlinetype() && tkls->offset() == in_bytes(InstanceKlass::acmp_maps_offset_offset())) {
2493 return TypeInt::make(klass->as_inline_klass()->field_map_offset());
2494 }
2495 if (klass->is_obj_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset())) {
2496 // Fold loads from LibraryCallKit::load_default_refined_array_klass
2497 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr();
2498 }
2499 if (klass->is_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::properties_offset())) {
2500 assert(klass->is_type_array_klass() || tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2501 return TypeInt::make((jint)klass->as_array_klass()->properties().value());
2502 }
2503 if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2504 assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2505 return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2506 }
2507 if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2508 // The field is Klass::_prototype_header. Return its (constant) value.
2509 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2510 return TypeX::make(klass->prototype_header());
2511 }
2512 // Compute index into primary_supers array
2513 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2514 // Check for overflowing; use unsigned compare to handle the negative case.
2515 if( depth < ciKlass::primary_super_limit() ) {
2516 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2517 // (Folds up type checking code.)
2518 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2519 ciKlass *ss = klass->super_of_depth(depth);
2520 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2521 }
2522 const Type* aift = load_array_final_field(tkls, klass);
2523 if (aift != nullptr) return aift;
2524 }
2525
2526 // We can still check if we are loading from the primary_supers array at a
2527 // shallow enough depth. Even though the klass is not exact, entries less
2528 // than or equal to its super depth are correct.
2529 if (tkls->is_loaded()) {
2530 ciKlass* klass = nullptr;
2564 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2565 // The key property of this type is that it folds up tests
2566 // for array-ness, since it proves that the layout_helper is positive.
2567 // Thus, a generic value like the basic object layout helper works fine.
2568 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2569 }
2570 }
2571
2572 // If we are loading from a freshly-allocated object/array, produce a zero.
2573 // Things to check:
2574 // 1. Load is beyond the header: headers are not guaranteed to be zero
2575 // 2. Load is not vectorized: vectors have no zero constant
2576 // 3. Load has no matching store, i.e. the input is the initial memory state
2577 const TypeOopPtr* tinst = tp->isa_oopptr();
2578 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2579 bool is_not_vect = (_type->isa_vect() == nullptr);
2580 if (is_not_header && is_not_vect) {
2581 Node* mem = in(MemNode::Memory);
2582 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2583 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2584 // TODO 8350865 Scalar replacement does not work well for flat arrays.
2585 // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2586 // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2587 // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2588 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2589 intptr_t offset = 0;
2590 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2591 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2592 if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2593 return _type;
2594 }
2595 }
2596 return Type::get_zero_type(_type->basic_type());
2597 }
2598 }
2599 if (!UseCompactObjectHeaders) {
2600 Node* alloc = is_new_object_mark_load();
2601 if (alloc != nullptr) {
2602 if (Arguments::is_valhalla_enabled()) {
2603 // The mark word may contain property bits (inline, flat, null-free)
2604 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2605 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2606 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2607 return TypeX::make(tkls->exact_klass()->prototype_header());
2608 }
2609 } else {
2610 return TypeX::make(markWord::prototype().value());
2611 }
2612 }
2613 }
2614
2615 return _type;
2616 }
2617
2618 //------------------------------match_edge-------------------------------------
2619 // Do we Match on this edge index or not? Match only the address.
2620 uint LoadNode::match_edge(uint idx) const {
2621 return idx == MemNode::Address;
2622 }
2623
2624 //--------------------------LoadBNode::Ideal--------------------------------------
2625 //
2626 // If the previous store is to the same address as this load,
2627 // and the value stored was larger than a byte, replace this load
2628 // with the value stored truncated to a byte. If no truncation is
2629 // needed, the replacement is done in LoadNode::Identity().
2630 //
2631 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2740 }
2741 }
2742 // Identity call will handle the case where truncation is not needed.
2743 return LoadNode::Ideal(phase, can_reshape);
2744 }
2745
2746 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2747 Node* mem = in(MemNode::Memory);
2748 Node* value = can_see_stored_value_through_membars(mem, phase);
2749 if (value != nullptr && value->is_Con() &&
2750 !value->bottom_type()->higher_equal(_type)) {
2751 // If the input to the store does not fit with the load's result type,
2752 // it must be truncated. We can't delay until Ideal call since
2753 // a singleton Value is needed for split_thru_phi optimization.
2754 int con = value->get_int();
2755 return TypeInt::make((con << 16) >> 16);
2756 }
2757 return LoadNode::Value(phase);
2758 }
2759
2760 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2761 // Can see the corresponding value, may need to add an EncodeP
2762 Node* value = can_see_stored_value_through_membars(in(Memory), phase);
2763 if (value != nullptr && phase->type(value)->isa_ptr() && type()->isa_narrowoop()) {
2764 return new EncodePNode(value, type());
2765 }
2766
2767 // Identity call will handle the case where EncodeP is unnecessary
2768 return LoadNode::Ideal(phase, can_reshape);
2769 }
2770
2771 //=============================================================================
2772 //----------------------------LoadKlassNode::make------------------------------
2773 // Polymorphic factory method:
2774 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2775 // sanity check the alias category against the created node type
2776 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2777 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2778 #ifdef _LP64
2779 if (adr_type->is_ptr_to_narrowklass()) {
2780 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2781 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2782 }
2783 #endif
2784 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2785 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2786 }
2787
2788 //------------------------------Value------------------------------------------
2789 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2790 return klass_value_common(phase);
2823 }
2824 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2825 }
2826 if (!t->is_klass()) {
2827 // a primitive Class (e.g., int.class) has null for a klass field
2828 return TypePtr::NULL_PTR;
2829 }
2830 // Fold up the load of the hidden field
2831 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2832 }
2833 // non-constant mirror, so we can't tell what's going on
2834 }
2835 if (!tinst->is_loaded())
2836 return _type; // Bail out if not loaded
2837 if (offset == oopDesc::klass_offset_in_bytes()) {
2838 return tinst->as_klass_type(true);
2839 }
2840 }
2841
2842 // Check for loading klass from an array
2843 const TypeAryPtr* tary = tp->isa_aryptr();
2844 if (tary != nullptr &&
2845 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2846 return tary->as_klass_type(true)->is_aryklassptr();
2847 }
2848
2849 // Check for loading klass from an array klass
2850 const TypeKlassPtr *tkls = tp->isa_klassptr();
2851 if (tkls != nullptr && !StressReflectiveCode) {
2852 if (!tkls->is_loaded())
2853 return _type; // Bail out if not loaded
2854 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2855 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2856 // // Always returning precise element type is incorrect,
2857 // // e.g., element type could be object and array may contain strings
2858 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2859
2860 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2861 // according to the element type's subclassing.
2862 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2863 }
2864 if (tkls->isa_aryklassptr() != nullptr && tkls->klass_is_exact() &&
2865 !tkls->exact_klass()->is_type_array_klass() &&
2866 tkls->offset() == in_bytes(Klass::super_offset())) {
2867 // We are loading the super klass of a refined array klass, return the non-refined klass pointer
2868 assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2869 return tkls->is_aryklassptr()->with_offset(0)->cast_to_non_refined();
2870 }
2871 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2872 tkls->offset() == in_bytes(Klass::super_offset())) {
2873 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2874 // The field is Klass::_super. Return its (constant) value.
2875 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2876 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2877 }
2878 }
2879
2880 if (tkls != nullptr && !UseSecondarySupersCache
2881 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2882 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2883 return TypePtr::NULL_PTR;
2884 }
2885
2886 // Bailout case
2887 return LoadNode::Value(phase);
2888 }
2889
2890 //------------------------------Identity---------------------------------------
2913 base = bs->step_over_gc_barrier(base);
2914 }
2915
2916 // We can fetch the klass directly through an AllocateNode.
2917 // This works even if the klass is not constant (clone or newArray).
2918 if (offset == oopDesc::klass_offset_in_bytes()) {
2919 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2920 if (allocated_klass != nullptr) {
2921 return allocated_klass;
2922 }
2923 }
2924
2925 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2926 // See inline_native_Class_query for occurrences of these patterns.
2927 // Java Example: x.getClass().isAssignableFrom(y)
2928 //
2929 // This improves reflective code, often making the Class
2930 // mirror go completely dead. (Current exception: Class
2931 // mirrors may appear in debug info, but we could clean them out by
2932 // introducing a new debug info operator for Klass.java_mirror).
2933 //
2934 // This optimization does not apply to arrays because if k is not a
2935 // constant, it was obtained via load_klass which returns the refined type
2936 // and '.java_mirror.as_klass' should return the Java type instead.
2937
2938 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2939 && offset == java_lang_Class::klass_offset()) {
2940 if (base->is_Load()) {
2941 Node* base2 = base->in(MemNode::Address);
2942 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2943 Node* adr2 = base2->in(MemNode::Address);
2944 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2945 if (tkls != nullptr && !tkls->empty()
2946 && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2947 && adr2->is_AddP()) {
2948 int mirror_field = in_bytes(Klass::java_mirror_offset());
2949 if (tkls->offset() == mirror_field) {
2950 #ifdef ASSERT
2951 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2952 assert(tkls2->offset() == 0, "not a load of java_mirror");
2953 #endif
2954 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2955 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2956 return adr2->in(AddPNode::Address);
2957 }
2958 }
2959 }
2960 }
2961 }
2962
2963 return this;
2964 }
2965
2966 LoadNode* LoadNode::clone_pinned() const {
2967 LoadNode* ld = clone()->as_Load();
3094 // Polymorphic factory method:
3095 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
3096 assert((mo == unordered || mo == release), "unexpected");
3097 Compile* C = gvn.C;
3098 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
3099 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
3100 ctl != nullptr, "raw memory operations should have control edge");
3101
3102 switch (bt) {
3103 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
3104 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
3105 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
3106 case T_CHAR:
3107 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
3108 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3109 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
3110 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3111 case T_METADATA:
3112 case T_ADDRESS:
3113 case T_OBJECT:
3114 case T_ARRAY:
3115 #ifdef _LP64
3116 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3117 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
3118 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
3119 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
3120 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
3121 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
3122 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
3123 }
3124 #endif
3125 {
3126 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
3127 }
3128 default:
3129 guarantee(false, "unexpected basic type %s", type2name(bt));
3130 return (StoreNode*)nullptr;
3131 }
3132 }
3133
3134 //--------------------------bottom_type----------------------------------------
3135 const Type *StoreNode::bottom_type() const {
3136 return Type::MEMORY;
3137 }
3138
3139 //------------------------------hash-------------------------------------------
3140 uint StoreNode::hash() const {
3141 // unroll addition of interesting fields
3142 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
3143
3144 // Since they are not commoned, do not hash them:
3145 return NO_HASH;
3146 }
3147
3148 // Link together multiple stores (B/S/C/I) into a longer one.
3149 //
3771 }
3772 ss.print_cr("[TraceMergeStores]: with");
3773 merged_input_value->dump("\n", false, &ss);
3774 merged_store->dump("\n", false, &ss);
3775 tty->print("%s", ss.as_string());
3776 }
3777 #endif
3778
3779 //------------------------------Ideal------------------------------------------
3780 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3781 // When a store immediately follows a relevant allocation/initialization,
3782 // try to capture it into the initialization, or hoist it above.
3783 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3784 Node* p = MemNode::Ideal_common(phase, can_reshape);
3785 if (p) return (p == NodeSentinel) ? nullptr : p;
3786
3787 Node* mem = in(MemNode::Memory);
3788 Node* address = in(MemNode::Address);
3789 Node* value = in(MemNode::ValueIn);
3790 // Back-to-back stores to same address? Fold em up. Generally
3791 // unsafe if I have intervening uses...
3792 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3793 Node* st = mem;
3794 // If Store 'st' has more than one use, we cannot fold 'st' away.
3795 // For example, 'st' might be the final state at a conditional
3796 // return. Or, 'st' might be used by some node which is live at
3797 // the same time 'st' is live, which might be unschedulable. So,
3798 // require exactly ONE user until such time as we clone 'mem' for
3799 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3800 // true).
3801 while (st->is_Store() && st->outcnt() == 1) {
3802 // Looking at a dead closed cycle of memory?
3803 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3804 assert(Opcode() == st->Opcode() ||
3805 st->Opcode() == Op_StoreVector ||
3806 Opcode() == Op_StoreVector ||
3807 st->Opcode() == Op_StoreVectorScatter ||
3808 Opcode() == Op_StoreVectorScatter ||
3809 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3810 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3811 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3812 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3813 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3814 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3815
3816 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3817 st->as_Store()->memory_size() <= this->memory_size()) {
3818 Node* use = st->raw_out(0);
3819 if (phase->is_IterGVN()) {
3820 phase->is_IterGVN()->rehash_node_delayed(use);
3821 }
3822 // It's OK to do this in the parser, since DU info is always accurate,
3823 // and the parser always refers to nodes via SafePointNode maps.
3824 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3825 return this;
3826 }
3827 st = st->in(MemNode::Memory);
3828 }
3829 }
3830
3831
3832 // Capture an unaliased, unconditional, simple store into an initializer.
3933 const StoreVectorNode* store_vector = as_StoreVector();
3934 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3935 const Node* store_indices = store_vector->indices();
3936 const Node* mem_indices = mem_vector->indices();
3937 const Node* store_mask = store_vector->mask();
3938 const Node* mem_mask = mem_vector->mask();
3939 // Ensure types, indices, and masks match
3940 if (store_vector->vect_type() == mem_vector->vect_type() &&
3941 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3942 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3943 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3944 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3945 result = mem;
3946 }
3947 }
3948 }
3949
3950 // Store of zero anywhere into a freshly-allocated object?
3951 // Then the store is useless.
3952 // (It must already have been captured by the InitializeNode.)
3953 if (result == this && ReduceFieldZeroing) {
3954 // a newly allocated object is already all-zeroes everywhere
3955 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3956 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3957 result = mem;
3958 }
3959
3960 if (result == this && phase->type(val)->is_zero_type()) {
3961 // the store may also apply to zero-bits in an earlier object
3962 Node* prev_mem = find_previous_store(phase);
3963 // Steps (a), (b): Walk past independent stores to find an exact match.
3964 if (prev_mem != nullptr) {
3965 if (prev_mem->is_top()) {
3966 // find_previous_store returns top when the access is dead
3967 return prev_mem;
3968 }
3969 Node* prev_val = can_see_stored_value(prev_mem, phase);
3970 if (prev_val != nullptr && prev_val == val) {
3971 // prev_val and val might differ by a cast; it would be good
3972 // to keep the more informative of the two.
3973 result = mem;
3974 }
3975 }
3976 }
3977 }
3978
3979 PhaseIterGVN* igvn = phase->is_IterGVN();
3980 if (result != this && igvn != nullptr) {
4453 // Clearing a short array is faster with stores
4454 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4455 // Already know this is a large node, do not try to ideal it
4456 if (_is_large) return nullptr;
4457
4458 const int unit = BytesPerLong;
4459 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4460 if (!t) return nullptr;
4461 if (!t->is_con()) return nullptr;
4462 intptr_t raw_count = t->get_con();
4463 intptr_t size = raw_count;
4464 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4465 // Clearing nothing uses the Identity call.
4466 // Negative clears are possible on dead ClearArrays
4467 // (see jck test stmt114.stmt11402.val).
4468 if (size <= 0 || size % unit != 0) return nullptr;
4469 intptr_t count = size / unit;
4470 // Length too long; communicate this to matchers and assemblers.
4471 // Assemblers are responsible to produce fast hardware clears for it.
4472 if (size > InitArrayShortSize) {
4473 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4474 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4475 return nullptr;
4476 }
4477 if (!IdealizeClearArrayNode) return nullptr;
4478 Node *mem = in(1);
4479 if( phase->type(mem)==Type::TOP ) return nullptr;
4480 Node *adr = in(3);
4481 const Type* at = phase->type(adr);
4482 if( at==Type::TOP ) return nullptr;
4483 const TypePtr* atp = at->isa_ptr();
4484 // adjust atp to be the correct array element address type
4485 if (atp == nullptr) atp = TypePtr::BOTTOM;
4486 else atp = atp->add_offset(Type::OffsetBot);
4487 // Get base for derived pointer purposes
4488 if( adr->Opcode() != Op_AddP ) Unimplemented();
4489 Node *base = adr->in(1);
4490
4491 Node *val = in(4);
4492 Node *off = phase->MakeConX(BytesPerLong);
4493 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4494 count--;
4495 while (count--) {
4496 mem = phase->transform(mem);
4497 adr = phase->transform(AddPNode::make_with_base(base,adr,off));
4498 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4499 }
4500 return mem;
4501 }
4502
4503 //----------------------------step_through----------------------------------
4504 // Return allocation input memory edge if it is different instance
4505 // or itself if it is the one we are looking for.
4506 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4507 Node* n = *np;
4508 assert(n->is_ClearArray(), "sanity");
4509 intptr_t offset;
4510 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4511 // This method is called only before Allocate nodes are expanded
4512 // during macro nodes expansion. Before that ClearArray nodes are
4513 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4514 // Allocate nodes are expanded) which follows allocations.
4515 assert(alloc != nullptr, "should have allocation");
4516 if (alloc->_idx == instance_id) {
4517 // Can not bypass initialization of the instance we are looking for.
4518 return false;
4521 InitializeNode* init = alloc->initialization();
4522 if (init != nullptr)
4523 *np = init->in(TypeFunc::Memory);
4524 else
4525 *np = alloc->in(TypeFunc::Memory);
4526 return true;
4527 }
4528
4529 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4530 Node* base = dest;
4531 if (raw_base) {
4532 // May be called as part of the initialization of a just allocated object
4533 base = phase->C->top();
4534 }
4535 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4536 }
4537
4538 //----------------------------clear_memory-------------------------------------
4539 // Generate code to initialize object storage to zero.
4540 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4541 Node* val,
4542 Node* raw_val,
4543 intptr_t start_offset,
4544 Node* end_offset,
4545 bool raw_base,
4546 PhaseGVN* phase) {
4547 intptr_t offset = start_offset;
4548
4549 int unit = BytesPerLong;
4550 if ((offset % unit) != 0) {
4551 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4552 const TypePtr* atp = TypeRawPtr::BOTTOM;
4553 if (val != nullptr) {
4554 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4555 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4556 } else {
4557 assert(raw_val == nullptr, "val may not be null");
4558 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4559 }
4560 mem = phase->transform(mem);
4561 offset += BytesPerInt;
4562 }
4563 assert((offset % unit) == 0, "");
4564
4565 // Initialize the remaining stuff, if any, with a ClearArray.
4566 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, raw_base, phase);
4567 }
4568
4569 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4570 Node* raw_val,
4571 Node* start_offset,
4572 Node* end_offset,
4573 bool raw_base,
4574 PhaseGVN* phase) {
4575 if (start_offset == end_offset) {
4576 // nothing to do
4577 return mem;
4578 }
4579
4580 int unit = BytesPerLong;
4581 Node* zbase = start_offset;
4582 Node* zend = end_offset;
4583
4584 // Scale to the unit required by the CPU:
4585 if (!Matcher::init_array_count_is_in_bytes) {
4586 Node* shift = phase->intcon(exact_log2(unit));
4587 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4588 zend = phase->transform(new URShiftXNode(zend, shift) );
4589 }
4590
4591 // Bulk clear double-words
4592 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4593 Node* adr = make_address(dest, start_offset, raw_base, phase);
4594 if (raw_val == nullptr) {
4595 raw_val = phase->MakeConX(0);
4596 }
4597 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4598 return phase->transform(mem);
4599 }
4600
4601 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4602 Node* val,
4603 Node* raw_val,
4604 intptr_t start_offset,
4605 intptr_t end_offset,
4606 bool raw_base,
4607 PhaseGVN* phase) {
4608 if (start_offset == end_offset) {
4609 // nothing to do
4610 return mem;
4611 }
4612
4613 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4614 intptr_t done_offset = end_offset;
4615 if ((done_offset % BytesPerLong) != 0) {
4616 done_offset -= BytesPerInt;
4617 }
4618 if (done_offset > start_offset) {
4619 mem = clear_memory(ctl, mem, dest, val, raw_val,
4620 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4621 }
4622 if (done_offset < end_offset) { // emit the final 32-bit store
4623 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4624 const TypePtr* atp = TypeRawPtr::BOTTOM;
4625 if (val != nullptr) {
4626 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4627 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4628 } else {
4629 assert(raw_val == nullptr, "val may not be null");
4630 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4631 }
4632 mem = phase->transform(mem);
4633 done_offset += BytesPerInt;
4634 }
4635 assert(done_offset == end_offset, "");
4636 return mem;
4637 }
4638
4639 //=============================================================================
4640 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4641 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4642 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4643 #ifdef ASSERT
4644 , _pair_idx(0)
4645 #endif
4646 {
4647 init_class_id(Class_MemBar);
4648 Node* top = C->top();
4649 init_req(TypeFunc::I_O,top);
4650 init_req(TypeFunc::FramePtr,top);
4651 init_req(TypeFunc::ReturnAdr,top);
4760 PhaseIterGVN* igvn = phase->is_IterGVN();
4761 remove(igvn);
4762 // Must return either the original node (now dead) or a new node
4763 // (Do not return a top here, since that would break the uniqueness of top.)
4764 return new ConINode(TypeInt::ZERO);
4765 }
4766 }
4767 return progress ? this : nullptr;
4768 }
4769
4770 //------------------------------Value------------------------------------------
4771 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4772 if( !in(0) ) return Type::TOP;
4773 if( phase->type(in(0)) == Type::TOP )
4774 return Type::TOP;
4775 return TypeTuple::MEMBAR;
4776 }
4777
4778 //------------------------------match------------------------------------------
4779 // Construct projections for memory.
4780 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4781 switch (proj->_con) {
4782 case TypeFunc::Control:
4783 case TypeFunc::Memory:
4784 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4785 }
4786 ShouldNotReachHere();
4787 return nullptr;
4788 }
4789
4790 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4791 trailing->_kind = TrailingStore;
4792 leading->_kind = LeadingStore;
4793 #ifdef ASSERT
4794 trailing->_pair_idx = leading->_idx;
4795 leading->_pair_idx = leading->_idx;
4796 #endif
4797 }
4798
4799 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4800 trailing->_kind = TrailingLoadStore;
5047 return (req() > RawStores);
5048 }
5049
5050 void InitializeNode::set_complete(PhaseGVN* phase) {
5051 assert(!is_complete(), "caller responsibility");
5052 _is_complete = Complete;
5053
5054 // After this node is complete, it contains a bunch of
5055 // raw-memory initializations. There is no need for
5056 // it to have anything to do with non-raw memory effects.
5057 // Therefore, tell all non-raw users to re-optimize themselves,
5058 // after skipping the memory effects of this initialization.
5059 PhaseIterGVN* igvn = phase->is_IterGVN();
5060 if (igvn) igvn->add_users_to_worklist(this);
5061 }
5062
5063 // convenience function
5064 // return false if the init contains any stores already
5065 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
5066 InitializeNode* init = initialization();
5067 if (init == nullptr || init->is_complete()) {
5068 return false;
5069 }
5070 init->remove_extra_zeroes();
5071 // for now, if this allocation has already collected any inits, bail:
5072 if (init->is_non_zero()) return false;
5073 init->set_complete(phase);
5074 return true;
5075 }
5076
5077 void InitializeNode::remove_extra_zeroes() {
5078 if (req() == RawStores) return;
5079 Node* zmem = zero_memory();
5080 uint fill = RawStores;
5081 for (uint i = fill; i < req(); i++) {
5082 Node* n = in(i);
5083 if (n->is_top() || n == zmem) continue; // skip
5084 if (fill < i) set_req(fill, n); // compact
5085 ++fill;
5086 }
5087 // delete any empty spaces created:
5088 while (fill < req()) {
5089 del_req(fill);
5233 // store node that we'd like to capture. We need to check
5234 // the uses of the MergeMemNode.
5235 mems.push(n);
5236 }
5237 } else if (n->is_Mem()) {
5238 Node* other_adr = n->in(MemNode::Address);
5239 if (other_adr == adr) {
5240 failed = true;
5241 break;
5242 } else {
5243 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5244 if (other_t_adr != nullptr) {
5245 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5246 if (other_alias_idx == alias_idx) {
5247 // A load from the same memory slice as the store right
5248 // after the InitializeNode. We check the control of the
5249 // object/array that is loaded from. If it's the same as
5250 // the store control then we cannot capture the store.
5251 assert(!n->is_Store(), "2 stores to same slice on same control?");
5252 Node* base = other_adr;
5253 if (base->is_Phi()) {
5254 // In rare case, base may be a PhiNode and it may read
5255 // the same memory slice between InitializeNode and store.
5256 failed = true;
5257 break;
5258 }
5259 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5260 base = base->in(AddPNode::Base);
5261 if (base != nullptr) {
5262 base = base->uncast();
5263 if (base->is_Proj() && base->in(0) == alloc) {
5264 failed = true;
5265 break;
5266 }
5267 }
5268 }
5269 }
5270 }
5271 } else {
5272 failed = true;
5273 break;
5274 }
5275 }
5276 }
5277 }
5278 if (failed) {
5824 // z's_done 12 16 16 16 12 16 12
5825 // z's_needed 12 16 16 16 16 16 16
5826 // zsize 0 0 0 0 4 0 4
5827 if (next_full_store < 0) {
5828 // Conservative tack: Zero to end of current word.
5829 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5830 } else {
5831 // Zero to beginning of next fully initialized word.
5832 // Or, don't zero at all, if we are already in that word.
5833 assert(next_full_store >= zeroes_needed, "must go forward");
5834 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5835 zeroes_needed = next_full_store;
5836 }
5837 }
5838
5839 if (zeroes_needed > zeroes_done) {
5840 intptr_t zsize = zeroes_needed - zeroes_done;
5841 // Do some incremental zeroing on rawmem, in parallel with inits.
5842 zeroes_done = align_down(zeroes_done, BytesPerInt);
5843 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5844 allocation()->in(AllocateNode::InitValue),
5845 allocation()->in(AllocateNode::RawInitValue),
5846 zeroes_done, zeroes_needed,
5847 true,
5848 phase);
5849 zeroes_done = zeroes_needed;
5850 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5851 do_zeroing = false; // leave the hole, next time
5852 }
5853 }
5854
5855 // Collect the store and move on:
5856 phase->replace_input_of(st, MemNode::Memory, inits);
5857 inits = st; // put it on the linearized chain
5858 set_req(i, zmem); // unhook from previous position
5859
5860 if (zeroes_done == st_off)
5861 zeroes_done = next_init_off;
5862
5863 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5864
5865 #ifdef ASSERT
5886 remove_extra_zeroes(); // clear out all the zmems left over
5887 add_req(inits);
5888
5889 if (!(UseTLAB && ZeroTLAB)) {
5890 // If anything remains to be zeroed, zero it all now.
5891 zeroes_done = align_down(zeroes_done, BytesPerInt);
5892 // if it is the last unused 4 bytes of an instance, forget about it
5893 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5894 if (zeroes_done + BytesPerLong >= size_limit) {
5895 AllocateNode* alloc = allocation();
5896 assert(alloc != nullptr, "must be present");
5897 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5898 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5899 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5900 if (zeroes_done == k->layout_helper())
5901 zeroes_done = size_limit;
5902 }
5903 }
5904 if (zeroes_done < size_limit) {
5905 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5906 allocation()->in(AllocateNode::InitValue),
5907 allocation()->in(AllocateNode::RawInitValue),
5908 zeroes_done, size_in_bytes, true, phase);
5909 }
5910 }
5911
5912 set_complete(phase);
5913 return rawmem;
5914 }
5915
5916 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5917 auto replace_proj = [&](ProjNode* proj) {
5918 C->gvn_replace_by(proj, mem);
5919 return CONTINUE;
5920 };
5921 apply_to_projs(replace_proj, TypeFunc::Memory);
5922 }
5923
5924 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5925 DUIterator_Fast imax, i = fast_outs(imax);
5926 auto replace_proj = [&](ProjNode* proj) {
5927 igvn->replace_node(proj, mem);
6125 //------------------------------Identity---------------------------------------
6126 Node* MergeMemNode::Identity(PhaseGVN* phase) {
6127 // Identity if this merge point does not record any interesting memory
6128 // disambiguations.
6129 Node* base_mem = base_memory();
6130 Node* empty_mem = empty_memory();
6131 if (base_mem != empty_mem) { // Memory path is not dead?
6132 for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
6133 Node* mem = in(i);
6134 if (mem != empty_mem && mem != base_mem) {
6135 return this; // Many memory splits; no change
6136 }
6137 }
6138 }
6139 return base_mem; // No memory splits; ID on the one true input
6140 }
6141
6142 //------------------------------Ideal------------------------------------------
6143 // This method is invoked recursively on chains of MergeMem nodes
6144 Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) {
6145 if (Identity(phase) != this) {
6146 // Let Identity handle this case
6147 return nullptr;
6148 }
6149
6150 // Remove chain'd MergeMems
6151 //
6152 // This is delicate, because the each "in(i)" (i >= Raw) is interpreted
6153 // relative to the "in(Bot)". Since we are patching both at the same time,
6154 // we have to be careful to read each "in(i)" relative to the old "in(Bot)",
6155 // but rewrite each "in(i)" relative to the new "in(Bot)".
6156 Node *progress = nullptr;
6157
6158
6159 Node* old_base = base_memory();
6160 Node* empty_mem = empty_memory();
6161 if (old_base == empty_mem)
6162 return nullptr; // Dead memory path.
6163
6164 MergeMemNode* old_mbase;
6165 if (old_base != nullptr && old_base->is_MergeMem())
6166 old_mbase = old_base->as_MergeMem();
6167 else
6168 old_mbase = nullptr;
6169 Node* new_base = old_base;
|