6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/compile.hpp"
38 #include "opto/connode.hpp"
39 #include "opto/convertnode.hpp"
40 #include "opto/loopnode.hpp"
41 #include "opto/machnode.hpp"
42 #include "opto/matcher.hpp"
43 #include "opto/memnode.hpp"
44 #include "opto/mempointer.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/opcodes.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regalloc.hpp"
50 #include "opto/regmask.hpp"
51 #include "opto/rootnode.hpp"
52 #include "opto/traceMergeStoresTag.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/macros.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #include "utilities/vmError.hpp"
59
60 // Portions of code courtesy of Clifford Click
61
62 // Optimization - Graph Style
63
64 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
65
66 //=============================================================================
67 uint MemNode::size_of() const { return sizeof(*this); }
68
69 const TypePtr *MemNode::adr_type() const {
70 Node* adr = in(Address);
71 if (adr == nullptr) return nullptr; // node is dead
72 const TypePtr* cross_check = nullptr;
73 DEBUG_ONLY(cross_check = _adr_type);
74 return calculate_adr_type(adr->bottom_type(), cross_check);
75 }
124 st->print(", idx=Bot;");
125 else if (atp->index() == Compile::AliasIdxTop)
126 st->print(", idx=Top;");
127 else if (atp->index() == Compile::AliasIdxRaw)
128 st->print(", idx=Raw;");
129 else {
130 ciField* field = atp->field();
131 if (field) {
132 st->print(", name=");
133 field->print_name_on(st);
134 }
135 st->print(", idx=%d;", atp->index());
136 }
137 }
138 }
139
140 extern void print_alias_types();
141
142 #endif
143
144 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
145 assert((t_oop != nullptr), "sanity");
146 bool is_instance = t_oop->is_known_instance_field();
147 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
148 (load != nullptr) && load->is_Load() &&
149 (phase->is_IterGVN() != nullptr);
150 if (!(is_instance || is_boxed_value_load))
151 return mchain; // don't try to optimize non-instance types
152 uint instance_id = t_oop->instance_id();
153 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
154 Node *prev = nullptr;
155 Node *result = mchain;
156 while (prev != result) {
157 prev = result;
158 if (result == start_mem)
159 break; // hit one of our sentinels
160 // skip over a call which does not affect this memory slice
161 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
162 Node *proj_in = result->in(0);
163 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
164 break; // hit one of our sentinels
165 } else if (proj_in->is_Call()) {
166 // ArrayCopyNodes processed here as well
167 CallNode *call = proj_in->as_Call();
168 if (!call->may_modify(t_oop, phase)) { // returns false for instances
169 result = call->in(TypeFunc::Memory);
170 }
171 } else if (proj_in->is_Initialize()) {
172 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
173 // Stop if this is the initialization for the object instance which
174 // contains this memory slice, otherwise skip over it.
175 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
176 break;
177 }
178 if (is_instance) {
179 result = proj_in->in(TypeFunc::Memory);
180 } else if (is_boxed_value_load) {
181 Node* klass = alloc->in(AllocateNode::KlassNode);
182 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
183 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
184 result = proj_in->in(TypeFunc::Memory); // not related allocation
185 }
186 }
187 } else if (proj_in->is_MemBar()) {
188 ArrayCopyNode* ac = nullptr;
189 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
190 break;
191 }
192 result = proj_in->in(TypeFunc::Memory);
193 } else if (proj_in->is_top()) {
194 break; // dead code
195 } else {
196 assert(false, "unexpected projection");
197 }
198 } else if (result->is_ClearArray()) {
199 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
200 // Can not bypass initialization of the instance
201 // we are looking for.
202 break;
203 }
204 // Otherwise skip it (the call updated 'result' value).
205 } else if (result->is_MergeMem()) {
206 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
207 }
208 }
209 return result;
210 }
211
212 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
213 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
214 if (t_oop == nullptr)
215 return mchain; // don't try to optimize non-oop types
216 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
217 bool is_instance = t_oop->is_known_instance_field();
218 PhaseIterGVN *igvn = phase->is_IterGVN();
219 if (is_instance && igvn != nullptr && result->is_Phi()) {
220 PhiNode *mphi = result->as_Phi();
221 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
222 const TypePtr *t = mphi->adr_type();
223 bool do_split = false;
224 // In the following cases, Load memory input can be further optimized based on
225 // its precise address type
226 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
227 do_split = true;
228 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
229 const TypeOopPtr* mem_t =
230 t->is_oopptr()->cast_to_exactness(true)
231 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
232 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
233 if (t_oop->isa_aryptr()) {
234 mem_t = mem_t->is_aryptr()
235 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
236 ->cast_to_size(t_oop->is_aryptr()->size())
237 ->with_offset(t_oop->is_aryptr()->offset())
238 ->is_aryptr();
239 }
240 do_split = mem_t == t_oop;
241 }
242 if (do_split) {
243 // clone the Phi with our address type
244 result = mphi->split_out_instance(t_adr, igvn);
245 } else {
246 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
247 }
248 }
249 return result;
250 }
251
252 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
253 uint alias_idx = phase->C->get_alias_index(tp);
254 Node *mem = mmem;
255 #ifdef ASSERT
256 {
257 // Check that current type is consistent with the alias index used during graph construction
258 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
259 bool consistent = adr_check == nullptr || adr_check->empty() ||
260 phase->C->must_alias(adr_check, alias_idx );
261 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
262 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
263 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
264 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
265 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
266 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
267 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
268 // don't assert if it is dead code.
269 consistent = true;
270 }
271 if( !consistent ) {
272 st->print("alias_idx==%d, adr_check==", alias_idx);
273 if( adr_check == nullptr ) {
274 st->print("null");
275 } else {
276 adr_check->dump();
277 }
278 st->cr();
279 print_alias_types();
280 assert(consistent, "adr_check must match alias idx");
281 }
282 }
283 #endif
1017 "use LoadKlassNode instead");
1018 assert(!(adr_type->isa_aryptr() &&
1019 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1020 "use LoadRangeNode instead");
1021 // Check control edge of raw loads
1022 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1023 // oop will be recorded in oop map if load crosses safepoint
1024 rt->isa_oopptr() || is_immutable_value(adr),
1025 "raw memory operations should have control edge");
1026 LoadNode* load = nullptr;
1027 switch (bt) {
1028 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1029 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1030 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1031 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1032 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1033 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1034 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1035 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1036 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1037 case T_OBJECT:
1038 case T_NARROWOOP:
1039 #ifdef _LP64
1040 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1041 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1042 } else
1043 #endif
1044 {
1045 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1046 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1047 }
1048 break;
1049 default:
1050 ShouldNotReachHere();
1051 break;
1052 }
1053 assert(load != nullptr, "LoadNode should have been created");
1054 if (unaligned) {
1055 load->set_unaligned_access();
1056 }
1057 if (mismatched) {
1058 load->set_mismatched_access();
1059 }
1060 if (unsafe) {
1061 load->set_unsafe_access();
1062 }
1063 load->set_barrier_data(barrier_data);
1064 if (load->Opcode() == Op_LoadN) {
1065 Node* ld = gvn.transform(load);
1066 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1067 }
1068
1069 return load;
1070 }
1071
1072 //------------------------------hash-------------------------------------------
1073 uint LoadNode::hash() const {
1074 // unroll addition of interesting fields
1075 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1076 }
1077
1078 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1079 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1080 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1081 bool is_stable_ary = FoldStableValues &&
1082 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1083 tp->isa_aryptr()->is_stable();
1084
1085 return (eliminate_boxing && non_volatile) || is_stable_ary;
1086 }
1087
1088 return false;
1089 }
1090
1091 // Is the value loaded previously stored by an arraycopy? If so return
1092 // a load node that reads from the source array so we may be able to
1093 // optimize out the ArrayCopy node later.
1094 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1095 Node* ld_adr = in(MemNode::Address);
1096 intptr_t ld_off = 0;
1097 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1098 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1099 if (ac != nullptr) {
1100 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1101
1102 Node* mem = ac->in(TypeFunc::Memory);
1103 Node* ctl = ac->in(0);
1104 Node* src = ac->in(ArrayCopyNode::Src);
1105
1113 if (ac->as_ArrayCopy()->is_clonebasic()) {
1114 assert(ld_alloc != nullptr, "need an alloc");
1115 assert(addp->is_AddP(), "address must be addp");
1116 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1117 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1118 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1119 addp->set_req(AddPNode::Base, src);
1120 addp->set_req(AddPNode::Address, src);
1121 } else {
1122 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1123 ac->as_ArrayCopy()->is_copyof_validated() ||
1124 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1125 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1126 addp->set_req(AddPNode::Base, src);
1127 addp->set_req(AddPNode::Address, src);
1128
1129 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1130 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1131 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1132
1133 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1134 uint shift = exact_log2(type2aelembytes(ary_elem));
1135
1136 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1137 #ifdef _LP64
1138 diff = phase->transform(new ConvI2LNode(diff));
1139 #endif
1140 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1141
1142 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1143 addp->set_req(AddPNode::Offset, offset);
1144 }
1145 addp = phase->transform(addp);
1146 #ifdef ASSERT
1147 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1148 ld->_adr_type = adr_type;
1149 #endif
1150 ld->set_req(MemNode::Address, addp);
1151 ld->set_req(0, ctl);
1152 ld->set_req(MemNode::Memory, mem);
1153 return ld;
1154 }
1155 return nullptr;
1156 }
1157
1158
1159 //---------------------------can_see_stored_value------------------------------
1160 // This routine exists to make sure this set of tests is done the same
1161 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1162 // will change the graph shape in a way which makes memory alive twice at the
1163 // same time (uses the Oracle model of aliasing), then some
1164 // LoadXNode::Identity will fold things back to the equivalence-class model
1165 // of aliasing.
1166 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1167 Node* ld_adr = in(MemNode::Address);
1168 intptr_t ld_off = 0;
1169 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1170 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1171 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1172 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1173 // This is more general than load from boxing objects.
1174 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1175 uint alias_idx = atp->index();
1176 Node* result = nullptr;
1177 Node* current = st;
1178 // Skip through chains of MemBarNodes checking the MergeMems for
1179 // new states for the slice of this load. Stop once any other
1180 // kind of node is encountered. Loads from final memory can skip
1181 // through any kind of MemBar but normal loads shouldn't skip
1182 // through MemBarAcquire since the could allow them to move out of
1183 // a synchronized region. It is not safe to step over MemBarCPUOrder,
1184 // because alias info above them may be inaccurate (e.g., due to
1185 // mixed/mismatched unsafe accesses).
1186 bool is_final_mem = !atp->is_rewritable();
1187 while (current->is_Proj()) {
1188 int opc = current->in(0)->Opcode();
1189 if ((is_final_mem && (opc == Op_MemBarAcquire ||
1256
1257 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1258 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1259 if (in_vt != out_vt) {
1260 return nullptr;
1261 }
1262 }
1263 return st->in(MemNode::ValueIn);
1264 }
1265
1266 // A load from a freshly-created object always returns zero.
1267 // (This can happen after LoadNode::Ideal resets the load's memory input
1268 // to find_captured_store, which returned InitializeNode::zero_memory.)
1269 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1270 (st->in(0) == ld_alloc) &&
1271 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1272 // return a zero value for the load's basic type
1273 // (This is one of the few places where a generic PhaseTransform
1274 // can create new nodes. Think of it as lazily manifesting
1275 // virtually pre-existing constants.)
1276 if (value_basic_type() != T_VOID) {
1277 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1278 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1279 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1280 // by the ArrayCopyNode.
1281 return phase->zerocon(value_basic_type());
1282 }
1283 } else {
1284 // TODO: materialize all-zero vector constant
1285 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1286 }
1287 }
1288
1289 // A load from an initialization barrier can match a captured store.
1290 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1291 InitializeNode* init = st->in(0)->as_Initialize();
1292 AllocateNode* alloc = init->allocation();
1293 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1294 // examine a captured store value
1295 st = init->find_captured_store(ld_off, memory_size(), phase);
1308 base = bs->step_over_gc_barrier(base);
1309 if (base != nullptr && base->is_Proj() &&
1310 base->as_Proj()->_con == TypeFunc::Parms &&
1311 base->in(0)->is_CallStaticJava() &&
1312 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1313 return base->in(0)->in(TypeFunc::Parms);
1314 }
1315 }
1316
1317 break;
1318 }
1319
1320 return nullptr;
1321 }
1322
1323 //----------------------is_instance_field_load_with_local_phi------------------
1324 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1325 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1326 in(Address)->is_AddP() ) {
1327 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1328 // Only instances and boxed values.
1329 if( t_oop != nullptr &&
1330 (t_oop->is_ptr_to_boxed_value() ||
1331 t_oop->is_known_instance_field()) &&
1332 t_oop->offset() != Type::OffsetBot &&
1333 t_oop->offset() != Type::OffsetTop) {
1334 return true;
1335 }
1336 }
1337 return false;
1338 }
1339
1340 //------------------------------Identity---------------------------------------
1341 // Loads are identity if previous store is to same address
1342 Node* LoadNode::Identity(PhaseGVN* phase) {
1343 // If the previous store-maker is the right kind of Store, and the store is
1344 // to the same address, then we are equal to the value stored.
1345 Node* mem = in(Memory);
1346 Node* value = can_see_stored_value(mem, phase);
1347 if( value ) {
1348 // byte, short & char stores truncate naturally.
1349 // A load has to load the truncated value which requires
1350 // some sort of masking operation and that requires an
1351 // Ideal call instead of an Identity call.
1352 if (memory_size() < BytesPerInt) {
1353 // If the input to the store does not fit with the load's result type,
1354 // it must be truncated via an Ideal call.
1355 if (!phase->type(value)->higher_equal(phase->type(this)))
1356 return this;
1357 }
1358 // (This works even when value is a Con, but LoadNode::Value
1359 // usually runs first, producing the singleton type of the Con.)
1360 if (!has_pinned_control_dependency() || value->is_Con()) {
1361 return value;
1362 } else {
1363 return this;
1364 }
1365 }
1366
1367 if (has_pinned_control_dependency()) {
1368 return this;
1369 }
1370 // Search for an existing data phi which was generated before for the same
1371 // instance's field to avoid infinite generation of phis in a loop.
1372 Node *region = mem->in(0);
1373 if (is_instance_field_load_with_local_phi(region)) {
1374 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1375 int this_index = phase->C->get_alias_index(addr_t);
1376 int this_offset = addr_t->offset();
1377 int this_iid = addr_t->instance_id();
1378 if (!addr_t->is_known_instance() &&
1379 addr_t->is_ptr_to_boxed_value()) {
1380 // Use _idx of address base (could be Phi node) for boxed values.
1381 intptr_t ignore = 0;
1382 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1383 if (base == nullptr) {
1384 return this;
1385 }
1386 this_iid = base->_idx;
1387 }
1388 const Type* this_type = bottom_type();
1389 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1390 Node* phi = region->fast_out(i);
1391 if (phi->is_Phi() && phi != mem &&
1392 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1393 return phi;
1394 }
1395 }
1396 }
1397
1398 return this;
1399 }
1400
1915 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1916 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1917
1918 // Skip up past a SafePoint control. Cannot do this for Stores because
1919 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1920 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1921 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1922 !addr_mark &&
1923 (depends_only_on_test() || has_unknown_control_dependency())) {
1924 ctrl = ctrl->in(0);
1925 set_req(MemNode::Control,ctrl);
1926 return this;
1927 }
1928
1929 intptr_t ignore = 0;
1930 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1931 if (base != nullptr
1932 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1933 // Check for useless control edge in some common special cases
1934 if (in(MemNode::Control) != nullptr
1935 && can_remove_control()
1936 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1937 && all_controls_dominate(base, phase->C->start())) {
1938 // A method-invariant, non-null address (constant or 'this' argument).
1939 set_req(MemNode::Control, nullptr);
1940 return this;
1941 }
1942 }
1943
1944 Node* mem = in(MemNode::Memory);
1945 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1946
1947 if (can_reshape && (addr_t != nullptr)) {
1948 // try to optimize our memory input
1949 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1950 if (opt_mem != mem) {
1951 set_req_X(MemNode::Memory, opt_mem, phase);
1952 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1953 return this;
1954 }
2067 // No match.
2068 return nullptr;
2069 }
2070
2071 //------------------------------Value-----------------------------------------
2072 const Type* LoadNode::Value(PhaseGVN* phase) const {
2073 // Either input is TOP ==> the result is TOP
2074 Node* mem = in(MemNode::Memory);
2075 const Type *t1 = phase->type(mem);
2076 if (t1 == Type::TOP) return Type::TOP;
2077 Node* adr = in(MemNode::Address);
2078 const TypePtr* tp = phase->type(adr)->isa_ptr();
2079 if (tp == nullptr || tp->empty()) return Type::TOP;
2080 int off = tp->offset();
2081 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2082 Compile* C = phase->C;
2083
2084 // If load can see a previous constant store, use that.
2085 Node* value = can_see_stored_value(mem, phase);
2086 if (value != nullptr && value->is_Con()) {
2087 assert(value->bottom_type()->higher_equal(_type), "sanity");
2088 return value->bottom_type();
2089 }
2090
2091 // Try to guess loaded type from pointer type
2092 if (tp->isa_aryptr()) {
2093 const TypeAryPtr* ary = tp->is_aryptr();
2094 const Type* t = ary->elem();
2095
2096 // Determine whether the reference is beyond the header or not, by comparing
2097 // the offset against the offset of the start of the array's data.
2098 // Different array types begin at slightly different offsets (12 vs. 16).
2099 // We choose T_BYTE as an example base type that is least restrictive
2100 // as to alignment, which will therefore produce the smallest
2101 // possible base offset.
2102 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2103 const bool off_beyond_header = (off >= min_base_off);
2104
2105 // Try to constant-fold a stable array element.
2106 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2107 // Make sure the reference is not into the header and the offset is constant
2108 ciObject* aobj = ary->const_oop();
2109 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2110 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2111 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off,
2112 stable_dimension,
2113 value_basic_type(), is_unsigned());
2114 if (con_type != nullptr) {
2115 return con_type;
2116 }
2117 }
2118 }
2119
2120 // Don't do this for integer types. There is only potential profit if
2121 // the element type t is lower than _type; that is, for int types, if _type is
2122 // more restrictive than t. This only happens here if one is short and the other
2123 // char (both 16 bits), and in those cases we've made an intentional decision
2124 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2125 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2126 //
2127 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2128 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2129 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2130 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2131 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2132 // In fact, that could have been the original type of p1, and p1 could have
2133 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2134 // expression (LShiftL quux 3) independently optimized to the constant 8.
2135 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2136 && (_type->isa_vect() == nullptr)
2137 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2138 // t might actually be lower than _type, if _type is a unique
2139 // concrete subclass of abstract class t.
2140 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2141 const Type* jt = t->join_speculative(_type);
2142 // In any case, do not allow the join, per se, to empty out the type.
2143 if (jt->empty() && !t->empty()) {
2144 // This can happen if a interface-typed array narrows to a class type.
2145 jt = _type;
2146 }
2147 #ifdef ASSERT
2148 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2149 // The pointers in the autobox arrays are always non-null
2150 Node* base = adr->in(AddPNode::Base);
2151 if ((base != nullptr) && base->is_DecodeN()) {
2152 // Get LoadN node which loads IntegerCache.cache field
2153 base = base->in(1);
2154 }
2155 if ((base != nullptr) && base->is_Con()) {
2156 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2157 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2158 // It could be narrow oop
2159 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2160 }
2161 }
2162 }
2163 #endif
2164 return jt;
2165 }
2166 }
2167 } else if (tp->base() == Type::InstPtr) {
2168 assert( off != Type::OffsetBot ||
2169 // arrays can be cast to Objects
2170 !tp->isa_instptr() ||
2171 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2172 // unsafe field access may not have a constant offset
2173 C->has_unsafe_access(),
2174 "Field accesses must be precise" );
2175 // For oop loads, we expect the _type to be precise.
2176
2177 // Optimize loads from constant fields.
2178 const TypeInstPtr* tinst = tp->is_instptr();
2179 ciObject* const_oop = tinst->const_oop();
2180 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2181 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2182 if (con_type != nullptr) {
2183 return con_type;
2184 }
2185 }
2186 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2187 assert(off != Type::OffsetBot ||
2188 !tp->isa_instklassptr() ||
2189 // arrays can be cast to Objects
2190 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2191 // also allow array-loading from the primary supertype
2192 // array during subtype checks
2193 Opcode() == Op_LoadKlass,
2194 "Field accesses must be precise");
2195 // For klass/static loads, we expect the _type to be precise
2196 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2197 /* With mirrors being an indirect in the Klass*
2198 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2199 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2200 *
2201 * So check the type and klass of the node before the LoadP.
2208 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2209 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2210 return TypeInstPtr::make(klass->java_mirror());
2211 }
2212 }
2213 }
2214
2215 const TypeKlassPtr *tkls = tp->isa_klassptr();
2216 if (tkls != nullptr) {
2217 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2218 ciKlass* klass = tkls->exact_klass();
2219 // We are loading a field from a Klass metaobject whose identity
2220 // is known at compile time (the type is "exact" or "precise").
2221 // Check for fields we know are maintained as constants by the VM.
2222 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2223 // The field is Klass::_super_check_offset. Return its (constant) value.
2224 // (Folds up type checking code.)
2225 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2226 return TypeInt::make(klass->super_check_offset());
2227 }
2228 if (UseCompactObjectHeaders) {
2229 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2230 // The field is Klass::_prototype_header. Return its (constant) value.
2231 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2232 return TypeX::make(klass->prototype_header());
2233 }
2234 }
2235 // Compute index into primary_supers array
2236 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2237 // Check for overflowing; use unsigned compare to handle the negative case.
2238 if( depth < ciKlass::primary_super_limit() ) {
2239 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2240 // (Folds up type checking code.)
2241 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2242 ciKlass *ss = klass->super_of_depth(depth);
2243 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2244 }
2245 const Type* aift = load_array_final_field(tkls, klass);
2246 if (aift != nullptr) return aift;
2247 }
2248
2249 // We can still check if we are loading from the primary_supers array at a
2250 // shallow enough depth. Even though the klass is not exact, entries less
2251 // than or equal to its super depth are correct.
2252 if (tkls->is_loaded()) {
2253 ciKlass* klass = nullptr;
2287 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2288 // The key property of this type is that it folds up tests
2289 // for array-ness, since it proves that the layout_helper is positive.
2290 // Thus, a generic value like the basic object layout helper works fine.
2291 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2292 }
2293 }
2294
2295 // If we are loading from a freshly-allocated object/array, produce a zero.
2296 // Things to check:
2297 // 1. Load is beyond the header: headers are not guaranteed to be zero
2298 // 2. Load is not vectorized: vectors have no zero constant
2299 // 3. Load has no matching store, i.e. the input is the initial memory state
2300 const TypeOopPtr* tinst = tp->isa_oopptr();
2301 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2302 bool is_not_vect = (_type->isa_vect() == nullptr);
2303 if (is_not_header && is_not_vect) {
2304 Node* mem = in(MemNode::Memory);
2305 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2306 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2307 return Type::get_zero_type(_type->basic_type());
2308 }
2309 }
2310
2311 if (!UseCompactObjectHeaders) {
2312 Node* alloc = is_new_object_mark_load();
2313 if (alloc != nullptr) {
2314 return TypeX::make(markWord::prototype().value());
2315 }
2316 }
2317
2318 return _type;
2319 }
2320
2321 //------------------------------match_edge-------------------------------------
2322 // Do we Match on this edge index or not? Match only the address.
2323 uint LoadNode::match_edge(uint idx) const {
2324 return idx == MemNode::Address;
2325 }
2326
2327 //--------------------------LoadBNode::Ideal--------------------------------------
2328 //
2329 // If the previous store is to the same address as this load,
2330 // and the value stored was larger than a byte, replace this load
2331 // with the value stored truncated to a byte. If no truncation is
2332 // needed, the replacement is done in LoadNode::Identity().
2333 //
2334 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2443 }
2444 }
2445 // Identity call will handle the case where truncation is not needed.
2446 return LoadNode::Ideal(phase, can_reshape);
2447 }
2448
2449 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2450 Node* mem = in(MemNode::Memory);
2451 Node* value = can_see_stored_value(mem,phase);
2452 if (value != nullptr && value->is_Con() &&
2453 !value->bottom_type()->higher_equal(_type)) {
2454 // If the input to the store does not fit with the load's result type,
2455 // it must be truncated. We can't delay until Ideal call since
2456 // a singleton Value is needed for split_thru_phi optimization.
2457 int con = value->get_int();
2458 return TypeInt::make((con << 16) >> 16);
2459 }
2460 return LoadNode::Value(phase);
2461 }
2462
2463 //=============================================================================
2464 //----------------------------LoadKlassNode::make------------------------------
2465 // Polymorphic factory method:
2466 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2467 // sanity check the alias category against the created node type
2468 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2469 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2470 #ifdef _LP64
2471 if (adr_type->is_ptr_to_narrowklass()) {
2472 assert(UseCompressedClassPointers, "no compressed klasses");
2473 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2474 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2475 }
2476 #endif
2477 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2478 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2479 }
2480
2481 //------------------------------Value------------------------------------------
2482 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2516 }
2517 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2518 }
2519 if (!t->is_klass()) {
2520 // a primitive Class (e.g., int.class) has null for a klass field
2521 return TypePtr::NULL_PTR;
2522 }
2523 // Fold up the load of the hidden field
2524 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2525 }
2526 // non-constant mirror, so we can't tell what's going on
2527 }
2528 if (!tinst->is_loaded())
2529 return _type; // Bail out if not loaded
2530 if (offset == oopDesc::klass_offset_in_bytes()) {
2531 return tinst->as_klass_type(true);
2532 }
2533 }
2534
2535 // Check for loading klass from an array
2536 const TypeAryPtr *tary = tp->isa_aryptr();
2537 if (tary != nullptr &&
2538 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2539 return tary->as_klass_type(true);
2540 }
2541
2542 // Check for loading klass from an array klass
2543 const TypeKlassPtr *tkls = tp->isa_klassptr();
2544 if (tkls != nullptr && !StressReflectiveCode) {
2545 if (!tkls->is_loaded())
2546 return _type; // Bail out if not loaded
2547 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2548 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2549 // // Always returning precise element type is incorrect,
2550 // // e.g., element type could be object and array may contain strings
2551 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2552
2553 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2554 // according to the element type's subclassing.
2555 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2556 }
2557 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2558 tkls->offset() == in_bytes(Klass::super_offset())) {
2559 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2560 // The field is Klass::_super. Return its (constant) value.
2561 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2562 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2563 }
2564 }
2565
2566 if (tkls != nullptr && !UseSecondarySupersCache
2567 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2568 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2569 return TypePtr::NULL_PTR;
2570 }
2571
2572 // Bailout case
2573 return LoadNode::Value(phase);
2574 }
2575
2576 //------------------------------Identity---------------------------------------
2599 base = bs->step_over_gc_barrier(base);
2600 }
2601
2602 // We can fetch the klass directly through an AllocateNode.
2603 // This works even if the klass is not constant (clone or newArray).
2604 if (offset == oopDesc::klass_offset_in_bytes()) {
2605 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2606 if (allocated_klass != nullptr) {
2607 return allocated_klass;
2608 }
2609 }
2610
2611 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2612 // See inline_native_Class_query for occurrences of these patterns.
2613 // Java Example: x.getClass().isAssignableFrom(y)
2614 //
2615 // This improves reflective code, often making the Class
2616 // mirror go completely dead. (Current exception: Class
2617 // mirrors may appear in debug info, but we could clean them out by
2618 // introducing a new debug info operator for Klass.java_mirror).
2619
2620 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2621 && offset == java_lang_Class::klass_offset()) {
2622 if (base->is_Load()) {
2623 Node* base2 = base->in(MemNode::Address);
2624 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2625 Node* adr2 = base2->in(MemNode::Address);
2626 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2627 if (tkls != nullptr && !tkls->empty()
2628 && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2629 && adr2->is_AddP()
2630 ) {
2631 int mirror_field = in_bytes(Klass::java_mirror_offset());
2632 if (tkls->offset() == mirror_field) {
2633 #ifdef ASSERT
2634 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2635 assert(tkls2->offset() == 0, "not a load of java_mirror");
2636 #endif
2637 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2638 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2639 return adr2->in(AddPNode::Address);
2640 }
2641 }
2642 }
2643 }
2644 }
2645
2646 return this;
2647 }
2648
2649 LoadNode* LoadNode::clone_pinned() const {
2650 LoadNode* ld = clone()->as_Load();
2776 //---------------------------StoreNode::make-----------------------------------
2777 // Polymorphic factory method:
2778 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2779 assert((mo == unordered || mo == release), "unexpected");
2780 Compile* C = gvn.C;
2781 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2782 ctl != nullptr, "raw memory operations should have control edge");
2783
2784 switch (bt) {
2785 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2786 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2787 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2788 case T_CHAR:
2789 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2790 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2791 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2792 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2793 case T_METADATA:
2794 case T_ADDRESS:
2795 case T_OBJECT:
2796 #ifdef _LP64
2797 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2798 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2799 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2800 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2801 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2802 adr->bottom_type()->isa_rawptr())) {
2803 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2804 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2805 }
2806 #endif
2807 {
2808 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2809 }
2810 default:
2811 ShouldNotReachHere();
2812 return (StoreNode*)nullptr;
2813 }
2814 }
2815
2816 //--------------------------bottom_type----------------------------------------
2817 const Type *StoreNode::bottom_type() const {
2818 return Type::MEMORY;
2819 }
2820
2821 //------------------------------hash-------------------------------------------
2822 uint StoreNode::hash() const {
2823 // unroll addition of interesting fields
2824 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2825
2826 // Since they are not commoned, do not hash them:
2827 return NO_HASH;
2828 }
2829
2830 // Link together multiple stores (B/S/C/I) into a longer one.
2831 //
3453 }
3454 ss.print_cr("[TraceMergeStores]: with");
3455 merged_input_value->dump("\n", false, &ss);
3456 merged_store->dump("\n", false, &ss);
3457 tty->print("%s", ss.as_string());
3458 }
3459 #endif
3460
3461 //------------------------------Ideal------------------------------------------
3462 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3463 // When a store immediately follows a relevant allocation/initialization,
3464 // try to capture it into the initialization, or hoist it above.
3465 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3466 Node* p = MemNode::Ideal_common(phase, can_reshape);
3467 if (p) return (p == NodeSentinel) ? nullptr : p;
3468
3469 Node* mem = in(MemNode::Memory);
3470 Node* address = in(MemNode::Address);
3471 Node* value = in(MemNode::ValueIn);
3472 // Back-to-back stores to same address? Fold em up. Generally
3473 // unsafe if I have intervening uses.
3474 {
3475 Node* st = mem;
3476 // If Store 'st' has more than one use, we cannot fold 'st' away.
3477 // For example, 'st' might be the final state at a conditional
3478 // return. Or, 'st' might be used by some node which is live at
3479 // the same time 'st' is live, which might be unschedulable. So,
3480 // require exactly ONE user until such time as we clone 'mem' for
3481 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3482 // true).
3483 while (st->is_Store() && st->outcnt() == 1) {
3484 // Looking at a dead closed cycle of memory?
3485 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3486 assert(Opcode() == st->Opcode() ||
3487 st->Opcode() == Op_StoreVector ||
3488 Opcode() == Op_StoreVector ||
3489 st->Opcode() == Op_StoreVectorScatter ||
3490 Opcode() == Op_StoreVectorScatter ||
3491 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3492 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3493 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3494 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3495 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3496
3497 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3498 st->as_Store()->memory_size() <= this->memory_size()) {
3499 Node* use = st->raw_out(0);
3500 if (phase->is_IterGVN()) {
3501 phase->is_IterGVN()->rehash_node_delayed(use);
3502 }
3503 // It's OK to do this in the parser, since DU info is always accurate,
3504 // and the parser always refers to nodes via SafePointNode maps.
3505 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3506 return this;
3507 }
3508 st = st->in(MemNode::Memory);
3509 }
3510 }
3511
3512
3513 // Capture an unaliased, unconditional, simple store into an initializer.
3614 const StoreVectorNode* store_vector = as_StoreVector();
3615 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3616 const Node* store_indices = store_vector->indices();
3617 const Node* mem_indices = mem_vector->indices();
3618 const Node* store_mask = store_vector->mask();
3619 const Node* mem_mask = mem_vector->mask();
3620 // Ensure types, indices, and masks match
3621 if (store_vector->vect_type() == mem_vector->vect_type() &&
3622 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3623 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3624 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3625 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3626 result = mem;
3627 }
3628 }
3629 }
3630
3631 // Store of zero anywhere into a freshly-allocated object?
3632 // Then the store is useless.
3633 // (It must already have been captured by the InitializeNode.)
3634 if (result == this &&
3635 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3636 // a newly allocated object is already all-zeroes everywhere
3637 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3638 result = mem;
3639 }
3640
3641 if (result == this) {
3642 // the store may also apply to zero-bits in an earlier object
3643 Node* prev_mem = find_previous_store(phase);
3644 // Steps (a), (b): Walk past independent stores to find an exact match.
3645 if (prev_mem != nullptr) {
3646 if (prev_mem->is_top()) {
3647 // find_previous_store returns top when the access is dead
3648 return prev_mem;
3649 }
3650 Node* prev_val = can_see_stored_value(prev_mem, phase);
3651 if (prev_val != nullptr && prev_val == val) {
3652 // prev_val and val might differ by a cast; it would be good
3653 // to keep the more informative of the two.
3654 result = mem;
3655 }
3656 }
3657 }
3658 }
3659
3660 PhaseIterGVN* igvn = phase->is_IterGVN();
3661 if (result != this && igvn != nullptr) {
4134 // Clearing a short array is faster with stores
4135 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4136 // Already know this is a large node, do not try to ideal it
4137 if (_is_large) return nullptr;
4138
4139 const int unit = BytesPerLong;
4140 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4141 if (!t) return nullptr;
4142 if (!t->is_con()) return nullptr;
4143 intptr_t raw_count = t->get_con();
4144 intptr_t size = raw_count;
4145 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4146 // Clearing nothing uses the Identity call.
4147 // Negative clears are possible on dead ClearArrays
4148 // (see jck test stmt114.stmt11402.val).
4149 if (size <= 0 || size % unit != 0) return nullptr;
4150 intptr_t count = size / unit;
4151 // Length too long; communicate this to matchers and assemblers.
4152 // Assemblers are responsible to produce fast hardware clears for it.
4153 if (size > InitArrayShortSize) {
4154 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4155 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4156 return nullptr;
4157 }
4158 if (!IdealizeClearArrayNode) return nullptr;
4159 Node *mem = in(1);
4160 if( phase->type(mem)==Type::TOP ) return nullptr;
4161 Node *adr = in(3);
4162 const Type* at = phase->type(adr);
4163 if( at==Type::TOP ) return nullptr;
4164 const TypePtr* atp = at->isa_ptr();
4165 // adjust atp to be the correct array element address type
4166 if (atp == nullptr) atp = TypePtr::BOTTOM;
4167 else atp = atp->add_offset(Type::OffsetBot);
4168 // Get base for derived pointer purposes
4169 if( adr->Opcode() != Op_AddP ) Unimplemented();
4170 Node *base = adr->in(1);
4171
4172 Node *zero = phase->makecon(TypeLong::ZERO);
4173 Node *off = phase->MakeConX(BytesPerLong);
4174 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4175 count--;
4176 while( count-- ) {
4177 mem = phase->transform(mem);
4178 adr = phase->transform(new AddPNode(base,adr,off));
4179 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4180 }
4181 return mem;
4182 }
4183
4184 //----------------------------step_through----------------------------------
4185 // Return allocation input memory edge if it is different instance
4186 // or itself if it is the one we are looking for.
4187 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4188 Node* n = *np;
4189 assert(n->is_ClearArray(), "sanity");
4190 intptr_t offset;
4191 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4192 // This method is called only before Allocate nodes are expanded
4193 // during macro nodes expansion. Before that ClearArray nodes are
4194 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4195 // Allocate nodes are expanded) which follows allocations.
4196 assert(alloc != nullptr, "should have allocation");
4197 if (alloc->_idx == instance_id) {
4198 // Can not bypass initialization of the instance we are looking for.
4199 return false;
4202 InitializeNode* init = alloc->initialization();
4203 if (init != nullptr)
4204 *np = init->in(TypeFunc::Memory);
4205 else
4206 *np = alloc->in(TypeFunc::Memory);
4207 return true;
4208 }
4209
4210 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4211 Node* base = dest;
4212 if (raw_base) {
4213 // May be called as part of the initialization of a just allocated object
4214 base = phase->C->top();
4215 }
4216 return phase->transform(new AddPNode(base, dest, offset));
4217 }
4218
4219 //----------------------------clear_memory-------------------------------------
4220 // Generate code to initialize object storage to zero.
4221 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4222 intptr_t start_offset,
4223 Node* end_offset,
4224 bool raw_base,
4225 PhaseGVN* phase) {
4226 intptr_t offset = start_offset;
4227
4228 int unit = BytesPerLong;
4229 if ((offset % unit) != 0) {
4230 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4231 const TypePtr* atp = TypeRawPtr::BOTTOM;
4232 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4233 mem = phase->transform(mem);
4234 offset += BytesPerInt;
4235 }
4236 assert((offset % unit) == 0, "");
4237
4238 // Initialize the remaining stuff, if any, with a ClearArray.
4239 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, raw_base, phase);
4240 }
4241
4242 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4243 Node* start_offset,
4244 Node* end_offset,
4245 bool raw_base,
4246 PhaseGVN* phase) {
4247 if (start_offset == end_offset) {
4248 // nothing to do
4249 return mem;
4250 }
4251
4252 int unit = BytesPerLong;
4253 Node* zbase = start_offset;
4254 Node* zend = end_offset;
4255
4256 // Scale to the unit required by the CPU:
4257 if (!Matcher::init_array_count_is_in_bytes) {
4258 Node* shift = phase->intcon(exact_log2(unit));
4259 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4260 zend = phase->transform(new URShiftXNode(zend, shift) );
4261 }
4262
4263 // Bulk clear double-words
4264 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4265 Node* adr = make_address(dest, start_offset, raw_base, phase);
4266 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4267 return phase->transform(mem);
4268 }
4269
4270 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4271 intptr_t start_offset,
4272 intptr_t end_offset,
4273 bool raw_base,
4274 PhaseGVN* phase) {
4275 if (start_offset == end_offset) {
4276 // nothing to do
4277 return mem;
4278 }
4279
4280 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4281 intptr_t done_offset = end_offset;
4282 if ((done_offset % BytesPerLong) != 0) {
4283 done_offset -= BytesPerInt;
4284 }
4285 if (done_offset > start_offset) {
4286 mem = clear_memory(ctl, mem, dest,
4287 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4288 }
4289 if (done_offset < end_offset) { // emit the final 32-bit store
4290 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4291 const TypePtr* atp = TypeRawPtr::BOTTOM;
4292 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4293 mem = phase->transform(mem);
4294 done_offset += BytesPerInt;
4295 }
4296 assert(done_offset == end_offset, "");
4297 return mem;
4298 }
4299
4300 //=============================================================================
4301 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4302 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4303 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4304 #ifdef ASSERT
4305 , _pair_idx(0)
4306 #endif
4307 {
4308 init_class_id(Class_MemBar);
4309 Node* top = C->top();
4310 init_req(TypeFunc::I_O,top);
4311 init_req(TypeFunc::FramePtr,top);
4312 init_req(TypeFunc::ReturnAdr,top);
4419 PhaseIterGVN* igvn = phase->is_IterGVN();
4420 remove(igvn);
4421 // Must return either the original node (now dead) or a new node
4422 // (Do not return a top here, since that would break the uniqueness of top.)
4423 return new ConINode(TypeInt::ZERO);
4424 }
4425 }
4426 return progress ? this : nullptr;
4427 }
4428
4429 //------------------------------Value------------------------------------------
4430 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4431 if( !in(0) ) return Type::TOP;
4432 if( phase->type(in(0)) == Type::TOP )
4433 return Type::TOP;
4434 return TypeTuple::MEMBAR;
4435 }
4436
4437 //------------------------------match------------------------------------------
4438 // Construct projections for memory.
4439 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4440 switch (proj->_con) {
4441 case TypeFunc::Control:
4442 case TypeFunc::Memory:
4443 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4444 }
4445 ShouldNotReachHere();
4446 return nullptr;
4447 }
4448
4449 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4450 trailing->_kind = TrailingStore;
4451 leading->_kind = LeadingStore;
4452 #ifdef ASSERT
4453 trailing->_pair_idx = leading->_idx;
4454 leading->_pair_idx = leading->_idx;
4455 #endif
4456 }
4457
4458 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4459 trailing->_kind = TrailingLoadStore;
4706 return (req() > RawStores);
4707 }
4708
4709 void InitializeNode::set_complete(PhaseGVN* phase) {
4710 assert(!is_complete(), "caller responsibility");
4711 _is_complete = Complete;
4712
4713 // After this node is complete, it contains a bunch of
4714 // raw-memory initializations. There is no need for
4715 // it to have anything to do with non-raw memory effects.
4716 // Therefore, tell all non-raw users to re-optimize themselves,
4717 // after skipping the memory effects of this initialization.
4718 PhaseIterGVN* igvn = phase->is_IterGVN();
4719 if (igvn) igvn->add_users_to_worklist(this);
4720 }
4721
4722 // convenience function
4723 // return false if the init contains any stores already
4724 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4725 InitializeNode* init = initialization();
4726 if (init == nullptr || init->is_complete()) return false;
4727 init->remove_extra_zeroes();
4728 // for now, if this allocation has already collected any inits, bail:
4729 if (init->is_non_zero()) return false;
4730 init->set_complete(phase);
4731 return true;
4732 }
4733
4734 void InitializeNode::remove_extra_zeroes() {
4735 if (req() == RawStores) return;
4736 Node* zmem = zero_memory();
4737 uint fill = RawStores;
4738 for (uint i = fill; i < req(); i++) {
4739 Node* n = in(i);
4740 if (n->is_top() || n == zmem) continue; // skip
4741 if (fill < i) set_req(fill, n); // compact
4742 ++fill;
4743 }
4744 // delete any empty spaces created:
4745 while (fill < req()) {
4746 del_req(fill);
4890 // store node that we'd like to capture. We need to check
4891 // the uses of the MergeMemNode.
4892 mems.push(n);
4893 }
4894 } else if (n->is_Mem()) {
4895 Node* other_adr = n->in(MemNode::Address);
4896 if (other_adr == adr) {
4897 failed = true;
4898 break;
4899 } else {
4900 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4901 if (other_t_adr != nullptr) {
4902 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4903 if (other_alias_idx == alias_idx) {
4904 // A load from the same memory slice as the store right
4905 // after the InitializeNode. We check the control of the
4906 // object/array that is loaded from. If it's the same as
4907 // the store control then we cannot capture the store.
4908 assert(!n->is_Store(), "2 stores to same slice on same control?");
4909 Node* base = other_adr;
4910 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4911 base = base->in(AddPNode::Base);
4912 if (base != nullptr) {
4913 base = base->uncast();
4914 if (base->is_Proj() && base->in(0) == alloc) {
4915 failed = true;
4916 break;
4917 }
4918 }
4919 }
4920 }
4921 }
4922 } else {
4923 failed = true;
4924 break;
4925 }
4926 }
4927 }
4928 }
4929 if (failed) {
5476 // z's_done 12 16 16 16 12 16 12
5477 // z's_needed 12 16 16 16 16 16 16
5478 // zsize 0 0 0 0 4 0 4
5479 if (next_full_store < 0) {
5480 // Conservative tack: Zero to end of current word.
5481 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5482 } else {
5483 // Zero to beginning of next fully initialized word.
5484 // Or, don't zero at all, if we are already in that word.
5485 assert(next_full_store >= zeroes_needed, "must go forward");
5486 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5487 zeroes_needed = next_full_store;
5488 }
5489 }
5490
5491 if (zeroes_needed > zeroes_done) {
5492 intptr_t zsize = zeroes_needed - zeroes_done;
5493 // Do some incremental zeroing on rawmem, in parallel with inits.
5494 zeroes_done = align_down(zeroes_done, BytesPerInt);
5495 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5496 zeroes_done, zeroes_needed,
5497 true,
5498 phase);
5499 zeroes_done = zeroes_needed;
5500 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5501 do_zeroing = false; // leave the hole, next time
5502 }
5503 }
5504
5505 // Collect the store and move on:
5506 phase->replace_input_of(st, MemNode::Memory, inits);
5507 inits = st; // put it on the linearized chain
5508 set_req(i, zmem); // unhook from previous position
5509
5510 if (zeroes_done == st_off)
5511 zeroes_done = next_init_off;
5512
5513 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5514
5515 #ifdef ASSERT
5536 remove_extra_zeroes(); // clear out all the zmems left over
5537 add_req(inits);
5538
5539 if (!(UseTLAB && ZeroTLAB)) {
5540 // If anything remains to be zeroed, zero it all now.
5541 zeroes_done = align_down(zeroes_done, BytesPerInt);
5542 // if it is the last unused 4 bytes of an instance, forget about it
5543 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5544 if (zeroes_done + BytesPerLong >= size_limit) {
5545 AllocateNode* alloc = allocation();
5546 assert(alloc != nullptr, "must be present");
5547 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5548 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5549 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5550 if (zeroes_done == k->layout_helper())
5551 zeroes_done = size_limit;
5552 }
5553 }
5554 if (zeroes_done < size_limit) {
5555 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5556 zeroes_done, size_in_bytes, true, phase);
5557 }
5558 }
5559
5560 set_complete(phase);
5561 return rawmem;
5562 }
5563
5564 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5565 auto replace_proj = [&](ProjNode* proj) {
5566 C->gvn_replace_by(proj, mem);
5567 return CONTINUE;
5568 };
5569 apply_to_projs(replace_proj, TypeFunc::Memory);
5570 }
5571
5572 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5573 DUIterator_Fast imax, i = fast_outs(imax);
5574 auto replace_proj = [&](ProjNode* proj) {
5575 igvn->replace_node(proj, mem);
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "ci/ciInlineKlass.hpp"
28 #include "classfile/javaClasses.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "classfile/vmIntrinsics.hpp"
31 #include "compiler/compileLog.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/c2/barrierSetC2.hpp"
34 #include "gc/shared/tlab_globals.hpp"
35 #include "memory/allocation.inline.hpp"
36 #include "memory/resourceArea.hpp"
37 #include "oops/flatArrayKlass.hpp"
38 #include "oops/objArrayKlass.hpp"
39 #include "opto/addnode.hpp"
40 #include "opto/arraycopynode.hpp"
41 #include "opto/callnode.hpp"
42 #include "opto/cfgnode.hpp"
43 #include "opto/compile.hpp"
44 #include "opto/connode.hpp"
45 #include "opto/convertnode.hpp"
46 #include "opto/inlinetypenode.hpp"
47 #include "opto/loopnode.hpp"
48 #include "opto/machnode.hpp"
49 #include "opto/matcher.hpp"
50 #include "opto/memnode.hpp"
51 #include "opto/mempointer.hpp"
52 #include "opto/mulnode.hpp"
53 #include "opto/narrowptrnode.hpp"
54 #include "opto/opcodes.hpp"
55 #include "opto/phaseX.hpp"
56 #include "opto/regalloc.hpp"
57 #include "opto/regmask.hpp"
58 #include "opto/rootnode.hpp"
59 #include "opto/traceMergeStoresTag.hpp"
60 #include "opto/vectornode.hpp"
61 #include "runtime/arguments.hpp"
62 #include "utilities/align.hpp"
63 #include "utilities/copy.hpp"
64 #include "utilities/globalDefinitions.hpp"
65 #include "utilities/macros.hpp"
66 #include "utilities/powerOfTwo.hpp"
67 #include "utilities/vmError.hpp"
68
69 // Portions of code courtesy of Clifford Click
70
71 // Optimization - Graph Style
72
73 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
74
75 //=============================================================================
76 uint MemNode::size_of() const { return sizeof(*this); }
77
78 const TypePtr *MemNode::adr_type() const {
79 Node* adr = in(Address);
80 if (adr == nullptr) return nullptr; // node is dead
81 const TypePtr* cross_check = nullptr;
82 DEBUG_ONLY(cross_check = _adr_type);
83 return calculate_adr_type(adr->bottom_type(), cross_check);
84 }
133 st->print(", idx=Bot;");
134 else if (atp->index() == Compile::AliasIdxTop)
135 st->print(", idx=Top;");
136 else if (atp->index() == Compile::AliasIdxRaw)
137 st->print(", idx=Raw;");
138 else {
139 ciField* field = atp->field();
140 if (field) {
141 st->print(", name=");
142 field->print_name_on(st);
143 }
144 st->print(", idx=%d;", atp->index());
145 }
146 }
147 }
148
149 extern void print_alias_types();
150
151 #endif
152
153 // Find the memory output corresponding to the fall-through path of a call
154 static Node* find_call_fallthrough_mem_output(CallNode* call) {
155 ResourceMark rm;
156 CallProjections* projs = call->extract_projections(false, false);
157 Node* res = projs->fallthrough_memproj;
158 assert(res != nullptr, "must have a fallthrough mem output");
159 return res;
160 }
161
162 // Try to find a better memory input for a load from a strict final field
163 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
164 intptr_t offset = 0;
165 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
166 if (base == nullptr) {
167 return nullptr;
168 }
169
170 Node* base_uncasted = base->uncast();
171 if (base_uncasted->is_Proj()) {
172 Node* multi = base_uncasted->in(0);
173 if (multi->is_top()) {
174 // The pointer dies, make the memory die, too
175 return multi;
176 } else if (multi->is_Allocate()) {
177 base_local = base_uncasted->as_Proj();
178 return nullptr;
179 } else if (multi->is_Call()) {
180 if (!multi->is_CallJava() || multi->as_CallJava()->method() == nullptr || !multi->as_CallJava()->method()->return_value_is_larval()) {
181 // The oop is returned from a call, the memory can be the fallthrough output of the call
182 return find_call_fallthrough_mem_output(multi->as_Call());
183 }
184 } else if (multi->is_Start()) {
185 // The oop is a parameter
186 if (base_uncasted->as_Proj()->_con == TypeFunc::Parms && phase->C->method()->receiver_maybe_larval()) {
187 // The receiver of a constructor is similar to the result of an AllocateNode
188 base_local = base_uncasted->as_Proj();
189 return nullptr;
190 } else {
191 // Use the start memory otherwise
192 return multi->as_Start()->proj_out(TypeFunc::Memory);
193 }
194 }
195 }
196
197 return nullptr;
198 }
199
200 // Whether a call can modify a strict final field, given that the object is allocated inside the
201 // current compilation unit, or is the first parameter when the compilation root is a constructor.
202 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
203 // the target method is a subclass of the class declaring 'field'.
204 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
205 if (!call->is_CallJava()) {
206 return false;
207 }
208
209 ciMethod* target = call->as_CallJava()->method();
210 if (target == nullptr) {
211 return false;
212 } else if (target->intrinsic_id() == vmIntrinsicID::_linkToSpecial) {
213 // linkToSpecial can be used to call a constructor, used in the construction of objects in the
214 // reflection API
215 return true;
216 } else if (!target->is_object_constructor()) {
217 return false;
218 }
219
220 // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
221 // then the field is set inside the constructor, else the field must be set before the
222 // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
223 // while a field Sub.y must be set before Super::<init> is invoked.
224 // We can try to be more heroic and decide if the receiver of the constructor invocation is the
225 // object from which we are loading from. This, however, may be problematic as deciding if 2
226 // nodes are definitely different may not be trivial, especially if the graph is not canonical.
227 // As a result, it is made more conservative for now.
228 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
229 return target->holder()->is_subclass_of(field->holder());
230 }
231
232 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
233 assert(t_oop != nullptr, "sanity");
234 bool is_known_instance = t_oop->is_known_instance_field();
235 bool is_strict_final_load = false;
236
237 // After macro expansion, an allocation may become a call, changing the memory input to the
238 // memory output of that call would be illegal. As a result, disallow this transformation after
239 // macro expansion.
240 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
241 is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
242 #ifdef ASSERT
243 if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
244 assert(is_strict_final_load, "sanity check for basic cases");
245 }
246 #endif // ASSERT
247 }
248
249 if (!is_known_instance && !is_strict_final_load) {
250 return mchain;
251 }
252
253 Node* result = mchain;
254 ProjNode* base_local = nullptr;
255
256 ciField* field = nullptr;
257 if (is_strict_final_load) {
258 field = phase->C->alias_type(t_oop)->field();
259 assert(field != nullptr, "must point to a field");
260
261 Node* adr = load->in(MemNode::Address);
262 assert(phase->type(adr) == t_oop, "inconsistent type");
263 Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
264 if (tmp != nullptr) {
265 result = tmp;
266 }
267 }
268
269 uint instance_id = t_oop->instance_id();
270 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
271 Node* prev = nullptr;
272 while (prev != result) {
273 prev = result;
274 if (result == start_mem) {
275 // start_mem is the earliest memory possible
276 break;
277 }
278
279 // skip over a call which does not affect this memory slice
280 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
281 Node* proj_in = result->in(0);
282 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
283 // This is the allocation that creates the object from which we are loading from
284 break;
285 } else if (proj_in->is_Call()) {
286 // ArrayCopyNodes processed here as well
287 CallNode* call = proj_in->as_Call();
288 if (!call->may_modify(t_oop, phase)) {
289 result = call->in(TypeFunc::Memory);
290 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
291 result = call->in(TypeFunc::Memory);
292 }
293 } else if (proj_in->Opcode() == Op_Tuple) {
294 // The call will be folded, skip over it.
295 break;
296 } else if (proj_in->is_Initialize()) {
297 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
298 // Stop if this is the initialization for the object instance which
299 // contains this memory slice, otherwise skip over it.
300 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
301 break;
302 }
303 if (is_known_instance) {
304 result = proj_in->in(TypeFunc::Memory);
305 } else if (is_strict_final_load) {
306 Node* klass = alloc->in(AllocateNode::KlassNode);
307 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
308 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
309 // Allocation of another type, must be another object
310 result = proj_in->in(TypeFunc::Memory);
311 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
312 // Allocation of another object
313 result = proj_in->in(TypeFunc::Memory);
314 }
315 }
316 } else if (proj_in->is_MemBar()) {
317 ArrayCopyNode* ac = nullptr;
318 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
319 break;
320 }
321 result = proj_in->in(TypeFunc::Memory);
322 } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
323 if (is_strict_final_load) {
324 // LoadFlat and StoreFlat cannot happen to strict final fields
325 result = proj_in->in(TypeFunc::Memory);
326 }
327 } else if (proj_in->is_top()) {
328 break; // dead code
329 } else {
330 assert(false, "unexpected projection of %s", proj_in->Name());
331 }
332 } else if (result->is_ClearArray()) {
333 if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
334 // Can not bypass initialization of the instance
335 // we are looking for.
336 break;
337 }
338 // Otherwise skip it (the call updated 'result' value).
339 } else if (result->is_MergeMem()) {
340 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
341 }
342 }
343 return result;
344 }
345
346 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
347 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
348 if (t_oop == nullptr)
349 return mchain; // don't try to optimize non-oop types
350 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
351 bool is_instance = t_oop->is_known_instance_field();
352 PhaseIterGVN *igvn = phase->is_IterGVN();
353 if (is_instance && igvn != nullptr && result->is_Phi()) {
354 PhiNode *mphi = result->as_Phi();
355 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
356 const TypePtr *t = mphi->adr_type();
357 bool do_split = false;
358 // In the following cases, Load memory input can be further optimized based on
359 // its precise address type
360 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
361 do_split = true;
362 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
363 const TypeOopPtr* mem_t =
364 t->is_oopptr()->cast_to_exactness(true)
365 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
366 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
367 if (t_oop->isa_aryptr()) {
368 mem_t = mem_t->is_aryptr()
369 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
370 ->cast_to_size(t_oop->is_aryptr()->size())
371 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
372 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
373 ->with_offset(t_oop->is_aryptr()->offset())
374 ->is_aryptr();
375 }
376 do_split = mem_t == t_oop;
377 }
378 if (do_split) {
379 // clone the Phi with our address type
380 result = mphi->split_out_instance(t_adr, igvn);
381 } else {
382 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
383 }
384 }
385 return result;
386 }
387
388 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
389 uint alias_idx = phase->C->get_alias_index(tp);
390 Node *mem = mmem;
391 #ifdef ASSERT
392 {
393 // Check that current type is consistent with the alias index used during graph construction
394 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
395 bool consistent = adr_check == nullptr || adr_check->empty() ||
396 phase->C->must_alias(adr_check, alias_idx );
397 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
398 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
399 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
400 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
401 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
402 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
403 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
404 // don't assert if it is dead code.
405 consistent = true;
406 }
407 if( !consistent ) {
408 st->print("alias_idx==%d, adr_check==", alias_idx);
409 if( adr_check == nullptr ) {
410 st->print("null");
411 } else {
412 adr_check->dump();
413 }
414 st->cr();
415 print_alias_types();
416 assert(consistent, "adr_check must match alias idx");
417 }
418 }
419 #endif
1153 "use LoadKlassNode instead");
1154 assert(!(adr_type->isa_aryptr() &&
1155 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1156 "use LoadRangeNode instead");
1157 // Check control edge of raw loads
1158 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1159 // oop will be recorded in oop map if load crosses safepoint
1160 rt->isa_oopptr() || is_immutable_value(adr),
1161 "raw memory operations should have control edge");
1162 LoadNode* load = nullptr;
1163 switch (bt) {
1164 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1165 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1166 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1167 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1168 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1169 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1170 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1171 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1172 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1173 case T_ARRAY:
1174 case T_OBJECT:
1175 case T_NARROWOOP:
1176 #ifdef _LP64
1177 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1178 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1179 } else
1180 #endif
1181 {
1182 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1183 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1184 }
1185 break;
1186 default:
1187 guarantee(false, "unexpected basic type %s", type2name(bt));
1188 break;
1189 }
1190 assert(load != nullptr, "LoadNode should have been created");
1191 if (unaligned) {
1192 load->set_unaligned_access();
1193 }
1194 if (mismatched) {
1195 load->set_mismatched_access();
1196 }
1197 if (unsafe) {
1198 load->set_unsafe_access();
1199 }
1200 load->set_barrier_data(barrier_data);
1201 if (load->Opcode() == Op_LoadN) {
1202 Node* ld = gvn.transform(load);
1203 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1204 }
1205
1206 return load;
1207 }
1208
1209 //------------------------------hash-------------------------------------------
1210 uint LoadNode::hash() const {
1211 // unroll addition of interesting fields
1212 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1213 }
1214
1215 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1216 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1217 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1218 bool is_stable_ary = FoldStableValues &&
1219 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1220 tp->isa_aryptr()->is_stable();
1221
1222 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1223 }
1224
1225 return false;
1226 }
1227
1228 // Is the value loaded previously stored by an arraycopy? If so return
1229 // a load node that reads from the source array so we may be able to
1230 // optimize out the ArrayCopy node later.
1231 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1232 Node* ld_adr = in(MemNode::Address);
1233 intptr_t ld_off = 0;
1234 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1235 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1236 if (ac != nullptr) {
1237 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1238
1239 Node* mem = ac->in(TypeFunc::Memory);
1240 Node* ctl = ac->in(0);
1241 Node* src = ac->in(ArrayCopyNode::Src);
1242
1250 if (ac->as_ArrayCopy()->is_clonebasic()) {
1251 assert(ld_alloc != nullptr, "need an alloc");
1252 assert(addp->is_AddP(), "address must be addp");
1253 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1254 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1255 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1256 addp->set_req(AddPNode::Base, src);
1257 addp->set_req(AddPNode::Address, src);
1258 } else {
1259 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1260 ac->as_ArrayCopy()->is_copyof_validated() ||
1261 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1262 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1263 addp->set_req(AddPNode::Base, src);
1264 addp->set_req(AddPNode::Address, src);
1265
1266 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1267 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1268 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1269
1270 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1271
1272 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1273 #ifdef _LP64
1274 diff = phase->transform(new ConvI2LNode(diff));
1275 #endif
1276 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1277
1278 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1279 addp->set_req(AddPNode::Offset, offset);
1280 }
1281 addp = phase->transform(addp);
1282 #ifdef ASSERT
1283 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1284 ld->_adr_type = adr_type;
1285 #endif
1286 ld->set_req(MemNode::Address, addp);
1287 ld->set_req(0, ctl);
1288 ld->set_req(MemNode::Memory, mem);
1289 return ld;
1290 }
1291 return nullptr;
1292 }
1293
1294 static Node* see_through_inline_type(PhaseValues* phase, const MemNode* load, Node* base, int offset) {
1295 if (!load->is_mismatched_access() && base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1296 InlineTypeNode* vt = base->as_InlineType();
1297 Node* value = vt->field_value_by_offset(offset, true);
1298 assert(value != nullptr, "must see some value");
1299 return value;
1300 }
1301
1302 return nullptr;
1303 }
1304
1305 //---------------------------can_see_stored_value------------------------------
1306 // This routine exists to make sure this set of tests is done the same
1307 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1308 // will change the graph shape in a way which makes memory alive twice at the
1309 // same time (uses the Oracle model of aliasing), then some
1310 // LoadXNode::Identity will fold things back to the equivalence-class model
1311 // of aliasing.
1312 // This method may find an unencoded node instead of the corresponding encoded one.
1313 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1314 Node* ld_adr = in(MemNode::Address);
1315 intptr_t ld_off = 0;
1316 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1317 // Try to see through an InlineTypeNode
1318 // LoadN is special because the input is not compressed
1319 if (Opcode() != Op_LoadN) {
1320 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1321 if (value != nullptr) {
1322 return value;
1323 }
1324 }
1325
1326 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1327 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1328 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1329 // This is more general than load from boxing objects.
1330 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1331 uint alias_idx = atp->index();
1332 Node* result = nullptr;
1333 Node* current = st;
1334 // Skip through chains of MemBarNodes checking the MergeMems for
1335 // new states for the slice of this load. Stop once any other
1336 // kind of node is encountered. Loads from final memory can skip
1337 // through any kind of MemBar but normal loads shouldn't skip
1338 // through MemBarAcquire since the could allow them to move out of
1339 // a synchronized region. It is not safe to step over MemBarCPUOrder,
1340 // because alias info above them may be inaccurate (e.g., due to
1341 // mixed/mismatched unsafe accesses).
1342 bool is_final_mem = !atp->is_rewritable();
1343 while (current->is_Proj()) {
1344 int opc = current->in(0)->Opcode();
1345 if ((is_final_mem && (opc == Op_MemBarAcquire ||
1412
1413 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1414 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1415 if (in_vt != out_vt) {
1416 return nullptr;
1417 }
1418 }
1419 return st->in(MemNode::ValueIn);
1420 }
1421
1422 // A load from a freshly-created object always returns zero.
1423 // (This can happen after LoadNode::Ideal resets the load's memory input
1424 // to find_captured_store, which returned InitializeNode::zero_memory.)
1425 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1426 (st->in(0) == ld_alloc) &&
1427 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1428 // return a zero value for the load's basic type
1429 // (This is one of the few places where a generic PhaseTransform
1430 // can create new nodes. Think of it as lazily manifesting
1431 // virtually pre-existing constants.)
1432 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1433 if (init_value != nullptr) {
1434 const TypeAryPtr* ld_adr_type = phase->type(ld_adr)->isa_aryptr();
1435 if (ld_adr_type == nullptr) {
1436 return nullptr;
1437 }
1438
1439 // We know that this is not a flat array, the load should return the whole oop
1440 if (ld_adr_type->is_not_flat()) {
1441 return init_value;
1442 }
1443
1444 // If this is a flat array, try to see through init_value
1445 if (init_value->is_EncodeP()) {
1446 init_value = init_value->in(1);
1447 }
1448 if (!init_value->is_InlineType() || ld_adr_type->field_offset() == Type::Offset::bottom) {
1449 return nullptr;
1450 }
1451
1452 ciInlineKlass* vk = phase->type(init_value)->inline_klass();
1453 int field_offset_in_payload = ld_adr_type->field_offset().get();
1454 if (field_offset_in_payload == vk->null_marker_offset_in_payload()) {
1455 return init_value->as_InlineType()->get_null_marker();
1456 } else {
1457 return init_value->as_InlineType()->field_value_by_offset(field_offset_in_payload + vk->payload_offset(), true);
1458 }
1459 }
1460 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1461 if (value_basic_type() != T_VOID) {
1462 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1463 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1464 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1465 // by the ArrayCopyNode.
1466 return phase->zerocon(value_basic_type());
1467 }
1468 } else {
1469 // TODO: materialize all-zero vector constant
1470 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1471 }
1472 }
1473
1474 // A load from an initialization barrier can match a captured store.
1475 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1476 InitializeNode* init = st->in(0)->as_Initialize();
1477 AllocateNode* alloc = init->allocation();
1478 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1479 // examine a captured store value
1480 st = init->find_captured_store(ld_off, memory_size(), phase);
1493 base = bs->step_over_gc_barrier(base);
1494 if (base != nullptr && base->is_Proj() &&
1495 base->as_Proj()->_con == TypeFunc::Parms &&
1496 base->in(0)->is_CallStaticJava() &&
1497 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1498 return base->in(0)->in(TypeFunc::Parms);
1499 }
1500 }
1501
1502 break;
1503 }
1504
1505 return nullptr;
1506 }
1507
1508 //----------------------is_instance_field_load_with_local_phi------------------
1509 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1510 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1511 in(Address)->is_AddP() ) {
1512 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1513 // Only known instances and immutable fields
1514 if( t_oop != nullptr &&
1515 (t_oop->is_ptr_to_strict_final_field() ||
1516 t_oop->is_known_instance_field()) &&
1517 t_oop->offset() != Type::OffsetBot &&
1518 t_oop->offset() != Type::OffsetTop) {
1519 return true;
1520 }
1521 }
1522 return false;
1523 }
1524
1525 //------------------------------Identity---------------------------------------
1526 // Loads are identity if previous store is to same address
1527 Node* LoadNode::Identity(PhaseGVN* phase) {
1528 // If the previous store-maker is the right kind of Store, and the store is
1529 // to the same address, then we are equal to the value stored.
1530 Node* mem = in(Memory);
1531 Node* value = can_see_stored_value(mem, phase);
1532 if( value ) {
1533 // byte, short & char stores truncate naturally.
1534 // A load has to load the truncated value which requires
1535 // some sort of masking operation and that requires an
1536 // Ideal call instead of an Identity call.
1537 if (memory_size() < BytesPerInt) {
1538 // If the input to the store does not fit with the load's result type,
1539 // it must be truncated via an Ideal call.
1540 if (!phase->type(value)->higher_equal(phase->type(this)))
1541 return this;
1542 }
1543
1544 if (phase->type(value)->isa_ptr() && phase->type(this)->isa_narrowoop()) {
1545 return this;
1546 }
1547 // (This works even when value is a Con, but LoadNode::Value
1548 // usually runs first, producing the singleton type of the Con.)
1549 if (!has_pinned_control_dependency() || value->is_Con()) {
1550 return value;
1551 } else {
1552 return this;
1553 }
1554 }
1555
1556 if (has_pinned_control_dependency()) {
1557 return this;
1558 }
1559 // Search for an existing data phi which was generated before for the same
1560 // instance's field to avoid infinite generation of phis in a loop.
1561 Node *region = mem->in(0);
1562 if (is_instance_field_load_with_local_phi(region)) {
1563 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1564 int this_index = phase->C->get_alias_index(addr_t);
1565 int this_offset = addr_t->offset();
1566 int this_iid = addr_t->instance_id();
1567 if (!addr_t->is_known_instance() &&
1568 addr_t->is_ptr_to_strict_final_field()) {
1569 // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1570 intptr_t ignore = 0;
1571 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1572 if (base == nullptr) {
1573 return this;
1574 }
1575 this_iid = base->_idx;
1576 }
1577 const Type* this_type = bottom_type();
1578 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1579 Node* phi = region->fast_out(i);
1580 if (phi->is_Phi() && phi != mem &&
1581 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1582 return phi;
1583 }
1584 }
1585 }
1586
1587 return this;
1588 }
1589
2104 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2105 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2106
2107 // Skip up past a SafePoint control. Cannot do this for Stores because
2108 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2109 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2110 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2111 !addr_mark &&
2112 (depends_only_on_test() || has_unknown_control_dependency())) {
2113 ctrl = ctrl->in(0);
2114 set_req(MemNode::Control,ctrl);
2115 return this;
2116 }
2117
2118 intptr_t ignore = 0;
2119 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2120 if (base != nullptr
2121 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2122 // Check for useless control edge in some common special cases
2123 if (in(MemNode::Control) != nullptr
2124 // TODO 8350865 Can we re-enable this?
2125 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2126 && can_remove_control()
2127 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2128 && all_controls_dominate(base, phase->C->start())) {
2129 // A method-invariant, non-null address (constant or 'this' argument).
2130 set_req(MemNode::Control, nullptr);
2131 return this;
2132 }
2133 }
2134
2135 Node* mem = in(MemNode::Memory);
2136 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2137
2138 if (can_reshape && (addr_t != nullptr)) {
2139 // try to optimize our memory input
2140 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2141 if (opt_mem != mem) {
2142 set_req_X(MemNode::Memory, opt_mem, phase);
2143 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2144 return this;
2145 }
2258 // No match.
2259 return nullptr;
2260 }
2261
2262 //------------------------------Value-----------------------------------------
2263 const Type* LoadNode::Value(PhaseGVN* phase) const {
2264 // Either input is TOP ==> the result is TOP
2265 Node* mem = in(MemNode::Memory);
2266 const Type *t1 = phase->type(mem);
2267 if (t1 == Type::TOP) return Type::TOP;
2268 Node* adr = in(MemNode::Address);
2269 const TypePtr* tp = phase->type(adr)->isa_ptr();
2270 if (tp == nullptr || tp->empty()) return Type::TOP;
2271 int off = tp->offset();
2272 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2273 Compile* C = phase->C;
2274
2275 // If load can see a previous constant store, use that.
2276 Node* value = can_see_stored_value(mem, phase);
2277 if (value != nullptr && value->is_Con()) {
2278 if (phase->type(value)->isa_ptr() && _type->isa_narrowoop()) {
2279 return phase->type(value)->make_narrowoop();
2280 } else {
2281 assert(value->bottom_type()->higher_equal(_type), "sanity");
2282 return phase->type(value);
2283 }
2284 }
2285 // Try to guess loaded type from pointer type
2286 if (tp->isa_aryptr()) {
2287 const TypeAryPtr* ary = tp->is_aryptr();
2288 const Type* t = ary->elem();
2289
2290 // Determine whether the reference is beyond the header or not, by comparing
2291 // the offset against the offset of the start of the array's data.
2292 // Different array types begin at slightly different offsets (12 vs. 16).
2293 // We choose T_BYTE as an example base type that is least restrictive
2294 // as to alignment, which will therefore produce the smallest
2295 // possible base offset.
2296 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2297 const bool off_beyond_header = (off >= min_base_off);
2298
2299 // Try to constant-fold a stable array element.
2300 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2301 // Make sure the reference is not into the header and the offset is constant
2302 ciObject* aobj = ary->const_oop();
2303 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2304 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2305 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off, ary->field_offset().get(),
2306 stable_dimension,
2307 value_basic_type(), is_unsigned());
2308 if (con_type != nullptr) {
2309 return con_type;
2310 }
2311 }
2312 }
2313
2314 // Don't do this for integer types. There is only potential profit if
2315 // the element type t is lower than _type; that is, for int types, if _type is
2316 // more restrictive than t. This only happens here if one is short and the other
2317 // char (both 16 bits), and in those cases we've made an intentional decision
2318 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2319 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2320 //
2321 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2322 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2323 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2324 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2325 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2326 // In fact, that could have been the original type of p1, and p1 could have
2327 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2328 // expression (LShiftL quux 3) independently optimized to the constant 8.
2329 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2330 && (_type->isa_vect() == nullptr)
2331 && !ary->is_flat()
2332 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2333 // t might actually be lower than _type, if _type is a unique
2334 // concrete subclass of abstract class t.
2335 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2336 const Type* jt = t->join_speculative(_type);
2337 // In any case, do not allow the join, per se, to empty out the type.
2338 if (jt->empty() && !t->empty()) {
2339 // This can happen if a interface-typed array narrows to a class type.
2340 jt = _type;
2341 }
2342 #ifdef ASSERT
2343 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2344 // The pointers in the autobox arrays are always non-null
2345 Node* base = adr->in(AddPNode::Base);
2346 if ((base != nullptr) && base->is_DecodeN()) {
2347 // Get LoadN node which loads IntegerCache.cache field
2348 base = base->in(1);
2349 }
2350 if ((base != nullptr) && base->is_Con()) {
2351 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2352 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2353 // It could be narrow oop
2354 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2355 }
2356 }
2357 }
2358 #endif
2359 return jt;
2360 }
2361 }
2362 } else if (tp->base() == Type::InstPtr) {
2363 assert( off != Type::OffsetBot ||
2364 // arrays can be cast to Objects
2365 !tp->isa_instptr() ||
2366 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2367 // Default value load
2368 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2369 // unsafe field access may not have a constant offset
2370 C->has_unsafe_access(),
2371 "Field accesses must be precise" );
2372 // For oop loads, we expect the _type to be precise.
2373
2374 const TypeInstPtr* tinst = tp->is_instptr();
2375 BasicType bt = value_basic_type();
2376
2377 // Fold loads of the field map
2378 if (tinst != nullptr) {
2379 ciInstanceKlass* ik = tinst->instance_klass();
2380 int offset = tinst->offset();
2381 if (ik == phase->C->env()->Class_klass()) {
2382 ciType* t = tinst->java_mirror_type();
2383 if (t != nullptr && t->is_inlinetype() && offset == t->as_inline_klass()->field_map_offset()) {
2384 ciConstant map = t->as_inline_klass()->get_field_map();
2385 bool is_narrow_oop = (bt == T_NARROWOOP);
2386 return Type::make_from_constant(map, true, 1, is_narrow_oop);
2387 }
2388 }
2389 }
2390
2391 // Optimize loads from constant fields.
2392 ciObject* const_oop = tinst->const_oop();
2393 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2394 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2395 if (con_type != nullptr) {
2396 return con_type;
2397 }
2398 }
2399 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2400 assert(off != Type::OffsetBot ||
2401 !tp->isa_instklassptr() ||
2402 // arrays can be cast to Objects
2403 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2404 // also allow array-loading from the primary supertype
2405 // array during subtype checks
2406 Opcode() == Op_LoadKlass,
2407 "Field accesses must be precise");
2408 // For klass/static loads, we expect the _type to be precise
2409 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2410 /* With mirrors being an indirect in the Klass*
2411 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2412 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2413 *
2414 * So check the type and klass of the node before the LoadP.
2421 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2422 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2423 return TypeInstPtr::make(klass->java_mirror());
2424 }
2425 }
2426 }
2427
2428 const TypeKlassPtr *tkls = tp->isa_klassptr();
2429 if (tkls != nullptr) {
2430 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2431 ciKlass* klass = tkls->exact_klass();
2432 // We are loading a field from a Klass metaobject whose identity
2433 // is known at compile time (the type is "exact" or "precise").
2434 // Check for fields we know are maintained as constants by the VM.
2435 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2436 // The field is Klass::_super_check_offset. Return its (constant) value.
2437 // (Folds up type checking code.)
2438 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2439 return TypeInt::make(klass->super_check_offset());
2440 }
2441 if (klass->is_inlinetype() && tkls->offset() == in_bytes(InstanceKlass::acmp_maps_offset_offset())) {
2442 return TypeInt::make(klass->as_inline_klass()->field_map_offset());
2443 }
2444 if (klass->is_obj_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset())) {
2445 // Fold loads from LibraryCallKit::load_default_refined_array_klass
2446 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr();
2447 }
2448 if (klass->is_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::properties_offset())) {
2449 assert(klass->is_type_array_klass() || tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2450 return TypeInt::make((jint)klass->as_array_klass()->properties().value());
2451 }
2452 if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2453 assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2454 return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2455 }
2456 if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2457 // The field is Klass::_prototype_header. Return its (constant) value.
2458 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2459 return TypeX::make(klass->prototype_header());
2460 }
2461 // Compute index into primary_supers array
2462 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2463 // Check for overflowing; use unsigned compare to handle the negative case.
2464 if( depth < ciKlass::primary_super_limit() ) {
2465 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2466 // (Folds up type checking code.)
2467 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2468 ciKlass *ss = klass->super_of_depth(depth);
2469 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2470 }
2471 const Type* aift = load_array_final_field(tkls, klass);
2472 if (aift != nullptr) return aift;
2473 }
2474
2475 // We can still check if we are loading from the primary_supers array at a
2476 // shallow enough depth. Even though the klass is not exact, entries less
2477 // than or equal to its super depth are correct.
2478 if (tkls->is_loaded()) {
2479 ciKlass* klass = nullptr;
2513 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2514 // The key property of this type is that it folds up tests
2515 // for array-ness, since it proves that the layout_helper is positive.
2516 // Thus, a generic value like the basic object layout helper works fine.
2517 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2518 }
2519 }
2520
2521 // If we are loading from a freshly-allocated object/array, produce a zero.
2522 // Things to check:
2523 // 1. Load is beyond the header: headers are not guaranteed to be zero
2524 // 2. Load is not vectorized: vectors have no zero constant
2525 // 3. Load has no matching store, i.e. the input is the initial memory state
2526 const TypeOopPtr* tinst = tp->isa_oopptr();
2527 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2528 bool is_not_vect = (_type->isa_vect() == nullptr);
2529 if (is_not_header && is_not_vect) {
2530 Node* mem = in(MemNode::Memory);
2531 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2532 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2533 // TODO 8350865 Scalar replacement does not work well for flat arrays.
2534 // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2535 // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2536 // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2537 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2538 intptr_t offset = 0;
2539 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2540 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2541 if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2542 return _type;
2543 }
2544 }
2545 return Type::get_zero_type(_type->basic_type());
2546 }
2547 }
2548 if (!UseCompactObjectHeaders) {
2549 Node* alloc = is_new_object_mark_load();
2550 if (alloc != nullptr) {
2551 if (Arguments::is_valhalla_enabled()) {
2552 // The mark word may contain property bits (inline, flat, null-free)
2553 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2554 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2555 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2556 return TypeX::make(tkls->exact_klass()->prototype_header());
2557 }
2558 } else {
2559 return TypeX::make(markWord::prototype().value());
2560 }
2561 }
2562 }
2563
2564 return _type;
2565 }
2566
2567 //------------------------------match_edge-------------------------------------
2568 // Do we Match on this edge index or not? Match only the address.
2569 uint LoadNode::match_edge(uint idx) const {
2570 return idx == MemNode::Address;
2571 }
2572
2573 //--------------------------LoadBNode::Ideal--------------------------------------
2574 //
2575 // If the previous store is to the same address as this load,
2576 // and the value stored was larger than a byte, replace this load
2577 // with the value stored truncated to a byte. If no truncation is
2578 // needed, the replacement is done in LoadNode::Identity().
2579 //
2580 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2689 }
2690 }
2691 // Identity call will handle the case where truncation is not needed.
2692 return LoadNode::Ideal(phase, can_reshape);
2693 }
2694
2695 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2696 Node* mem = in(MemNode::Memory);
2697 Node* value = can_see_stored_value(mem,phase);
2698 if (value != nullptr && value->is_Con() &&
2699 !value->bottom_type()->higher_equal(_type)) {
2700 // If the input to the store does not fit with the load's result type,
2701 // it must be truncated. We can't delay until Ideal call since
2702 // a singleton Value is needed for split_thru_phi optimization.
2703 int con = value->get_int();
2704 return TypeInt::make((con << 16) >> 16);
2705 }
2706 return LoadNode::Value(phase);
2707 }
2708
2709 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2710 // Loading from an InlineType, find the input and make an EncodeP
2711 Node* addr = in(Address);
2712 intptr_t offset;
2713 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
2714 Node* value = see_through_inline_type(phase, this, base, offset);
2715 if (value != nullptr) {
2716 return new EncodePNode(value, type());
2717 }
2718
2719 // Can see the corresponding value, may need to add an EncodeP
2720 value = can_see_stored_value(in(Memory), phase);
2721 if (value != nullptr && phase->type(value)->isa_ptr() && type()->isa_narrowoop()) {
2722 return new EncodePNode(value, type());
2723 }
2724
2725 // Identity call will handle the case where EncodeP is unnecessary
2726 return LoadNode::Ideal(phase, can_reshape);
2727 }
2728
2729 //=============================================================================
2730 //----------------------------LoadKlassNode::make------------------------------
2731 // Polymorphic factory method:
2732 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2733 // sanity check the alias category against the created node type
2734 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2735 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2736 #ifdef _LP64
2737 if (adr_type->is_ptr_to_narrowklass()) {
2738 assert(UseCompressedClassPointers, "no compressed klasses");
2739 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2740 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2741 }
2742 #endif
2743 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2744 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2745 }
2746
2747 //------------------------------Value------------------------------------------
2748 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2782 }
2783 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2784 }
2785 if (!t->is_klass()) {
2786 // a primitive Class (e.g., int.class) has null for a klass field
2787 return TypePtr::NULL_PTR;
2788 }
2789 // Fold up the load of the hidden field
2790 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2791 }
2792 // non-constant mirror, so we can't tell what's going on
2793 }
2794 if (!tinst->is_loaded())
2795 return _type; // Bail out if not loaded
2796 if (offset == oopDesc::klass_offset_in_bytes()) {
2797 return tinst->as_klass_type(true);
2798 }
2799 }
2800
2801 // Check for loading klass from an array
2802 const TypeAryPtr* tary = tp->isa_aryptr();
2803 if (tary != nullptr &&
2804 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2805 return tary->as_klass_type(true)->is_aryklassptr();
2806 }
2807
2808 // Check for loading klass from an array klass
2809 const TypeKlassPtr *tkls = tp->isa_klassptr();
2810 if (tkls != nullptr && !StressReflectiveCode) {
2811 if (!tkls->is_loaded())
2812 return _type; // Bail out if not loaded
2813 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2814 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2815 // // Always returning precise element type is incorrect,
2816 // // e.g., element type could be object and array may contain strings
2817 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2818
2819 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2820 // according to the element type's subclassing.
2821 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2822 }
2823 if (tkls->isa_aryklassptr() != nullptr && tkls->klass_is_exact() &&
2824 !tkls->exact_klass()->is_type_array_klass() &&
2825 tkls->offset() == in_bytes(Klass::super_offset())) {
2826 // We are loading the super klass of a refined array klass, return the non-refined klass pointer
2827 assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2828 return tkls->is_aryklassptr()->with_offset(0)->cast_to_non_refined();
2829 }
2830 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2831 tkls->offset() == in_bytes(Klass::super_offset())) {
2832 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2833 // The field is Klass::_super. Return its (constant) value.
2834 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2835 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2836 }
2837 }
2838
2839 if (tkls != nullptr && !UseSecondarySupersCache
2840 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2841 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2842 return TypePtr::NULL_PTR;
2843 }
2844
2845 // Bailout case
2846 return LoadNode::Value(phase);
2847 }
2848
2849 //------------------------------Identity---------------------------------------
2872 base = bs->step_over_gc_barrier(base);
2873 }
2874
2875 // We can fetch the klass directly through an AllocateNode.
2876 // This works even if the klass is not constant (clone or newArray).
2877 if (offset == oopDesc::klass_offset_in_bytes()) {
2878 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2879 if (allocated_klass != nullptr) {
2880 return allocated_klass;
2881 }
2882 }
2883
2884 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2885 // See inline_native_Class_query for occurrences of these patterns.
2886 // Java Example: x.getClass().isAssignableFrom(y)
2887 //
2888 // This improves reflective code, often making the Class
2889 // mirror go completely dead. (Current exception: Class
2890 // mirrors may appear in debug info, but we could clean them out by
2891 // introducing a new debug info operator for Klass.java_mirror).
2892 //
2893 // This optimization does not apply to arrays because if k is not a
2894 // constant, it was obtained via load_klass which returns the VM type
2895 // and '.java_mirror.as_klass' should return the Java type instead.
2896
2897 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2898 && offset == java_lang_Class::klass_offset()) {
2899 if (base->is_Load()) {
2900 Node* base2 = base->in(MemNode::Address);
2901 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2902 Node* adr2 = base2->in(MemNode::Address);
2903 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2904 if (tkls != nullptr && !tkls->empty()
2905 && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2906 && adr2->is_AddP()) {
2907 int mirror_field = in_bytes(Klass::java_mirror_offset());
2908 if (tkls->offset() == mirror_field) {
2909 #ifdef ASSERT
2910 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2911 assert(tkls2->offset() == 0, "not a load of java_mirror");
2912 #endif
2913 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2914 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2915 return adr2->in(AddPNode::Address);
2916 }
2917 }
2918 }
2919 }
2920 }
2921
2922 return this;
2923 }
2924
2925 LoadNode* LoadNode::clone_pinned() const {
2926 LoadNode* ld = clone()->as_Load();
3052 //---------------------------StoreNode::make-----------------------------------
3053 // Polymorphic factory method:
3054 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
3055 assert((mo == unordered || mo == release), "unexpected");
3056 Compile* C = gvn.C;
3057 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
3058 ctl != nullptr, "raw memory operations should have control edge");
3059
3060 switch (bt) {
3061 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
3062 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
3063 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
3064 case T_CHAR:
3065 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
3066 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3067 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
3068 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3069 case T_METADATA:
3070 case T_ADDRESS:
3071 case T_OBJECT:
3072 case T_ARRAY:
3073 #ifdef _LP64
3074 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3075 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
3076 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
3077 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
3078 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
3079 adr->bottom_type()->isa_rawptr())) {
3080 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
3081 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
3082 }
3083 #endif
3084 {
3085 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
3086 }
3087 default:
3088 guarantee(false, "unexpected basic type %s", type2name(bt));
3089 return (StoreNode*)nullptr;
3090 }
3091 }
3092
3093 //--------------------------bottom_type----------------------------------------
3094 const Type *StoreNode::bottom_type() const {
3095 return Type::MEMORY;
3096 }
3097
3098 //------------------------------hash-------------------------------------------
3099 uint StoreNode::hash() const {
3100 // unroll addition of interesting fields
3101 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
3102
3103 // Since they are not commoned, do not hash them:
3104 return NO_HASH;
3105 }
3106
3107 // Link together multiple stores (B/S/C/I) into a longer one.
3108 //
3730 }
3731 ss.print_cr("[TraceMergeStores]: with");
3732 merged_input_value->dump("\n", false, &ss);
3733 merged_store->dump("\n", false, &ss);
3734 tty->print("%s", ss.as_string());
3735 }
3736 #endif
3737
3738 //------------------------------Ideal------------------------------------------
3739 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3740 // When a store immediately follows a relevant allocation/initialization,
3741 // try to capture it into the initialization, or hoist it above.
3742 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3743 Node* p = MemNode::Ideal_common(phase, can_reshape);
3744 if (p) return (p == NodeSentinel) ? nullptr : p;
3745
3746 Node* mem = in(MemNode::Memory);
3747 Node* address = in(MemNode::Address);
3748 Node* value = in(MemNode::ValueIn);
3749 // Back-to-back stores to same address? Fold em up. Generally
3750 // unsafe if I have intervening uses...
3751 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3752 Node* st = mem;
3753 // If Store 'st' has more than one use, we cannot fold 'st' away.
3754 // For example, 'st' might be the final state at a conditional
3755 // return. Or, 'st' might be used by some node which is live at
3756 // the same time 'st' is live, which might be unschedulable. So,
3757 // require exactly ONE user until such time as we clone 'mem' for
3758 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3759 // true).
3760 while (st->is_Store() && st->outcnt() == 1) {
3761 // Looking at a dead closed cycle of memory?
3762 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3763 assert(Opcode() == st->Opcode() ||
3764 st->Opcode() == Op_StoreVector ||
3765 Opcode() == Op_StoreVector ||
3766 st->Opcode() == Op_StoreVectorScatter ||
3767 Opcode() == Op_StoreVectorScatter ||
3768 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3769 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3770 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3771 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3772 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3773 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3774
3775 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3776 st->as_Store()->memory_size() <= this->memory_size()) {
3777 Node* use = st->raw_out(0);
3778 if (phase->is_IterGVN()) {
3779 phase->is_IterGVN()->rehash_node_delayed(use);
3780 }
3781 // It's OK to do this in the parser, since DU info is always accurate,
3782 // and the parser always refers to nodes via SafePointNode maps.
3783 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3784 return this;
3785 }
3786 st = st->in(MemNode::Memory);
3787 }
3788 }
3789
3790
3791 // Capture an unaliased, unconditional, simple store into an initializer.
3892 const StoreVectorNode* store_vector = as_StoreVector();
3893 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3894 const Node* store_indices = store_vector->indices();
3895 const Node* mem_indices = mem_vector->indices();
3896 const Node* store_mask = store_vector->mask();
3897 const Node* mem_mask = mem_vector->mask();
3898 // Ensure types, indices, and masks match
3899 if (store_vector->vect_type() == mem_vector->vect_type() &&
3900 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3901 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3902 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3903 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3904 result = mem;
3905 }
3906 }
3907 }
3908
3909 // Store of zero anywhere into a freshly-allocated object?
3910 // Then the store is useless.
3911 // (It must already have been captured by the InitializeNode.)
3912 if (result == this && ReduceFieldZeroing) {
3913 // a newly allocated object is already all-zeroes everywhere
3914 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3915 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3916 result = mem;
3917 }
3918
3919 if (result == this && phase->type(val)->is_zero_type()) {
3920 // the store may also apply to zero-bits in an earlier object
3921 Node* prev_mem = find_previous_store(phase);
3922 // Steps (a), (b): Walk past independent stores to find an exact match.
3923 if (prev_mem != nullptr) {
3924 if (prev_mem->is_top()) {
3925 // find_previous_store returns top when the access is dead
3926 return prev_mem;
3927 }
3928 Node* prev_val = can_see_stored_value(prev_mem, phase);
3929 if (prev_val != nullptr && prev_val == val) {
3930 // prev_val and val might differ by a cast; it would be good
3931 // to keep the more informative of the two.
3932 result = mem;
3933 }
3934 }
3935 }
3936 }
3937
3938 PhaseIterGVN* igvn = phase->is_IterGVN();
3939 if (result != this && igvn != nullptr) {
4412 // Clearing a short array is faster with stores
4413 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4414 // Already know this is a large node, do not try to ideal it
4415 if (_is_large) return nullptr;
4416
4417 const int unit = BytesPerLong;
4418 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4419 if (!t) return nullptr;
4420 if (!t->is_con()) return nullptr;
4421 intptr_t raw_count = t->get_con();
4422 intptr_t size = raw_count;
4423 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4424 // Clearing nothing uses the Identity call.
4425 // Negative clears are possible on dead ClearArrays
4426 // (see jck test stmt114.stmt11402.val).
4427 if (size <= 0 || size % unit != 0) return nullptr;
4428 intptr_t count = size / unit;
4429 // Length too long; communicate this to matchers and assemblers.
4430 // Assemblers are responsible to produce fast hardware clears for it.
4431 if (size > InitArrayShortSize) {
4432 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4433 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4434 return nullptr;
4435 }
4436 if (!IdealizeClearArrayNode) return nullptr;
4437 Node *mem = in(1);
4438 if( phase->type(mem)==Type::TOP ) return nullptr;
4439 Node *adr = in(3);
4440 const Type* at = phase->type(adr);
4441 if( at==Type::TOP ) return nullptr;
4442 const TypePtr* atp = at->isa_ptr();
4443 // adjust atp to be the correct array element address type
4444 if (atp == nullptr) atp = TypePtr::BOTTOM;
4445 else atp = atp->add_offset(Type::OffsetBot);
4446 // Get base for derived pointer purposes
4447 if( adr->Opcode() != Op_AddP ) Unimplemented();
4448 Node *base = adr->in(1);
4449
4450 Node *val = in(4);
4451 Node *off = phase->MakeConX(BytesPerLong);
4452 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4453 count--;
4454 while( count-- ) {
4455 mem = phase->transform(mem);
4456 adr = phase->transform(new AddPNode(base,adr,off));
4457 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4458 }
4459 return mem;
4460 }
4461
4462 //----------------------------step_through----------------------------------
4463 // Return allocation input memory edge if it is different instance
4464 // or itself if it is the one we are looking for.
4465 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4466 Node* n = *np;
4467 assert(n->is_ClearArray(), "sanity");
4468 intptr_t offset;
4469 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4470 // This method is called only before Allocate nodes are expanded
4471 // during macro nodes expansion. Before that ClearArray nodes are
4472 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4473 // Allocate nodes are expanded) which follows allocations.
4474 assert(alloc != nullptr, "should have allocation");
4475 if (alloc->_idx == instance_id) {
4476 // Can not bypass initialization of the instance we are looking for.
4477 return false;
4480 InitializeNode* init = alloc->initialization();
4481 if (init != nullptr)
4482 *np = init->in(TypeFunc::Memory);
4483 else
4484 *np = alloc->in(TypeFunc::Memory);
4485 return true;
4486 }
4487
4488 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4489 Node* base = dest;
4490 if (raw_base) {
4491 // May be called as part of the initialization of a just allocated object
4492 base = phase->C->top();
4493 }
4494 return phase->transform(new AddPNode(base, dest, offset));
4495 }
4496
4497 //----------------------------clear_memory-------------------------------------
4498 // Generate code to initialize object storage to zero.
4499 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4500 Node* val,
4501 Node* raw_val,
4502 intptr_t start_offset,
4503 Node* end_offset,
4504 bool raw_base,
4505 PhaseGVN* phase) {
4506 intptr_t offset = start_offset;
4507
4508 int unit = BytesPerLong;
4509 if ((offset % unit) != 0) {
4510 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4511 const TypePtr* atp = TypeRawPtr::BOTTOM;
4512 if (val != nullptr) {
4513 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4514 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4515 } else {
4516 assert(raw_val == nullptr, "val may not be null");
4517 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4518 }
4519 mem = phase->transform(mem);
4520 offset += BytesPerInt;
4521 }
4522 assert((offset % unit) == 0, "");
4523
4524 // Initialize the remaining stuff, if any, with a ClearArray.
4525 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, raw_base, phase);
4526 }
4527
4528 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4529 Node* raw_val,
4530 Node* start_offset,
4531 Node* end_offset,
4532 bool raw_base,
4533 PhaseGVN* phase) {
4534 if (start_offset == end_offset) {
4535 // nothing to do
4536 return mem;
4537 }
4538
4539 int unit = BytesPerLong;
4540 Node* zbase = start_offset;
4541 Node* zend = end_offset;
4542
4543 // Scale to the unit required by the CPU:
4544 if (!Matcher::init_array_count_is_in_bytes) {
4545 Node* shift = phase->intcon(exact_log2(unit));
4546 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4547 zend = phase->transform(new URShiftXNode(zend, shift) );
4548 }
4549
4550 // Bulk clear double-words
4551 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4552 Node* adr = make_address(dest, start_offset, raw_base, phase);
4553 if (raw_val == nullptr) {
4554 raw_val = phase->MakeConX(0);
4555 }
4556 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4557 return phase->transform(mem);
4558 }
4559
4560 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4561 Node* val,
4562 Node* raw_val,
4563 intptr_t start_offset,
4564 intptr_t end_offset,
4565 bool raw_base,
4566 PhaseGVN* phase) {
4567 if (start_offset == end_offset) {
4568 // nothing to do
4569 return mem;
4570 }
4571
4572 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4573 intptr_t done_offset = end_offset;
4574 if ((done_offset % BytesPerLong) != 0) {
4575 done_offset -= BytesPerInt;
4576 }
4577 if (done_offset > start_offset) {
4578 mem = clear_memory(ctl, mem, dest, val, raw_val,
4579 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4580 }
4581 if (done_offset < end_offset) { // emit the final 32-bit store
4582 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4583 const TypePtr* atp = TypeRawPtr::BOTTOM;
4584 if (val != nullptr) {
4585 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4586 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4587 } else {
4588 assert(raw_val == nullptr, "val may not be null");
4589 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4590 }
4591 mem = phase->transform(mem);
4592 done_offset += BytesPerInt;
4593 }
4594 assert(done_offset == end_offset, "");
4595 return mem;
4596 }
4597
4598 //=============================================================================
4599 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4600 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4601 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4602 #ifdef ASSERT
4603 , _pair_idx(0)
4604 #endif
4605 {
4606 init_class_id(Class_MemBar);
4607 Node* top = C->top();
4608 init_req(TypeFunc::I_O,top);
4609 init_req(TypeFunc::FramePtr,top);
4610 init_req(TypeFunc::ReturnAdr,top);
4717 PhaseIterGVN* igvn = phase->is_IterGVN();
4718 remove(igvn);
4719 // Must return either the original node (now dead) or a new node
4720 // (Do not return a top here, since that would break the uniqueness of top.)
4721 return new ConINode(TypeInt::ZERO);
4722 }
4723 }
4724 return progress ? this : nullptr;
4725 }
4726
4727 //------------------------------Value------------------------------------------
4728 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4729 if( !in(0) ) return Type::TOP;
4730 if( phase->type(in(0)) == Type::TOP )
4731 return Type::TOP;
4732 return TypeTuple::MEMBAR;
4733 }
4734
4735 //------------------------------match------------------------------------------
4736 // Construct projections for memory.
4737 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4738 switch (proj->_con) {
4739 case TypeFunc::Control:
4740 case TypeFunc::Memory:
4741 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4742 }
4743 ShouldNotReachHere();
4744 return nullptr;
4745 }
4746
4747 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4748 trailing->_kind = TrailingStore;
4749 leading->_kind = LeadingStore;
4750 #ifdef ASSERT
4751 trailing->_pair_idx = leading->_idx;
4752 leading->_pair_idx = leading->_idx;
4753 #endif
4754 }
4755
4756 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4757 trailing->_kind = TrailingLoadStore;
5004 return (req() > RawStores);
5005 }
5006
5007 void InitializeNode::set_complete(PhaseGVN* phase) {
5008 assert(!is_complete(), "caller responsibility");
5009 _is_complete = Complete;
5010
5011 // After this node is complete, it contains a bunch of
5012 // raw-memory initializations. There is no need for
5013 // it to have anything to do with non-raw memory effects.
5014 // Therefore, tell all non-raw users to re-optimize themselves,
5015 // after skipping the memory effects of this initialization.
5016 PhaseIterGVN* igvn = phase->is_IterGVN();
5017 if (igvn) igvn->add_users_to_worklist(this);
5018 }
5019
5020 // convenience function
5021 // return false if the init contains any stores already
5022 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
5023 InitializeNode* init = initialization();
5024 if (init == nullptr || init->is_complete()) {
5025 return false;
5026 }
5027 init->remove_extra_zeroes();
5028 // for now, if this allocation has already collected any inits, bail:
5029 if (init->is_non_zero()) return false;
5030 init->set_complete(phase);
5031 return true;
5032 }
5033
5034 void InitializeNode::remove_extra_zeroes() {
5035 if (req() == RawStores) return;
5036 Node* zmem = zero_memory();
5037 uint fill = RawStores;
5038 for (uint i = fill; i < req(); i++) {
5039 Node* n = in(i);
5040 if (n->is_top() || n == zmem) continue; // skip
5041 if (fill < i) set_req(fill, n); // compact
5042 ++fill;
5043 }
5044 // delete any empty spaces created:
5045 while (fill < req()) {
5046 del_req(fill);
5190 // store node that we'd like to capture. We need to check
5191 // the uses of the MergeMemNode.
5192 mems.push(n);
5193 }
5194 } else if (n->is_Mem()) {
5195 Node* other_adr = n->in(MemNode::Address);
5196 if (other_adr == adr) {
5197 failed = true;
5198 break;
5199 } else {
5200 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5201 if (other_t_adr != nullptr) {
5202 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5203 if (other_alias_idx == alias_idx) {
5204 // A load from the same memory slice as the store right
5205 // after the InitializeNode. We check the control of the
5206 // object/array that is loaded from. If it's the same as
5207 // the store control then we cannot capture the store.
5208 assert(!n->is_Store(), "2 stores to same slice on same control?");
5209 Node* base = other_adr;
5210 if (base->is_Phi()) {
5211 // In rare case, base may be a PhiNode and it may read
5212 // the same memory slice between InitializeNode and store.
5213 failed = true;
5214 break;
5215 }
5216 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5217 base = base->in(AddPNode::Base);
5218 if (base != nullptr) {
5219 base = base->uncast();
5220 if (base->is_Proj() && base->in(0) == alloc) {
5221 failed = true;
5222 break;
5223 }
5224 }
5225 }
5226 }
5227 }
5228 } else {
5229 failed = true;
5230 break;
5231 }
5232 }
5233 }
5234 }
5235 if (failed) {
5782 // z's_done 12 16 16 16 12 16 12
5783 // z's_needed 12 16 16 16 16 16 16
5784 // zsize 0 0 0 0 4 0 4
5785 if (next_full_store < 0) {
5786 // Conservative tack: Zero to end of current word.
5787 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5788 } else {
5789 // Zero to beginning of next fully initialized word.
5790 // Or, don't zero at all, if we are already in that word.
5791 assert(next_full_store >= zeroes_needed, "must go forward");
5792 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5793 zeroes_needed = next_full_store;
5794 }
5795 }
5796
5797 if (zeroes_needed > zeroes_done) {
5798 intptr_t zsize = zeroes_needed - zeroes_done;
5799 // Do some incremental zeroing on rawmem, in parallel with inits.
5800 zeroes_done = align_down(zeroes_done, BytesPerInt);
5801 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5802 allocation()->in(AllocateNode::InitValue),
5803 allocation()->in(AllocateNode::RawInitValue),
5804 zeroes_done, zeroes_needed,
5805 true,
5806 phase);
5807 zeroes_done = zeroes_needed;
5808 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5809 do_zeroing = false; // leave the hole, next time
5810 }
5811 }
5812
5813 // Collect the store and move on:
5814 phase->replace_input_of(st, MemNode::Memory, inits);
5815 inits = st; // put it on the linearized chain
5816 set_req(i, zmem); // unhook from previous position
5817
5818 if (zeroes_done == st_off)
5819 zeroes_done = next_init_off;
5820
5821 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5822
5823 #ifdef ASSERT
5844 remove_extra_zeroes(); // clear out all the zmems left over
5845 add_req(inits);
5846
5847 if (!(UseTLAB && ZeroTLAB)) {
5848 // If anything remains to be zeroed, zero it all now.
5849 zeroes_done = align_down(zeroes_done, BytesPerInt);
5850 // if it is the last unused 4 bytes of an instance, forget about it
5851 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5852 if (zeroes_done + BytesPerLong >= size_limit) {
5853 AllocateNode* alloc = allocation();
5854 assert(alloc != nullptr, "must be present");
5855 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5856 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5857 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5858 if (zeroes_done == k->layout_helper())
5859 zeroes_done = size_limit;
5860 }
5861 }
5862 if (zeroes_done < size_limit) {
5863 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5864 allocation()->in(AllocateNode::InitValue),
5865 allocation()->in(AllocateNode::RawInitValue),
5866 zeroes_done, size_in_bytes, true, phase);
5867 }
5868 }
5869
5870 set_complete(phase);
5871 return rawmem;
5872 }
5873
5874 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5875 auto replace_proj = [&](ProjNode* proj) {
5876 C->gvn_replace_by(proj, mem);
5877 return CONTINUE;
5878 };
5879 apply_to_projs(replace_proj, TypeFunc::Memory);
5880 }
5881
5882 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5883 DUIterator_Fast imax, i = fast_outs(imax);
5884 auto replace_proj = [&](ProjNode* proj) {
5885 igvn->replace_node(proj, mem);
|