6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/compile.hpp"
38 #include "opto/connode.hpp"
39 #include "opto/convertnode.hpp"
40 #include "opto/loopnode.hpp"
41 #include "opto/machnode.hpp"
42 #include "opto/matcher.hpp"
43 #include "opto/memnode.hpp"
44 #include "opto/mempointer.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/opcodes.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regalloc.hpp"
50 #include "opto/regmask.hpp"
51 #include "opto/rootnode.hpp"
52 #include "opto/traceMergeStoresTag.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/macros.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #include "utilities/vmError.hpp"
59
60 // Portions of code courtesy of Clifford Click
61
62 // Optimization - Graph Style
63
64 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
65
66 //=============================================================================
67 uint MemNode::size_of() const { return sizeof(*this); }
68
69 const TypePtr *MemNode::adr_type() const {
70 Node* adr = in(Address);
71 if (adr == nullptr) return nullptr; // node is dead
72 const TypePtr* cross_check = nullptr;
73 DEBUG_ONLY(cross_check = _adr_type);
74 return calculate_adr_type(adr->bottom_type(), cross_check);
75 }
124 st->print(", idx=Bot;");
125 else if (atp->index() == Compile::AliasIdxTop)
126 st->print(", idx=Top;");
127 else if (atp->index() == Compile::AliasIdxRaw)
128 st->print(", idx=Raw;");
129 else {
130 ciField* field = atp->field();
131 if (field) {
132 st->print(", name=");
133 field->print_name_on(st);
134 }
135 st->print(", idx=%d;", atp->index());
136 }
137 }
138 }
139
140 extern void print_alias_types();
141
142 #endif
143
144 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
145 assert((t_oop != nullptr), "sanity");
146 bool is_instance = t_oop->is_known_instance_field();
147 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
148 (load != nullptr) && load->is_Load() &&
149 (phase->is_IterGVN() != nullptr);
150 if (!(is_instance || is_boxed_value_load))
151 return mchain; // don't try to optimize non-instance types
152 uint instance_id = t_oop->instance_id();
153 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
154 Node *prev = nullptr;
155 Node *result = mchain;
156 while (prev != result) {
157 prev = result;
158 if (result == start_mem)
159 break; // hit one of our sentinels
160 // skip over a call which does not affect this memory slice
161 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
162 Node *proj_in = result->in(0);
163 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
164 break; // hit one of our sentinels
165 } else if (proj_in->is_Call()) {
166 // ArrayCopyNodes processed here as well
167 CallNode *call = proj_in->as_Call();
168 if (!call->may_modify(t_oop, phase)) { // returns false for instances
169 result = call->in(TypeFunc::Memory);
170 }
171 } else if (proj_in->is_Initialize()) {
172 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
173 // Stop if this is the initialization for the object instance which
174 // contains this memory slice, otherwise skip over it.
175 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
176 break;
177 }
178 if (is_instance) {
179 result = proj_in->in(TypeFunc::Memory);
180 } else if (is_boxed_value_load) {
181 Node* klass = alloc->in(AllocateNode::KlassNode);
182 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
183 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
184 result = proj_in->in(TypeFunc::Memory); // not related allocation
185 }
186 }
187 } else if (proj_in->is_MemBar()) {
188 ArrayCopyNode* ac = nullptr;
189 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
190 break;
191 }
192 result = proj_in->in(TypeFunc::Memory);
193 } else if (proj_in->is_top()) {
194 break; // dead code
195 } else {
196 assert(false, "unexpected projection");
197 }
198 } else if (result->is_ClearArray()) {
199 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
200 // Can not bypass initialization of the instance
201 // we are looking for.
202 break;
203 }
204 // Otherwise skip it (the call updated 'result' value).
205 } else if (result->is_MergeMem()) {
206 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
207 }
208 }
209 return result;
210 }
211
212 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
213 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
214 if (t_oop == nullptr)
215 return mchain; // don't try to optimize non-oop types
216 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
217 bool is_instance = t_oop->is_known_instance_field();
218 PhaseIterGVN *igvn = phase->is_IterGVN();
219 if (is_instance && igvn != nullptr && result->is_Phi()) {
220 PhiNode *mphi = result->as_Phi();
221 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
222 const TypePtr *t = mphi->adr_type();
223 bool do_split = false;
224 // In the following cases, Load memory input can be further optimized based on
225 // its precise address type
226 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
227 do_split = true;
228 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
229 const TypeOopPtr* mem_t =
230 t->is_oopptr()->cast_to_exactness(true)
231 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
232 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
233 if (t_oop->isa_aryptr()) {
234 mem_t = mem_t->is_aryptr()
235 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
236 ->cast_to_size(t_oop->is_aryptr()->size())
237 ->with_offset(t_oop->is_aryptr()->offset())
238 ->is_aryptr();
239 }
240 do_split = mem_t == t_oop;
241 }
242 if (do_split) {
243 // clone the Phi with our address type
244 result = mphi->split_out_instance(t_adr, igvn);
245 } else {
246 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
247 }
248 }
249 return result;
250 }
251
252 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
253 uint alias_idx = phase->C->get_alias_index(tp);
254 Node *mem = mmem;
255 #ifdef ASSERT
256 {
257 // Check that current type is consistent with the alias index used during graph construction
258 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
259 bool consistent = adr_check == nullptr || adr_check->empty() ||
260 phase->C->must_alias(adr_check, alias_idx );
261 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
262 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
263 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
264 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
265 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
266 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
267 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
268 // don't assert if it is dead code.
269 consistent = true;
270 }
271 if( !consistent ) {
272 st->print("alias_idx==%d, adr_check==", alias_idx);
273 if( adr_check == nullptr ) {
274 st->print("null");
275 } else {
276 adr_check->dump();
277 }
278 st->cr();
279 print_alias_types();
280 assert(consistent, "adr_check must match alias idx");
281 }
282 }
283 #endif
996 "use LoadKlassNode instead");
997 assert(!(adr_type->isa_aryptr() &&
998 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
999 "use LoadRangeNode instead");
1000 // Check control edge of raw loads
1001 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1002 // oop will be recorded in oop map if load crosses safepoint
1003 rt->isa_oopptr() || is_immutable_value(adr),
1004 "raw memory operations should have control edge");
1005 LoadNode* load = nullptr;
1006 switch (bt) {
1007 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1008 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1009 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1010 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1011 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1012 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1013 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1014 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1015 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1016 case T_OBJECT:
1017 case T_NARROWOOP:
1018 #ifdef _LP64
1019 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1020 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1021 } else
1022 #endif
1023 {
1024 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1025 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1026 }
1027 break;
1028 default:
1029 ShouldNotReachHere();
1030 break;
1031 }
1032 assert(load != nullptr, "LoadNode should have been created");
1033 if (unaligned) {
1034 load->set_unaligned_access();
1035 }
1036 if (mismatched) {
1037 load->set_mismatched_access();
1038 }
1039 if (unsafe) {
1040 load->set_unsafe_access();
1041 }
1042 load->set_barrier_data(barrier_data);
1043 if (load->Opcode() == Op_LoadN) {
1044 Node* ld = gvn.transform(load);
1045 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1046 }
1047
1048 return load;
1049 }
1050
1051 //------------------------------hash-------------------------------------------
1052 uint LoadNode::hash() const {
1053 // unroll addition of interesting fields
1054 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1055 }
1056
1057 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1058 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1059 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1060 bool is_stable_ary = FoldStableValues &&
1061 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1062 tp->isa_aryptr()->is_stable();
1063
1064 return (eliminate_boxing && non_volatile) || is_stable_ary;
1065 }
1066
1067 return false;
1068 }
1069
1070 // Is the value loaded previously stored by an arraycopy? If so return
1071 // a load node that reads from the source array so we may be able to
1072 // optimize out the ArrayCopy node later.
1073 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1074 Node* ld_adr = in(MemNode::Address);
1075 intptr_t ld_off = 0;
1076 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1077 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1078 if (ac != nullptr) {
1079 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1080
1081 Node* mem = ac->in(TypeFunc::Memory);
1082 Node* ctl = ac->in(0);
1083 Node* src = ac->in(ArrayCopyNode::Src);
1084
1092 if (ac->as_ArrayCopy()->is_clonebasic()) {
1093 assert(ld_alloc != nullptr, "need an alloc");
1094 assert(addp->is_AddP(), "address must be addp");
1095 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1096 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1097 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1098 addp->set_req(AddPNode::Base, src);
1099 addp->set_req(AddPNode::Address, src);
1100 } else {
1101 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1102 ac->as_ArrayCopy()->is_copyof_validated() ||
1103 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1104 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1105 addp->set_req(AddPNode::Base, src);
1106 addp->set_req(AddPNode::Address, src);
1107
1108 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1109 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1110 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1111
1112 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1113 uint shift = exact_log2(type2aelembytes(ary_elem));
1114
1115 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1116 #ifdef _LP64
1117 diff = phase->transform(new ConvI2LNode(diff));
1118 #endif
1119 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1120
1121 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1122 addp->set_req(AddPNode::Offset, offset);
1123 }
1124 addp = phase->transform(addp);
1125 #ifdef ASSERT
1126 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1127 ld->_adr_type = adr_type;
1128 #endif
1129 ld->set_req(MemNode::Address, addp);
1130 ld->set_req(0, ctl);
1131 ld->set_req(MemNode::Memory, mem);
1132 return ld;
1133 }
1134 return nullptr;
1135 }
1136
1137
1138 //---------------------------can_see_stored_value------------------------------
1139 // This routine exists to make sure this set of tests is done the same
1140 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1141 // will change the graph shape in a way which makes memory alive twice at the
1142 // same time (uses the Oracle model of aliasing), then some
1143 // LoadXNode::Identity will fold things back to the equivalence-class model
1144 // of aliasing.
1145 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1146 Node* ld_adr = in(MemNode::Address);
1147 intptr_t ld_off = 0;
1148 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1149 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1150 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1151 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1152 // This is more general than load from boxing objects.
1153 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1154 uint alias_idx = atp->index();
1155 Node* result = nullptr;
1156 Node* current = st;
1157 // Skip through chains of MemBarNodes checking the MergeMems for
1158 // new states for the slice of this load. Stop once any other
1159 // kind of node is encountered. Loads from final memory can skip
1160 // through any kind of MemBar but normal loads shouldn't skip
1161 // through MemBarAcquire since the could allow them to move out of
1162 // a synchronized region. It is not safe to step over MemBarCPUOrder,
1163 // because alias info above them may be inaccurate (e.g., due to
1164 // mixed/mismatched unsafe accesses).
1165 bool is_final_mem = !atp->is_rewritable();
1166 while (current->is_Proj()) {
1167 int opc = current->in(0)->Opcode();
1168 if ((is_final_mem && (opc == Op_MemBarAcquire ||
1235
1236 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1237 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1238 if (in_vt != out_vt) {
1239 return nullptr;
1240 }
1241 }
1242 return st->in(MemNode::ValueIn);
1243 }
1244
1245 // A load from a freshly-created object always returns zero.
1246 // (This can happen after LoadNode::Ideal resets the load's memory input
1247 // to find_captured_store, which returned InitializeNode::zero_memory.)
1248 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1249 (st->in(0) == ld_alloc) &&
1250 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1251 // return a zero value for the load's basic type
1252 // (This is one of the few places where a generic PhaseTransform
1253 // can create new nodes. Think of it as lazily manifesting
1254 // virtually pre-existing constants.)
1255 if (value_basic_type() != T_VOID) {
1256 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1257 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1258 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1259 // by the ArrayCopyNode.
1260 return phase->zerocon(value_basic_type());
1261 }
1262 } else {
1263 // TODO: materialize all-zero vector constant
1264 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1265 }
1266 }
1267
1268 // A load from an initialization barrier can match a captured store.
1269 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1270 InitializeNode* init = st->in(0)->as_Initialize();
1271 AllocateNode* alloc = init->allocation();
1272 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1273 // examine a captured store value
1274 st = init->find_captured_store(ld_off, memory_size(), phase);
1287 base = bs->step_over_gc_barrier(base);
1288 if (base != nullptr && base->is_Proj() &&
1289 base->as_Proj()->_con == TypeFunc::Parms &&
1290 base->in(0)->is_CallStaticJava() &&
1291 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1292 return base->in(0)->in(TypeFunc::Parms);
1293 }
1294 }
1295
1296 break;
1297 }
1298
1299 return nullptr;
1300 }
1301
1302 //----------------------is_instance_field_load_with_local_phi------------------
1303 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1304 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1305 in(Address)->is_AddP() ) {
1306 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1307 // Only instances and boxed values.
1308 if( t_oop != nullptr &&
1309 (t_oop->is_ptr_to_boxed_value() ||
1310 t_oop->is_known_instance_field()) &&
1311 t_oop->offset() != Type::OffsetBot &&
1312 t_oop->offset() != Type::OffsetTop) {
1313 return true;
1314 }
1315 }
1316 return false;
1317 }
1318
1319 //------------------------------Identity---------------------------------------
1320 // Loads are identity if previous store is to same address
1321 Node* LoadNode::Identity(PhaseGVN* phase) {
1322 // If the previous store-maker is the right kind of Store, and the store is
1323 // to the same address, then we are equal to the value stored.
1324 Node* mem = in(Memory);
1325 Node* value = can_see_stored_value(mem, phase);
1326 if( value ) {
1327 // byte, short & char stores truncate naturally.
1328 // A load has to load the truncated value which requires
1329 // some sort of masking operation and that requires an
1330 // Ideal call instead of an Identity call.
1331 if (memory_size() < BytesPerInt) {
1332 // If the input to the store does not fit with the load's result type,
1333 // it must be truncated via an Ideal call.
1334 if (!phase->type(value)->higher_equal(phase->type(this)))
1335 return this;
1336 }
1337 // (This works even when value is a Con, but LoadNode::Value
1338 // usually runs first, producing the singleton type of the Con.)
1339 if (!has_pinned_control_dependency() || value->is_Con()) {
1340 return value;
1341 } else {
1342 return this;
1343 }
1344 }
1345
1346 if (has_pinned_control_dependency()) {
1347 return this;
1348 }
1349 // Search for an existing data phi which was generated before for the same
1350 // instance's field to avoid infinite generation of phis in a loop.
1351 Node *region = mem->in(0);
1352 if (is_instance_field_load_with_local_phi(region)) {
1353 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1354 int this_index = phase->C->get_alias_index(addr_t);
1355 int this_offset = addr_t->offset();
1356 int this_iid = addr_t->instance_id();
1357 if (!addr_t->is_known_instance() &&
1358 addr_t->is_ptr_to_boxed_value()) {
1359 // Use _idx of address base (could be Phi node) for boxed values.
1360 intptr_t ignore = 0;
1361 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1362 if (base == nullptr) {
1363 return this;
1364 }
1365 this_iid = base->_idx;
1366 }
1367 const Type* this_type = bottom_type();
1368 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1369 Node* phi = region->fast_out(i);
1370 if (phi->is_Phi() && phi != mem &&
1371 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1372 return phi;
1373 }
1374 }
1375 }
1376
1377 return this;
1378 }
1379
1894 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1895 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1896
1897 // Skip up past a SafePoint control. Cannot do this for Stores because
1898 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1899 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1900 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1901 !addr_mark &&
1902 (depends_only_on_test() || has_unknown_control_dependency())) {
1903 ctrl = ctrl->in(0);
1904 set_req(MemNode::Control,ctrl);
1905 return this;
1906 }
1907
1908 intptr_t ignore = 0;
1909 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1910 if (base != nullptr
1911 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1912 // Check for useless control edge in some common special cases
1913 if (in(MemNode::Control) != nullptr
1914 && can_remove_control()
1915 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1916 && all_controls_dominate(base, phase->C->start())) {
1917 // A method-invariant, non-null address (constant or 'this' argument).
1918 set_req(MemNode::Control, nullptr);
1919 return this;
1920 }
1921 }
1922
1923 Node* mem = in(MemNode::Memory);
1924 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1925
1926 if (can_reshape && (addr_t != nullptr)) {
1927 // try to optimize our memory input
1928 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1929 if (opt_mem != mem) {
1930 set_req_X(MemNode::Memory, opt_mem, phase);
1931 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1932 return this;
1933 }
2046 // No match.
2047 return nullptr;
2048 }
2049
2050 //------------------------------Value-----------------------------------------
2051 const Type* LoadNode::Value(PhaseGVN* phase) const {
2052 // Either input is TOP ==> the result is TOP
2053 Node* mem = in(MemNode::Memory);
2054 const Type *t1 = phase->type(mem);
2055 if (t1 == Type::TOP) return Type::TOP;
2056 Node* adr = in(MemNode::Address);
2057 const TypePtr* tp = phase->type(adr)->isa_ptr();
2058 if (tp == nullptr || tp->empty()) return Type::TOP;
2059 int off = tp->offset();
2060 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2061 Compile* C = phase->C;
2062
2063 // If load can see a previous constant store, use that.
2064 Node* value = can_see_stored_value(mem, phase);
2065 if (value != nullptr && value->is_Con()) {
2066 assert(value->bottom_type()->higher_equal(_type), "sanity");
2067 return value->bottom_type();
2068 }
2069
2070 // Try to guess loaded type from pointer type
2071 if (tp->isa_aryptr()) {
2072 const TypeAryPtr* ary = tp->is_aryptr();
2073 const Type* t = ary->elem();
2074
2075 // Determine whether the reference is beyond the header or not, by comparing
2076 // the offset against the offset of the start of the array's data.
2077 // Different array types begin at slightly different offsets (12 vs. 16).
2078 // We choose T_BYTE as an example base type that is least restrictive
2079 // as to alignment, which will therefore produce the smallest
2080 // possible base offset.
2081 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2082 const bool off_beyond_header = (off >= min_base_off);
2083
2084 // Try to constant-fold a stable array element.
2085 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2086 // Make sure the reference is not into the header and the offset is constant
2087 ciObject* aobj = ary->const_oop();
2088 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2089 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2090 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off,
2091 stable_dimension,
2092 value_basic_type(), is_unsigned());
2093 if (con_type != nullptr) {
2094 return con_type;
2095 }
2096 }
2097 }
2098
2099 // Don't do this for integer types. There is only potential profit if
2100 // the element type t is lower than _type; that is, for int types, if _type is
2101 // more restrictive than t. This only happens here if one is short and the other
2102 // char (both 16 bits), and in those cases we've made an intentional decision
2103 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2104 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2105 //
2106 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2107 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2108 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2109 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2110 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2111 // In fact, that could have been the original type of p1, and p1 could have
2112 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2113 // expression (LShiftL quux 3) independently optimized to the constant 8.
2114 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2115 && (_type->isa_vect() == nullptr)
2116 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2117 // t might actually be lower than _type, if _type is a unique
2118 // concrete subclass of abstract class t.
2119 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2120 const Type* jt = t->join_speculative(_type);
2121 // In any case, do not allow the join, per se, to empty out the type.
2122 if (jt->empty() && !t->empty()) {
2123 // This can happen if a interface-typed array narrows to a class type.
2124 jt = _type;
2125 }
2126 #ifdef ASSERT
2127 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2128 // The pointers in the autobox arrays are always non-null
2129 Node* base = adr->in(AddPNode::Base);
2130 if ((base != nullptr) && base->is_DecodeN()) {
2131 // Get LoadN node which loads IntegerCache.cache field
2132 base = base->in(1);
2133 }
2134 if ((base != nullptr) && base->is_Con()) {
2135 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2136 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2137 // It could be narrow oop
2138 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2139 }
2140 }
2141 }
2142 #endif
2143 return jt;
2144 }
2145 }
2146 } else if (tp->base() == Type::InstPtr) {
2147 assert( off != Type::OffsetBot ||
2148 // arrays can be cast to Objects
2149 !tp->isa_instptr() ||
2150 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2151 // unsafe field access may not have a constant offset
2152 C->has_unsafe_access(),
2153 "Field accesses must be precise" );
2154 // For oop loads, we expect the _type to be precise.
2155
2156 // Optimize loads from constant fields.
2157 const TypeInstPtr* tinst = tp->is_instptr();
2158 ciObject* const_oop = tinst->const_oop();
2159 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2160 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2161 if (con_type != nullptr) {
2162 return con_type;
2163 }
2164 }
2165 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2166 assert(off != Type::OffsetBot ||
2167 !tp->isa_instklassptr() ||
2168 // arrays can be cast to Objects
2169 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2170 // also allow array-loading from the primary supertype
2171 // array during subtype checks
2172 Opcode() == Op_LoadKlass,
2173 "Field accesses must be precise");
2174 // For klass/static loads, we expect the _type to be precise
2175 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2176 /* With mirrors being an indirect in the Klass*
2177 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2178 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2179 *
2180 * So check the type and klass of the node before the LoadP.
2187 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2188 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2189 return TypeInstPtr::make(klass->java_mirror());
2190 }
2191 }
2192 }
2193
2194 const TypeKlassPtr *tkls = tp->isa_klassptr();
2195 if (tkls != nullptr) {
2196 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2197 ciKlass* klass = tkls->exact_klass();
2198 // We are loading a field from a Klass metaobject whose identity
2199 // is known at compile time (the type is "exact" or "precise").
2200 // Check for fields we know are maintained as constants by the VM.
2201 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2202 // The field is Klass::_super_check_offset. Return its (constant) value.
2203 // (Folds up type checking code.)
2204 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2205 return TypeInt::make(klass->super_check_offset());
2206 }
2207 if (UseCompactObjectHeaders) {
2208 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2209 // The field is Klass::_prototype_header. Return its (constant) value.
2210 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2211 return TypeX::make(klass->prototype_header());
2212 }
2213 }
2214 // Compute index into primary_supers array
2215 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2216 // Check for overflowing; use unsigned compare to handle the negative case.
2217 if( depth < ciKlass::primary_super_limit() ) {
2218 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2219 // (Folds up type checking code.)
2220 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2221 ciKlass *ss = klass->super_of_depth(depth);
2222 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2223 }
2224 const Type* aift = load_array_final_field(tkls, klass);
2225 if (aift != nullptr) return aift;
2226 }
2227
2228 // We can still check if we are loading from the primary_supers array at a
2229 // shallow enough depth. Even though the klass is not exact, entries less
2230 // than or equal to its super depth are correct.
2231 if (tkls->is_loaded()) {
2232 ciKlass* klass = nullptr;
2266 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2267 // The key property of this type is that it folds up tests
2268 // for array-ness, since it proves that the layout_helper is positive.
2269 // Thus, a generic value like the basic object layout helper works fine.
2270 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2271 }
2272 }
2273
2274 // If we are loading from a freshly-allocated object/array, produce a zero.
2275 // Things to check:
2276 // 1. Load is beyond the header: headers are not guaranteed to be zero
2277 // 2. Load is not vectorized: vectors have no zero constant
2278 // 3. Load has no matching store, i.e. the input is the initial memory state
2279 const TypeOopPtr* tinst = tp->isa_oopptr();
2280 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2281 bool is_not_vect = (_type->isa_vect() == nullptr);
2282 if (is_not_header && is_not_vect) {
2283 Node* mem = in(MemNode::Memory);
2284 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2285 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2286 return Type::get_zero_type(_type->basic_type());
2287 }
2288 }
2289
2290 if (!UseCompactObjectHeaders) {
2291 Node* alloc = is_new_object_mark_load();
2292 if (alloc != nullptr) {
2293 return TypeX::make(markWord::prototype().value());
2294 }
2295 }
2296
2297 return _type;
2298 }
2299
2300 //------------------------------match_edge-------------------------------------
2301 // Do we Match on this edge index or not? Match only the address.
2302 uint LoadNode::match_edge(uint idx) const {
2303 return idx == MemNode::Address;
2304 }
2305
2306 //--------------------------LoadBNode::Ideal--------------------------------------
2307 //
2308 // If the previous store is to the same address as this load,
2309 // and the value stored was larger than a byte, replace this load
2310 // with the value stored truncated to a byte. If no truncation is
2311 // needed, the replacement is done in LoadNode::Identity().
2312 //
2313 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2422 }
2423 }
2424 // Identity call will handle the case where truncation is not needed.
2425 return LoadNode::Ideal(phase, can_reshape);
2426 }
2427
2428 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2429 Node* mem = in(MemNode::Memory);
2430 Node* value = can_see_stored_value(mem,phase);
2431 if (value != nullptr && value->is_Con() &&
2432 !value->bottom_type()->higher_equal(_type)) {
2433 // If the input to the store does not fit with the load's result type,
2434 // it must be truncated. We can't delay until Ideal call since
2435 // a singleton Value is needed for split_thru_phi optimization.
2436 int con = value->get_int();
2437 return TypeInt::make((con << 16) >> 16);
2438 }
2439 return LoadNode::Value(phase);
2440 }
2441
2442 //=============================================================================
2443 //----------------------------LoadKlassNode::make------------------------------
2444 // Polymorphic factory method:
2445 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2446 // sanity check the alias category against the created node type
2447 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2448 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2449 #ifdef _LP64
2450 if (adr_type->is_ptr_to_narrowklass()) {
2451 assert(UseCompressedClassPointers, "no compressed klasses");
2452 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2453 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2454 }
2455 #endif
2456 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2457 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2458 }
2459
2460 //------------------------------Value------------------------------------------
2461 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2495 }
2496 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2497 }
2498 if (!t->is_klass()) {
2499 // a primitive Class (e.g., int.class) has null for a klass field
2500 return TypePtr::NULL_PTR;
2501 }
2502 // Fold up the load of the hidden field
2503 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2504 }
2505 // non-constant mirror, so we can't tell what's going on
2506 }
2507 if (!tinst->is_loaded())
2508 return _type; // Bail out if not loaded
2509 if (offset == oopDesc::klass_offset_in_bytes()) {
2510 return tinst->as_klass_type(true);
2511 }
2512 }
2513
2514 // Check for loading klass from an array
2515 const TypeAryPtr *tary = tp->isa_aryptr();
2516 if (tary != nullptr &&
2517 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2518 return tary->as_klass_type(true);
2519 }
2520
2521 // Check for loading klass from an array klass
2522 const TypeKlassPtr *tkls = tp->isa_klassptr();
2523 if (tkls != nullptr && !StressReflectiveCode) {
2524 if (!tkls->is_loaded())
2525 return _type; // Bail out if not loaded
2526 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2527 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2528 // // Always returning precise element type is incorrect,
2529 // // e.g., element type could be object and array may contain strings
2530 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2531
2532 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2533 // according to the element type's subclassing.
2534 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2535 }
2536 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2537 tkls->offset() == in_bytes(Klass::super_offset())) {
2538 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2539 // The field is Klass::_super. Return its (constant) value.
2540 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2541 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2542 }
2543 }
2544
2545 if (tkls != nullptr && !UseSecondarySupersCache
2546 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2547 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2548 return TypePtr::NULL_PTR;
2549 }
2550
2551 // Bailout case
2552 return LoadNode::Value(phase);
2553 }
2554
2555 //------------------------------Identity---------------------------------------
2578 base = bs->step_over_gc_barrier(base);
2579 }
2580
2581 // We can fetch the klass directly through an AllocateNode.
2582 // This works even if the klass is not constant (clone or newArray).
2583 if (offset == oopDesc::klass_offset_in_bytes()) {
2584 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2585 if (allocated_klass != nullptr) {
2586 return allocated_klass;
2587 }
2588 }
2589
2590 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2591 // See inline_native_Class_query for occurrences of these patterns.
2592 // Java Example: x.getClass().isAssignableFrom(y)
2593 //
2594 // This improves reflective code, often making the Class
2595 // mirror go completely dead. (Current exception: Class
2596 // mirrors may appear in debug info, but we could clean them out by
2597 // introducing a new debug info operator for Klass.java_mirror).
2598
2599 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2600 && offset == java_lang_Class::klass_offset()) {
2601 if (base->is_Load()) {
2602 Node* base2 = base->in(MemNode::Address);
2603 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2604 Node* adr2 = base2->in(MemNode::Address);
2605 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2606 if (tkls != nullptr && !tkls->empty()
2607 && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2608 && adr2->is_AddP()
2609 ) {
2610 int mirror_field = in_bytes(Klass::java_mirror_offset());
2611 if (tkls->offset() == mirror_field) {
2612 #ifdef ASSERT
2613 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2614 assert(tkls2->offset() == 0, "not a load of java_mirror");
2615 #endif
2616 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2617 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2618 return adr2->in(AddPNode::Address);
2619 }
2620 }
2621 }
2622 }
2623 }
2624
2625 return this;
2626 }
2627
2628 LoadNode* LoadNode::clone_pinned() const {
2629 LoadNode* ld = clone()->as_Load();
2755 //---------------------------StoreNode::make-----------------------------------
2756 // Polymorphic factory method:
2757 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2758 assert((mo == unordered || mo == release), "unexpected");
2759 Compile* C = gvn.C;
2760 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2761 ctl != nullptr, "raw memory operations should have control edge");
2762
2763 switch (bt) {
2764 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2765 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2766 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2767 case T_CHAR:
2768 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2769 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2770 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2771 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2772 case T_METADATA:
2773 case T_ADDRESS:
2774 case T_OBJECT:
2775 #ifdef _LP64
2776 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2777 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2778 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2779 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2780 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2781 adr->bottom_type()->isa_rawptr())) {
2782 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2783 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2784 }
2785 #endif
2786 {
2787 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2788 }
2789 default:
2790 ShouldNotReachHere();
2791 return (StoreNode*)nullptr;
2792 }
2793 }
2794
2795 //--------------------------bottom_type----------------------------------------
2796 const Type *StoreNode::bottom_type() const {
2797 return Type::MEMORY;
2798 }
2799
2800 //------------------------------hash-------------------------------------------
2801 uint StoreNode::hash() const {
2802 // unroll addition of interesting fields
2803 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2804
2805 // Since they are not commoned, do not hash them:
2806 return NO_HASH;
2807 }
2808
2809 // Link together multiple stores (B/S/C/I) into a longer one.
2810 //
3432 }
3433 ss.print_cr("[TraceMergeStores]: with");
3434 merged_input_value->dump("\n", false, &ss);
3435 merged_store->dump("\n", false, &ss);
3436 tty->print("%s", ss.as_string());
3437 }
3438 #endif
3439
3440 //------------------------------Ideal------------------------------------------
3441 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3442 // When a store immediately follows a relevant allocation/initialization,
3443 // try to capture it into the initialization, or hoist it above.
3444 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3445 Node* p = MemNode::Ideal_common(phase, can_reshape);
3446 if (p) return (p == NodeSentinel) ? nullptr : p;
3447
3448 Node* mem = in(MemNode::Memory);
3449 Node* address = in(MemNode::Address);
3450 Node* value = in(MemNode::ValueIn);
3451 // Back-to-back stores to same address? Fold em up. Generally
3452 // unsafe if I have intervening uses.
3453 {
3454 Node* st = mem;
3455 // If Store 'st' has more than one use, we cannot fold 'st' away.
3456 // For example, 'st' might be the final state at a conditional
3457 // return. Or, 'st' might be used by some node which is live at
3458 // the same time 'st' is live, which might be unschedulable. So,
3459 // require exactly ONE user until such time as we clone 'mem' for
3460 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3461 // true).
3462 while (st->is_Store() && st->outcnt() == 1) {
3463 // Looking at a dead closed cycle of memory?
3464 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3465 assert(Opcode() == st->Opcode() ||
3466 st->Opcode() == Op_StoreVector ||
3467 Opcode() == Op_StoreVector ||
3468 st->Opcode() == Op_StoreVectorScatter ||
3469 Opcode() == Op_StoreVectorScatter ||
3470 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3471 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3472 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3473 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3474 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3475
3476 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3477 st->as_Store()->memory_size() <= this->memory_size()) {
3478 Node* use = st->raw_out(0);
3479 if (phase->is_IterGVN()) {
3480 phase->is_IterGVN()->rehash_node_delayed(use);
3481 }
3482 // It's OK to do this in the parser, since DU info is always accurate,
3483 // and the parser always refers to nodes via SafePointNode maps.
3484 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3485 return this;
3486 }
3487 st = st->in(MemNode::Memory);
3488 }
3489 }
3490
3491
3492 // Capture an unaliased, unconditional, simple store into an initializer.
3593 const StoreVectorNode* store_vector = as_StoreVector();
3594 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3595 const Node* store_indices = store_vector->indices();
3596 const Node* mem_indices = mem_vector->indices();
3597 const Node* store_mask = store_vector->mask();
3598 const Node* mem_mask = mem_vector->mask();
3599 // Ensure types, indices, and masks match
3600 if (store_vector->vect_type() == mem_vector->vect_type() &&
3601 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3602 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3603 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3604 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3605 result = mem;
3606 }
3607 }
3608 }
3609
3610 // Store of zero anywhere into a freshly-allocated object?
3611 // Then the store is useless.
3612 // (It must already have been captured by the InitializeNode.)
3613 if (result == this &&
3614 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3615 // a newly allocated object is already all-zeroes everywhere
3616 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3617 result = mem;
3618 }
3619
3620 if (result == this) {
3621 // the store may also apply to zero-bits in an earlier object
3622 Node* prev_mem = find_previous_store(phase);
3623 // Steps (a), (b): Walk past independent stores to find an exact match.
3624 if (prev_mem != nullptr) {
3625 if (prev_mem->is_top()) {
3626 // find_previous_store returns top when the access is dead
3627 return prev_mem;
3628 }
3629 Node* prev_val = can_see_stored_value(prev_mem, phase);
3630 if (prev_val != nullptr && prev_val == val) {
3631 // prev_val and val might differ by a cast; it would be good
3632 // to keep the more informative of the two.
3633 result = mem;
3634 }
3635 }
3636 }
3637 }
3638
3639 PhaseIterGVN* igvn = phase->is_IterGVN();
3640 if (result != this && igvn != nullptr) {
4113 // Clearing a short array is faster with stores
4114 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4115 // Already know this is a large node, do not try to ideal it
4116 if (_is_large) return nullptr;
4117
4118 const int unit = BytesPerLong;
4119 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4120 if (!t) return nullptr;
4121 if (!t->is_con()) return nullptr;
4122 intptr_t raw_count = t->get_con();
4123 intptr_t size = raw_count;
4124 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4125 // Clearing nothing uses the Identity call.
4126 // Negative clears are possible on dead ClearArrays
4127 // (see jck test stmt114.stmt11402.val).
4128 if (size <= 0 || size % unit != 0) return nullptr;
4129 intptr_t count = size / unit;
4130 // Length too long; communicate this to matchers and assemblers.
4131 // Assemblers are responsible to produce fast hardware clears for it.
4132 if (size > InitArrayShortSize) {
4133 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4134 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4135 return nullptr;
4136 }
4137 if (!IdealizeClearArrayNode) return nullptr;
4138 Node *mem = in(1);
4139 if( phase->type(mem)==Type::TOP ) return nullptr;
4140 Node *adr = in(3);
4141 const Type* at = phase->type(adr);
4142 if( at==Type::TOP ) return nullptr;
4143 const TypePtr* atp = at->isa_ptr();
4144 // adjust atp to be the correct array element address type
4145 if (atp == nullptr) atp = TypePtr::BOTTOM;
4146 else atp = atp->add_offset(Type::OffsetBot);
4147 // Get base for derived pointer purposes
4148 if( adr->Opcode() != Op_AddP ) Unimplemented();
4149 Node *base = adr->in(1);
4150
4151 Node *zero = phase->makecon(TypeLong::ZERO);
4152 Node *off = phase->MakeConX(BytesPerLong);
4153 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4154 count--;
4155 while( count-- ) {
4156 mem = phase->transform(mem);
4157 adr = phase->transform(new AddPNode(base,adr,off));
4158 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4159 }
4160 return mem;
4161 }
4162
4163 //----------------------------step_through----------------------------------
4164 // Return allocation input memory edge if it is different instance
4165 // or itself if it is the one we are looking for.
4166 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4167 Node* n = *np;
4168 assert(n->is_ClearArray(), "sanity");
4169 intptr_t offset;
4170 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4171 // This method is called only before Allocate nodes are expanded
4172 // during macro nodes expansion. Before that ClearArray nodes are
4173 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4174 // Allocate nodes are expanded) which follows allocations.
4175 assert(alloc != nullptr, "should have allocation");
4176 if (alloc->_idx == instance_id) {
4177 // Can not bypass initialization of the instance we are looking for.
4178 return false;
4181 InitializeNode* init = alloc->initialization();
4182 if (init != nullptr)
4183 *np = init->in(TypeFunc::Memory);
4184 else
4185 *np = alloc->in(TypeFunc::Memory);
4186 return true;
4187 }
4188
4189 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4190 Node* base = dest;
4191 if (raw_base) {
4192 // May be called as part of the initialization of a just allocated object
4193 base = phase->C->top();
4194 }
4195 return phase->transform(new AddPNode(base, dest, offset));
4196 }
4197
4198 //----------------------------clear_memory-------------------------------------
4199 // Generate code to initialize object storage to zero.
4200 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4201 intptr_t start_offset,
4202 Node* end_offset,
4203 bool raw_base,
4204 PhaseGVN* phase) {
4205 intptr_t offset = start_offset;
4206
4207 int unit = BytesPerLong;
4208 if ((offset % unit) != 0) {
4209 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4210 const TypePtr* atp = TypeRawPtr::BOTTOM;
4211 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4212 mem = phase->transform(mem);
4213 offset += BytesPerInt;
4214 }
4215 assert((offset % unit) == 0, "");
4216
4217 // Initialize the remaining stuff, if any, with a ClearArray.
4218 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, raw_base, phase);
4219 }
4220
4221 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4222 Node* start_offset,
4223 Node* end_offset,
4224 bool raw_base,
4225 PhaseGVN* phase) {
4226 if (start_offset == end_offset) {
4227 // nothing to do
4228 return mem;
4229 }
4230
4231 int unit = BytesPerLong;
4232 Node* zbase = start_offset;
4233 Node* zend = end_offset;
4234
4235 // Scale to the unit required by the CPU:
4236 if (!Matcher::init_array_count_is_in_bytes) {
4237 Node* shift = phase->intcon(exact_log2(unit));
4238 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4239 zend = phase->transform(new URShiftXNode(zend, shift) );
4240 }
4241
4242 // Bulk clear double-words
4243 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4244 Node* adr = make_address(dest, start_offset, raw_base, phase);
4245 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4246 return phase->transform(mem);
4247 }
4248
4249 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4250 intptr_t start_offset,
4251 intptr_t end_offset,
4252 bool raw_base,
4253 PhaseGVN* phase) {
4254 if (start_offset == end_offset) {
4255 // nothing to do
4256 return mem;
4257 }
4258
4259 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4260 intptr_t done_offset = end_offset;
4261 if ((done_offset % BytesPerLong) != 0) {
4262 done_offset -= BytesPerInt;
4263 }
4264 if (done_offset > start_offset) {
4265 mem = clear_memory(ctl, mem, dest,
4266 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4267 }
4268 if (done_offset < end_offset) { // emit the final 32-bit store
4269 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4270 const TypePtr* atp = TypeRawPtr::BOTTOM;
4271 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4272 mem = phase->transform(mem);
4273 done_offset += BytesPerInt;
4274 }
4275 assert(done_offset == end_offset, "");
4276 return mem;
4277 }
4278
4279 //=============================================================================
4280 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4281 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4282 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4283 #ifdef ASSERT
4284 , _pair_idx(0)
4285 #endif
4286 {
4287 init_class_id(Class_MemBar);
4288 Node* top = C->top();
4289 init_req(TypeFunc::I_O,top);
4290 init_req(TypeFunc::FramePtr,top);
4291 init_req(TypeFunc::ReturnAdr,top);
4398 PhaseIterGVN* igvn = phase->is_IterGVN();
4399 remove(igvn);
4400 // Must return either the original node (now dead) or a new node
4401 // (Do not return a top here, since that would break the uniqueness of top.)
4402 return new ConINode(TypeInt::ZERO);
4403 }
4404 }
4405 return progress ? this : nullptr;
4406 }
4407
4408 //------------------------------Value------------------------------------------
4409 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4410 if( !in(0) ) return Type::TOP;
4411 if( phase->type(in(0)) == Type::TOP )
4412 return Type::TOP;
4413 return TypeTuple::MEMBAR;
4414 }
4415
4416 //------------------------------match------------------------------------------
4417 // Construct projections for memory.
4418 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4419 switch (proj->_con) {
4420 case TypeFunc::Control:
4421 case TypeFunc::Memory:
4422 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4423 }
4424 ShouldNotReachHere();
4425 return nullptr;
4426 }
4427
4428 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4429 trailing->_kind = TrailingStore;
4430 leading->_kind = LeadingStore;
4431 #ifdef ASSERT
4432 trailing->_pair_idx = leading->_idx;
4433 leading->_pair_idx = leading->_idx;
4434 #endif
4435 }
4436
4437 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4438 trailing->_kind = TrailingLoadStore;
4685 return (req() > RawStores);
4686 }
4687
4688 void InitializeNode::set_complete(PhaseGVN* phase) {
4689 assert(!is_complete(), "caller responsibility");
4690 _is_complete = Complete;
4691
4692 // After this node is complete, it contains a bunch of
4693 // raw-memory initializations. There is no need for
4694 // it to have anything to do with non-raw memory effects.
4695 // Therefore, tell all non-raw users to re-optimize themselves,
4696 // after skipping the memory effects of this initialization.
4697 PhaseIterGVN* igvn = phase->is_IterGVN();
4698 if (igvn) igvn->add_users_to_worklist(this);
4699 }
4700
4701 // convenience function
4702 // return false if the init contains any stores already
4703 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4704 InitializeNode* init = initialization();
4705 if (init == nullptr || init->is_complete()) return false;
4706 init->remove_extra_zeroes();
4707 // for now, if this allocation has already collected any inits, bail:
4708 if (init->is_non_zero()) return false;
4709 init->set_complete(phase);
4710 return true;
4711 }
4712
4713 void InitializeNode::remove_extra_zeroes() {
4714 if (req() == RawStores) return;
4715 Node* zmem = zero_memory();
4716 uint fill = RawStores;
4717 for (uint i = fill; i < req(); i++) {
4718 Node* n = in(i);
4719 if (n->is_top() || n == zmem) continue; // skip
4720 if (fill < i) set_req(fill, n); // compact
4721 ++fill;
4722 }
4723 // delete any empty spaces created:
4724 while (fill < req()) {
4725 del_req(fill);
4869 // store node that we'd like to capture. We need to check
4870 // the uses of the MergeMemNode.
4871 mems.push(n);
4872 }
4873 } else if (n->is_Mem()) {
4874 Node* other_adr = n->in(MemNode::Address);
4875 if (other_adr == adr) {
4876 failed = true;
4877 break;
4878 } else {
4879 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4880 if (other_t_adr != nullptr) {
4881 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4882 if (other_alias_idx == alias_idx) {
4883 // A load from the same memory slice as the store right
4884 // after the InitializeNode. We check the control of the
4885 // object/array that is loaded from. If it's the same as
4886 // the store control then we cannot capture the store.
4887 assert(!n->is_Store(), "2 stores to same slice on same control?");
4888 Node* base = other_adr;
4889 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4890 base = base->in(AddPNode::Base);
4891 if (base != nullptr) {
4892 base = base->uncast();
4893 if (base->is_Proj() && base->in(0) == alloc) {
4894 failed = true;
4895 break;
4896 }
4897 }
4898 }
4899 }
4900 }
4901 } else {
4902 failed = true;
4903 break;
4904 }
4905 }
4906 }
4907 }
4908 if (failed) {
5455 // z's_done 12 16 16 16 12 16 12
5456 // z's_needed 12 16 16 16 16 16 16
5457 // zsize 0 0 0 0 4 0 4
5458 if (next_full_store < 0) {
5459 // Conservative tack: Zero to end of current word.
5460 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5461 } else {
5462 // Zero to beginning of next fully initialized word.
5463 // Or, don't zero at all, if we are already in that word.
5464 assert(next_full_store >= zeroes_needed, "must go forward");
5465 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5466 zeroes_needed = next_full_store;
5467 }
5468 }
5469
5470 if (zeroes_needed > zeroes_done) {
5471 intptr_t zsize = zeroes_needed - zeroes_done;
5472 // Do some incremental zeroing on rawmem, in parallel with inits.
5473 zeroes_done = align_down(zeroes_done, BytesPerInt);
5474 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5475 zeroes_done, zeroes_needed,
5476 true,
5477 phase);
5478 zeroes_done = zeroes_needed;
5479 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5480 do_zeroing = false; // leave the hole, next time
5481 }
5482 }
5483
5484 // Collect the store and move on:
5485 phase->replace_input_of(st, MemNode::Memory, inits);
5486 inits = st; // put it on the linearized chain
5487 set_req(i, zmem); // unhook from previous position
5488
5489 if (zeroes_done == st_off)
5490 zeroes_done = next_init_off;
5491
5492 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5493
5494 #ifdef ASSERT
5515 remove_extra_zeroes(); // clear out all the zmems left over
5516 add_req(inits);
5517
5518 if (!(UseTLAB && ZeroTLAB)) {
5519 // If anything remains to be zeroed, zero it all now.
5520 zeroes_done = align_down(zeroes_done, BytesPerInt);
5521 // if it is the last unused 4 bytes of an instance, forget about it
5522 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5523 if (zeroes_done + BytesPerLong >= size_limit) {
5524 AllocateNode* alloc = allocation();
5525 assert(alloc != nullptr, "must be present");
5526 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5527 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5528 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5529 if (zeroes_done == k->layout_helper())
5530 zeroes_done = size_limit;
5531 }
5532 }
5533 if (zeroes_done < size_limit) {
5534 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5535 zeroes_done, size_in_bytes, true, phase);
5536 }
5537 }
5538
5539 set_complete(phase);
5540 return rawmem;
5541 }
5542
5543 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5544 auto replace_proj = [&](ProjNode* proj) {
5545 C->gvn_replace_by(proj, mem);
5546 return CONTINUE;
5547 };
5548 apply_to_projs(replace_proj, TypeFunc::Memory);
5549 }
5550
5551 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5552 DUIterator_Fast imax, i = fast_outs(imax);
5553 auto replace_proj = [&](ProjNode* proj) {
5554 igvn->replace_node(proj, mem);
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "ci/ciInlineKlass.hpp"
28 #include "classfile/javaClasses.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "compiler/compileLog.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/c2/barrierSetC2.hpp"
33 #include "gc/shared/tlab_globals.hpp"
34 #include "memory/allocation.inline.hpp"
35 #include "memory/resourceArea.hpp"
36 #include "oops/flatArrayKlass.hpp"
37 #include "oops/objArrayKlass.hpp"
38 #include "opto/addnode.hpp"
39 #include "opto/arraycopynode.hpp"
40 #include "opto/callnode.hpp"
41 #include "opto/cfgnode.hpp"
42 #include "opto/compile.hpp"
43 #include "opto/connode.hpp"
44 #include "opto/convertnode.hpp"
45 #include "opto/inlinetypenode.hpp"
46 #include "opto/loopnode.hpp"
47 #include "opto/machnode.hpp"
48 #include "opto/matcher.hpp"
49 #include "opto/memnode.hpp"
50 #include "opto/mempointer.hpp"
51 #include "opto/mulnode.hpp"
52 #include "opto/narrowptrnode.hpp"
53 #include "opto/opcodes.hpp"
54 #include "opto/phaseX.hpp"
55 #include "opto/regalloc.hpp"
56 #include "opto/regmask.hpp"
57 #include "opto/rootnode.hpp"
58 #include "opto/traceMergeStoresTag.hpp"
59 #include "opto/vectornode.hpp"
60 #include "runtime/arguments.hpp"
61 #include "utilities/align.hpp"
62 #include "utilities/copy.hpp"
63 #include "utilities/globalDefinitions.hpp"
64 #include "utilities/macros.hpp"
65 #include "utilities/powerOfTwo.hpp"
66 #include "utilities/vmError.hpp"
67
68 // Portions of code courtesy of Clifford Click
69
70 // Optimization - Graph Style
71
72 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
73
74 //=============================================================================
75 uint MemNode::size_of() const { return sizeof(*this); }
76
77 const TypePtr *MemNode::adr_type() const {
78 Node* adr = in(Address);
79 if (adr == nullptr) return nullptr; // node is dead
80 const TypePtr* cross_check = nullptr;
81 DEBUG_ONLY(cross_check = _adr_type);
82 return calculate_adr_type(adr->bottom_type(), cross_check);
83 }
132 st->print(", idx=Bot;");
133 else if (atp->index() == Compile::AliasIdxTop)
134 st->print(", idx=Top;");
135 else if (atp->index() == Compile::AliasIdxRaw)
136 st->print(", idx=Raw;");
137 else {
138 ciField* field = atp->field();
139 if (field) {
140 st->print(", name=");
141 field->print_name_on(st);
142 }
143 st->print(", idx=%d;", atp->index());
144 }
145 }
146 }
147
148 extern void print_alias_types();
149
150 #endif
151
152 // Find the memory output corresponding to the fall-through path of a call
153 static Node* find_call_fallthrough_mem_output(CallNode* call) {
154 ResourceMark rm;
155 CallProjections* projs = call->extract_projections(false, false);
156 Node* res = projs->fallthrough_memproj;
157 assert(res != nullptr, "must have a fallthrough mem output");
158 return res;
159 }
160
161 // Try to find a better memory input for a load from a strict final field
162 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
163 intptr_t offset = 0;
164 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
165 if (base == nullptr) {
166 return nullptr;
167 }
168
169 Node* base_uncasted = base->uncast();
170 if (base_uncasted->is_Proj()) {
171 Node* multi = base_uncasted->in(0);
172 if (multi->is_top()) {
173 // The pointer dies, make the memory die, too
174 return multi;
175 } else if (multi->is_Allocate()) {
176 base_local = base_uncasted->as_Proj();
177 return nullptr;
178 } else if (multi->is_Call()) {
179 // The oop is returned from a call, the memory can be the fallthrough output of the call
180 return find_call_fallthrough_mem_output(multi->as_Call());
181 } else if (multi->is_Start()) {
182 // The oop is a parameter
183 if (phase->C->method()->is_object_constructor() && base_uncasted->as_Proj()->_con == TypeFunc::Parms) {
184 // The receiver of a constructor is similar to the result of an AllocateNode
185 base_local = base_uncasted->as_Proj();
186 return nullptr;
187 } else {
188 // Use the start memory otherwise
189 return multi->as_Start()->proj_out(TypeFunc::Memory);
190 }
191 }
192 }
193
194 return nullptr;
195 }
196
197 // Whether a call can modify a strict final field, given that the object is allocated inside the
198 // current compilation unit, or is the first parameter when the compilation root is a constructor.
199 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
200 // the target method is a subclass of the class declaring 'field'.
201 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
202 if (!call->is_CallJava()) {
203 return false;
204 }
205
206 ciMethod* target = call->as_CallJava()->method();
207 if (target == nullptr || !target->is_object_constructor()) {
208 return false;
209 }
210
211 // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
212 // then the field is set inside the constructor, else the field must be set before the
213 // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
214 // while a field Sub.y must be set before Super::<init> is invoked.
215 // We can try to be more heroic and decide if the receiver of the constructor invocation is the
216 // object from which we are loading from. This, however, may be problematic as deciding if 2
217 // nodes are definitely different may not be trivial, especially if the graph is not canonical.
218 // As a result, it is made more conservative for now.
219 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
220 return target->holder()->is_subclass_of(field->holder());
221 }
222
223 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
224 assert(t_oop != nullptr, "sanity");
225 bool is_known_instance = t_oop->is_known_instance_field();
226 bool is_strict_final_load = false;
227
228 // After macro expansion, an allocation may become a call, changing the memory input to the
229 // memory output of that call would be illegal. As a result, disallow this transformation after
230 // macro expansion.
231 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
232 is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
233 #ifdef ASSERT
234 if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
235 assert(is_strict_final_load, "sanity check for basic cases");
236 }
237 #endif // ASSERT
238 }
239
240 if (!is_known_instance && !is_strict_final_load) {
241 return mchain;
242 }
243
244 Node* result = mchain;
245 ProjNode* base_local = nullptr;
246
247 ciField* field = nullptr;
248 if (is_strict_final_load) {
249 field = phase->C->alias_type(t_oop)->field();
250 assert(field != nullptr, "must point to a field");
251
252 Node* adr = load->in(MemNode::Address);
253 assert(phase->type(adr) == t_oop, "inconsistent type");
254 Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
255 if (tmp != nullptr) {
256 result = tmp;
257 }
258 }
259
260 uint instance_id = t_oop->instance_id();
261 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
262 Node* prev = nullptr;
263 while (prev != result) {
264 prev = result;
265 if (result == start_mem) {
266 // start_mem is the earliest memory possible
267 break;
268 }
269
270 // skip over a call which does not affect this memory slice
271 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
272 Node* proj_in = result->in(0);
273 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
274 // This is the allocation that creates the object from which we are loading from
275 break;
276 } else if (proj_in->is_Call()) {
277 // ArrayCopyNodes processed here as well
278 CallNode* call = proj_in->as_Call();
279 if (!call->may_modify(t_oop, phase)) {
280 result = call->in(TypeFunc::Memory);
281 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
282 result = call->in(TypeFunc::Memory);
283 }
284 } else if (proj_in->Opcode() == Op_Tuple) {
285 // The call will be folded, skip over it.
286 break;
287 } else if (proj_in->is_Initialize()) {
288 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
289 // Stop if this is the initialization for the object instance which
290 // contains this memory slice, otherwise skip over it.
291 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
292 break;
293 }
294 if (is_known_instance) {
295 result = proj_in->in(TypeFunc::Memory);
296 } else if (is_strict_final_load) {
297 Node* klass = alloc->in(AllocateNode::KlassNode);
298 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
299 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
300 // Allocation of another type, must be another object
301 result = proj_in->in(TypeFunc::Memory);
302 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
303 // Allocation of another object
304 result = proj_in->in(TypeFunc::Memory);
305 }
306 }
307 } else if (proj_in->is_MemBar()) {
308 ArrayCopyNode* ac = nullptr;
309 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
310 break;
311 }
312 result = proj_in->in(TypeFunc::Memory);
313 } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
314 if (is_strict_final_load) {
315 // LoadFlat and StoreFlat cannot happen to strict final fields
316 result = proj_in->in(TypeFunc::Memory);
317 }
318 } else if (proj_in->is_top()) {
319 break; // dead code
320 } else {
321 assert(false, "unexpected projection of %s", proj_in->Name());
322 }
323 } else if (result->is_ClearArray()) {
324 if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
325 // Can not bypass initialization of the instance
326 // we are looking for.
327 break;
328 }
329 // Otherwise skip it (the call updated 'result' value).
330 } else if (result->is_MergeMem()) {
331 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
332 }
333 }
334 return result;
335 }
336
337 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
338 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
339 if (t_oop == nullptr)
340 return mchain; // don't try to optimize non-oop types
341 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
342 bool is_instance = t_oop->is_known_instance_field();
343 PhaseIterGVN *igvn = phase->is_IterGVN();
344 if (is_instance && igvn != nullptr && result->is_Phi()) {
345 PhiNode *mphi = result->as_Phi();
346 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
347 const TypePtr *t = mphi->adr_type();
348 bool do_split = false;
349 // In the following cases, Load memory input can be further optimized based on
350 // its precise address type
351 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
352 do_split = true;
353 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
354 const TypeOopPtr* mem_t =
355 t->is_oopptr()->cast_to_exactness(true)
356 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
357 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
358 if (t_oop->isa_aryptr()) {
359 mem_t = mem_t->is_aryptr()
360 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
361 ->cast_to_size(t_oop->is_aryptr()->size())
362 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
363 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
364 ->with_offset(t_oop->is_aryptr()->offset())
365 ->is_aryptr();
366 }
367 do_split = mem_t == t_oop;
368 }
369 if (do_split) {
370 // clone the Phi with our address type
371 result = mphi->split_out_instance(t_adr, igvn);
372 } else {
373 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
374 }
375 }
376 return result;
377 }
378
379 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
380 uint alias_idx = phase->C->get_alias_index(tp);
381 Node *mem = mmem;
382 #ifdef ASSERT
383 {
384 // Check that current type is consistent with the alias index used during graph construction
385 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
386 bool consistent = adr_check == nullptr || adr_check->empty() ||
387 phase->C->must_alias(adr_check, alias_idx );
388 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
389 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
390 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
391 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
392 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
393 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
394 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
395 // don't assert if it is dead code.
396 consistent = true;
397 }
398 if( !consistent ) {
399 st->print("alias_idx==%d, adr_check==", alias_idx);
400 if( adr_check == nullptr ) {
401 st->print("null");
402 } else {
403 adr_check->dump();
404 }
405 st->cr();
406 print_alias_types();
407 assert(consistent, "adr_check must match alias idx");
408 }
409 }
410 #endif
1123 "use LoadKlassNode instead");
1124 assert(!(adr_type->isa_aryptr() &&
1125 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1126 "use LoadRangeNode instead");
1127 // Check control edge of raw loads
1128 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1129 // oop will be recorded in oop map if load crosses safepoint
1130 rt->isa_oopptr() || is_immutable_value(adr),
1131 "raw memory operations should have control edge");
1132 LoadNode* load = nullptr;
1133 switch (bt) {
1134 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1135 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1136 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1137 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1138 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1139 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1140 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1141 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1142 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1143 case T_ARRAY:
1144 case T_OBJECT:
1145 case T_NARROWOOP:
1146 #ifdef _LP64
1147 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1148 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1149 } else
1150 #endif
1151 {
1152 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1153 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1154 }
1155 break;
1156 default:
1157 guarantee(false, "unexpected basic type %s", type2name(bt));
1158 break;
1159 }
1160 assert(load != nullptr, "LoadNode should have been created");
1161 if (unaligned) {
1162 load->set_unaligned_access();
1163 }
1164 if (mismatched) {
1165 load->set_mismatched_access();
1166 }
1167 if (unsafe) {
1168 load->set_unsafe_access();
1169 }
1170 load->set_barrier_data(barrier_data);
1171 if (load->Opcode() == Op_LoadN) {
1172 Node* ld = gvn.transform(load);
1173 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1174 }
1175
1176 return load;
1177 }
1178
1179 //------------------------------hash-------------------------------------------
1180 uint LoadNode::hash() const {
1181 // unroll addition of interesting fields
1182 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1183 }
1184
1185 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1186 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1187 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1188 bool is_stable_ary = FoldStableValues &&
1189 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1190 tp->isa_aryptr()->is_stable();
1191
1192 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1193 }
1194
1195 return false;
1196 }
1197
1198 // Is the value loaded previously stored by an arraycopy? If so return
1199 // a load node that reads from the source array so we may be able to
1200 // optimize out the ArrayCopy node later.
1201 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1202 Node* ld_adr = in(MemNode::Address);
1203 intptr_t ld_off = 0;
1204 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1205 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1206 if (ac != nullptr) {
1207 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1208
1209 Node* mem = ac->in(TypeFunc::Memory);
1210 Node* ctl = ac->in(0);
1211 Node* src = ac->in(ArrayCopyNode::Src);
1212
1220 if (ac->as_ArrayCopy()->is_clonebasic()) {
1221 assert(ld_alloc != nullptr, "need an alloc");
1222 assert(addp->is_AddP(), "address must be addp");
1223 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1224 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1225 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1226 addp->set_req(AddPNode::Base, src);
1227 addp->set_req(AddPNode::Address, src);
1228 } else {
1229 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1230 ac->as_ArrayCopy()->is_copyof_validated() ||
1231 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1232 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1233 addp->set_req(AddPNode::Base, src);
1234 addp->set_req(AddPNode::Address, src);
1235
1236 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1237 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1238 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1239
1240 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1241
1242 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1243 #ifdef _LP64
1244 diff = phase->transform(new ConvI2LNode(diff));
1245 #endif
1246 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1247
1248 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1249 addp->set_req(AddPNode::Offset, offset);
1250 }
1251 addp = phase->transform(addp);
1252 #ifdef ASSERT
1253 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1254 ld->_adr_type = adr_type;
1255 #endif
1256 ld->set_req(MemNode::Address, addp);
1257 ld->set_req(0, ctl);
1258 ld->set_req(MemNode::Memory, mem);
1259 return ld;
1260 }
1261 return nullptr;
1262 }
1263
1264 static Node* see_through_inline_type(PhaseValues* phase, const MemNode* load, Node* base, int offset) {
1265 if (!load->is_mismatched_access() && base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1266 InlineTypeNode* vt = base->as_InlineType();
1267 Node* value = vt->field_value_by_offset(offset, true);
1268 assert(value != nullptr, "must see some value");
1269 return value;
1270 }
1271
1272 return nullptr;
1273 }
1274
1275 //---------------------------can_see_stored_value------------------------------
1276 // This routine exists to make sure this set of tests is done the same
1277 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1278 // will change the graph shape in a way which makes memory alive twice at the
1279 // same time (uses the Oracle model of aliasing), then some
1280 // LoadXNode::Identity will fold things back to the equivalence-class model
1281 // of aliasing.
1282 // This method may find an unencoded node instead of the corresponding encoded one.
1283 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1284 Node* ld_adr = in(MemNode::Address);
1285 intptr_t ld_off = 0;
1286 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1287 // Try to see through an InlineTypeNode
1288 // LoadN is special because the input is not compressed
1289 if (Opcode() != Op_LoadN) {
1290 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1291 if (value != nullptr) {
1292 return value;
1293 }
1294 }
1295
1296 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1297 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1298 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1299 // This is more general than load from boxing objects.
1300 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1301 uint alias_idx = atp->index();
1302 Node* result = nullptr;
1303 Node* current = st;
1304 // Skip through chains of MemBarNodes checking the MergeMems for
1305 // new states for the slice of this load. Stop once any other
1306 // kind of node is encountered. Loads from final memory can skip
1307 // through any kind of MemBar but normal loads shouldn't skip
1308 // through MemBarAcquire since the could allow them to move out of
1309 // a synchronized region. It is not safe to step over MemBarCPUOrder,
1310 // because alias info above them may be inaccurate (e.g., due to
1311 // mixed/mismatched unsafe accesses).
1312 bool is_final_mem = !atp->is_rewritable();
1313 while (current->is_Proj()) {
1314 int opc = current->in(0)->Opcode();
1315 if ((is_final_mem && (opc == Op_MemBarAcquire ||
1382
1383 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1384 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1385 if (in_vt != out_vt) {
1386 return nullptr;
1387 }
1388 }
1389 return st->in(MemNode::ValueIn);
1390 }
1391
1392 // A load from a freshly-created object always returns zero.
1393 // (This can happen after LoadNode::Ideal resets the load's memory input
1394 // to find_captured_store, which returned InitializeNode::zero_memory.)
1395 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1396 (st->in(0) == ld_alloc) &&
1397 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1398 // return a zero value for the load's basic type
1399 // (This is one of the few places where a generic PhaseTransform
1400 // can create new nodes. Think of it as lazily manifesting
1401 // virtually pre-existing constants.)
1402 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1403 if (init_value != nullptr) {
1404 const TypeAryPtr* ld_adr_type = phase->type(ld_adr)->isa_aryptr();
1405 if (ld_adr_type == nullptr) {
1406 return nullptr;
1407 }
1408
1409 // We know that this is not a flat array, the load should return the whole oop
1410 if (ld_adr_type->is_not_flat()) {
1411 return init_value;
1412 }
1413
1414 // If this is a flat array, try to see through init_value
1415 if (init_value->is_EncodeP()) {
1416 init_value = init_value->in(1);
1417 }
1418 if (!init_value->is_InlineType() || ld_adr_type->field_offset() == Type::Offset::bottom) {
1419 return nullptr;
1420 }
1421
1422 ciInlineKlass* vk = phase->type(init_value)->inline_klass();
1423 int field_offset_in_payload = ld_adr_type->field_offset().get();
1424 if (field_offset_in_payload == vk->null_marker_offset_in_payload()) {
1425 return init_value->as_InlineType()->get_null_marker();
1426 } else {
1427 return init_value->as_InlineType()->field_value_by_offset(field_offset_in_payload + vk->payload_offset(), true);
1428 }
1429 }
1430 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1431 if (value_basic_type() != T_VOID) {
1432 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1433 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1434 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1435 // by the ArrayCopyNode.
1436 return phase->zerocon(value_basic_type());
1437 }
1438 } else {
1439 // TODO: materialize all-zero vector constant
1440 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1441 }
1442 }
1443
1444 // A load from an initialization barrier can match a captured store.
1445 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1446 InitializeNode* init = st->in(0)->as_Initialize();
1447 AllocateNode* alloc = init->allocation();
1448 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1449 // examine a captured store value
1450 st = init->find_captured_store(ld_off, memory_size(), phase);
1463 base = bs->step_over_gc_barrier(base);
1464 if (base != nullptr && base->is_Proj() &&
1465 base->as_Proj()->_con == TypeFunc::Parms &&
1466 base->in(0)->is_CallStaticJava() &&
1467 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1468 return base->in(0)->in(TypeFunc::Parms);
1469 }
1470 }
1471
1472 break;
1473 }
1474
1475 return nullptr;
1476 }
1477
1478 //----------------------is_instance_field_load_with_local_phi------------------
1479 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1480 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1481 in(Address)->is_AddP() ) {
1482 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1483 // Only known instances and immutable fields
1484 if( t_oop != nullptr &&
1485 (t_oop->is_ptr_to_strict_final_field() ||
1486 t_oop->is_known_instance_field()) &&
1487 t_oop->offset() != Type::OffsetBot &&
1488 t_oop->offset() != Type::OffsetTop) {
1489 return true;
1490 }
1491 }
1492 return false;
1493 }
1494
1495 //------------------------------Identity---------------------------------------
1496 // Loads are identity if previous store is to same address
1497 Node* LoadNode::Identity(PhaseGVN* phase) {
1498 // If the previous store-maker is the right kind of Store, and the store is
1499 // to the same address, then we are equal to the value stored.
1500 Node* mem = in(Memory);
1501 Node* value = can_see_stored_value(mem, phase);
1502 if( value ) {
1503 // byte, short & char stores truncate naturally.
1504 // A load has to load the truncated value which requires
1505 // some sort of masking operation and that requires an
1506 // Ideal call instead of an Identity call.
1507 if (memory_size() < BytesPerInt) {
1508 // If the input to the store does not fit with the load's result type,
1509 // it must be truncated via an Ideal call.
1510 if (!phase->type(value)->higher_equal(phase->type(this)))
1511 return this;
1512 }
1513
1514 if (phase->type(value)->isa_ptr() && phase->type(this)->isa_narrowoop()) {
1515 return this;
1516 }
1517 // (This works even when value is a Con, but LoadNode::Value
1518 // usually runs first, producing the singleton type of the Con.)
1519 if (!has_pinned_control_dependency() || value->is_Con()) {
1520 return value;
1521 } else {
1522 return this;
1523 }
1524 }
1525
1526 if (has_pinned_control_dependency()) {
1527 return this;
1528 }
1529 // Search for an existing data phi which was generated before for the same
1530 // instance's field to avoid infinite generation of phis in a loop.
1531 Node *region = mem->in(0);
1532 if (is_instance_field_load_with_local_phi(region)) {
1533 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1534 int this_index = phase->C->get_alias_index(addr_t);
1535 int this_offset = addr_t->offset();
1536 int this_iid = addr_t->instance_id();
1537 if (!addr_t->is_known_instance() &&
1538 addr_t->is_ptr_to_strict_final_field()) {
1539 // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1540 intptr_t ignore = 0;
1541 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1542 if (base == nullptr) {
1543 return this;
1544 }
1545 this_iid = base->_idx;
1546 }
1547 const Type* this_type = bottom_type();
1548 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1549 Node* phi = region->fast_out(i);
1550 if (phi->is_Phi() && phi != mem &&
1551 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1552 return phi;
1553 }
1554 }
1555 }
1556
1557 return this;
1558 }
1559
2074 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2075 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2076
2077 // Skip up past a SafePoint control. Cannot do this for Stores because
2078 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2079 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2080 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2081 !addr_mark &&
2082 (depends_only_on_test() || has_unknown_control_dependency())) {
2083 ctrl = ctrl->in(0);
2084 set_req(MemNode::Control,ctrl);
2085 return this;
2086 }
2087
2088 intptr_t ignore = 0;
2089 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2090 if (base != nullptr
2091 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2092 // Check for useless control edge in some common special cases
2093 if (in(MemNode::Control) != nullptr
2094 // TODO 8350865 Can we re-enable this?
2095 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2096 && can_remove_control()
2097 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2098 && all_controls_dominate(base, phase->C->start())) {
2099 // A method-invariant, non-null address (constant or 'this' argument).
2100 set_req(MemNode::Control, nullptr);
2101 return this;
2102 }
2103 }
2104
2105 Node* mem = in(MemNode::Memory);
2106 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2107
2108 if (can_reshape && (addr_t != nullptr)) {
2109 // try to optimize our memory input
2110 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2111 if (opt_mem != mem) {
2112 set_req_X(MemNode::Memory, opt_mem, phase);
2113 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2114 return this;
2115 }
2228 // No match.
2229 return nullptr;
2230 }
2231
2232 //------------------------------Value-----------------------------------------
2233 const Type* LoadNode::Value(PhaseGVN* phase) const {
2234 // Either input is TOP ==> the result is TOP
2235 Node* mem = in(MemNode::Memory);
2236 const Type *t1 = phase->type(mem);
2237 if (t1 == Type::TOP) return Type::TOP;
2238 Node* adr = in(MemNode::Address);
2239 const TypePtr* tp = phase->type(adr)->isa_ptr();
2240 if (tp == nullptr || tp->empty()) return Type::TOP;
2241 int off = tp->offset();
2242 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2243 Compile* C = phase->C;
2244
2245 // If load can see a previous constant store, use that.
2246 Node* value = can_see_stored_value(mem, phase);
2247 if (value != nullptr && value->is_Con()) {
2248 if (phase->type(value)->isa_ptr() && _type->isa_narrowoop()) {
2249 return phase->type(value)->make_narrowoop();
2250 } else {
2251 assert(value->bottom_type()->higher_equal(_type), "sanity");
2252 return phase->type(value);
2253 }
2254 }
2255 // Try to guess loaded type from pointer type
2256 if (tp->isa_aryptr()) {
2257 const TypeAryPtr* ary = tp->is_aryptr();
2258 const Type* t = ary->elem();
2259
2260 // Determine whether the reference is beyond the header or not, by comparing
2261 // the offset against the offset of the start of the array's data.
2262 // Different array types begin at slightly different offsets (12 vs. 16).
2263 // We choose T_BYTE as an example base type that is least restrictive
2264 // as to alignment, which will therefore produce the smallest
2265 // possible base offset.
2266 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2267 const bool off_beyond_header = (off >= min_base_off);
2268
2269 // Try to constant-fold a stable array element.
2270 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2271 // Make sure the reference is not into the header and the offset is constant
2272 ciObject* aobj = ary->const_oop();
2273 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2274 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2275 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off, ary->field_offset().get(),
2276 stable_dimension,
2277 value_basic_type(), is_unsigned());
2278 if (con_type != nullptr) {
2279 return con_type;
2280 }
2281 }
2282 }
2283
2284 // Don't do this for integer types. There is only potential profit if
2285 // the element type t is lower than _type; that is, for int types, if _type is
2286 // more restrictive than t. This only happens here if one is short and the other
2287 // char (both 16 bits), and in those cases we've made an intentional decision
2288 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2289 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2290 //
2291 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2292 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2293 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2294 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2295 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2296 // In fact, that could have been the original type of p1, and p1 could have
2297 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2298 // expression (LShiftL quux 3) independently optimized to the constant 8.
2299 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2300 && (_type->isa_vect() == nullptr)
2301 && !ary->is_flat()
2302 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2303 // t might actually be lower than _type, if _type is a unique
2304 // concrete subclass of abstract class t.
2305 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2306 const Type* jt = t->join_speculative(_type);
2307 // In any case, do not allow the join, per se, to empty out the type.
2308 if (jt->empty() && !t->empty()) {
2309 // This can happen if a interface-typed array narrows to a class type.
2310 jt = _type;
2311 }
2312 #ifdef ASSERT
2313 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2314 // The pointers in the autobox arrays are always non-null
2315 Node* base = adr->in(AddPNode::Base);
2316 if ((base != nullptr) && base->is_DecodeN()) {
2317 // Get LoadN node which loads IntegerCache.cache field
2318 base = base->in(1);
2319 }
2320 if ((base != nullptr) && base->is_Con()) {
2321 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2322 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2323 // It could be narrow oop
2324 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2325 }
2326 }
2327 }
2328 #endif
2329 return jt;
2330 }
2331 }
2332 } else if (tp->base() == Type::InstPtr) {
2333 assert( off != Type::OffsetBot ||
2334 // arrays can be cast to Objects
2335 !tp->isa_instptr() ||
2336 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2337 // Default value load
2338 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2339 // unsafe field access may not have a constant offset
2340 C->has_unsafe_access(),
2341 "Field accesses must be precise" );
2342 // For oop loads, we expect the _type to be precise.
2343
2344 const TypeInstPtr* tinst = tp->is_instptr();
2345 BasicType bt = value_basic_type();
2346
2347 // Fold loads of the field map
2348 if (tinst != nullptr) {
2349 ciInstanceKlass* ik = tinst->instance_klass();
2350 int offset = tinst->offset();
2351 if (ik == phase->C->env()->Class_klass()) {
2352 ciType* t = tinst->java_mirror_type();
2353 if (t != nullptr && t->is_inlinetype() && offset == t->as_inline_klass()->field_map_offset()) {
2354 ciConstant map = t->as_inline_klass()->get_field_map();
2355 bool is_narrow_oop = (bt == T_NARROWOOP);
2356 return Type::make_from_constant(map, true, 1, is_narrow_oop);
2357 }
2358 }
2359 }
2360
2361 // Optimize loads from constant fields.
2362 ciObject* const_oop = tinst->const_oop();
2363 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2364 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2365 if (con_type != nullptr) {
2366 return con_type;
2367 }
2368 }
2369 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2370 assert(off != Type::OffsetBot ||
2371 !tp->isa_instklassptr() ||
2372 // arrays can be cast to Objects
2373 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2374 // also allow array-loading from the primary supertype
2375 // array during subtype checks
2376 Opcode() == Op_LoadKlass,
2377 "Field accesses must be precise");
2378 // For klass/static loads, we expect the _type to be precise
2379 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2380 /* With mirrors being an indirect in the Klass*
2381 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2382 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2383 *
2384 * So check the type and klass of the node before the LoadP.
2391 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2392 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2393 return TypeInstPtr::make(klass->java_mirror());
2394 }
2395 }
2396 }
2397
2398 const TypeKlassPtr *tkls = tp->isa_klassptr();
2399 if (tkls != nullptr) {
2400 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2401 ciKlass* klass = tkls->exact_klass();
2402 // We are loading a field from a Klass metaobject whose identity
2403 // is known at compile time (the type is "exact" or "precise").
2404 // Check for fields we know are maintained as constants by the VM.
2405 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2406 // The field is Klass::_super_check_offset. Return its (constant) value.
2407 // (Folds up type checking code.)
2408 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2409 return TypeInt::make(klass->super_check_offset());
2410 }
2411 if (klass->is_inlinetype() && tkls->offset() == in_bytes(InstanceKlass::acmp_maps_offset_offset())) {
2412 return TypeInt::make(klass->as_inline_klass()->field_map_offset());
2413 }
2414 if (klass->is_obj_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset())) {
2415 // Fold loads from LibraryCallKit::load_default_refined_array_klass
2416 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr();
2417 }
2418 if (klass->is_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::properties_offset())) {
2419 assert(klass->is_type_array_klass() || tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2420 return TypeInt::make((jint)klass->as_array_klass()->properties().value());
2421 }
2422 if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2423 assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2424 return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2425 }
2426 if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2427 // The field is Klass::_prototype_header. Return its (constant) value.
2428 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2429 return TypeX::make(klass->prototype_header());
2430 }
2431 // Compute index into primary_supers array
2432 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2433 // Check for overflowing; use unsigned compare to handle the negative case.
2434 if( depth < ciKlass::primary_super_limit() ) {
2435 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2436 // (Folds up type checking code.)
2437 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2438 ciKlass *ss = klass->super_of_depth(depth);
2439 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2440 }
2441 const Type* aift = load_array_final_field(tkls, klass);
2442 if (aift != nullptr) return aift;
2443 }
2444
2445 // We can still check if we are loading from the primary_supers array at a
2446 // shallow enough depth. Even though the klass is not exact, entries less
2447 // than or equal to its super depth are correct.
2448 if (tkls->is_loaded()) {
2449 ciKlass* klass = nullptr;
2483 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2484 // The key property of this type is that it folds up tests
2485 // for array-ness, since it proves that the layout_helper is positive.
2486 // Thus, a generic value like the basic object layout helper works fine.
2487 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2488 }
2489 }
2490
2491 // If we are loading from a freshly-allocated object/array, produce a zero.
2492 // Things to check:
2493 // 1. Load is beyond the header: headers are not guaranteed to be zero
2494 // 2. Load is not vectorized: vectors have no zero constant
2495 // 3. Load has no matching store, i.e. the input is the initial memory state
2496 const TypeOopPtr* tinst = tp->isa_oopptr();
2497 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2498 bool is_not_vect = (_type->isa_vect() == nullptr);
2499 if (is_not_header && is_not_vect) {
2500 Node* mem = in(MemNode::Memory);
2501 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2502 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2503 // TODO 8350865 Scalar replacement does not work well for flat arrays.
2504 // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2505 // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2506 // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2507 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2508 intptr_t offset = 0;
2509 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2510 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2511 if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2512 return _type;
2513 }
2514 }
2515 return Type::get_zero_type(_type->basic_type());
2516 }
2517 }
2518 if (!UseCompactObjectHeaders) {
2519 Node* alloc = is_new_object_mark_load();
2520 if (alloc != nullptr) {
2521 if (Arguments::is_valhalla_enabled()) {
2522 // The mark word may contain property bits (inline, flat, null-free)
2523 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2524 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2525 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2526 return TypeX::make(tkls->exact_klass()->prototype_header());
2527 }
2528 } else {
2529 return TypeX::make(markWord::prototype().value());
2530 }
2531 }
2532 }
2533
2534 return _type;
2535 }
2536
2537 //------------------------------match_edge-------------------------------------
2538 // Do we Match on this edge index or not? Match only the address.
2539 uint LoadNode::match_edge(uint idx) const {
2540 return idx == MemNode::Address;
2541 }
2542
2543 //--------------------------LoadBNode::Ideal--------------------------------------
2544 //
2545 // If the previous store is to the same address as this load,
2546 // and the value stored was larger than a byte, replace this load
2547 // with the value stored truncated to a byte. If no truncation is
2548 // needed, the replacement is done in LoadNode::Identity().
2549 //
2550 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2659 }
2660 }
2661 // Identity call will handle the case where truncation is not needed.
2662 return LoadNode::Ideal(phase, can_reshape);
2663 }
2664
2665 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2666 Node* mem = in(MemNode::Memory);
2667 Node* value = can_see_stored_value(mem,phase);
2668 if (value != nullptr && value->is_Con() &&
2669 !value->bottom_type()->higher_equal(_type)) {
2670 // If the input to the store does not fit with the load's result type,
2671 // it must be truncated. We can't delay until Ideal call since
2672 // a singleton Value is needed for split_thru_phi optimization.
2673 int con = value->get_int();
2674 return TypeInt::make((con << 16) >> 16);
2675 }
2676 return LoadNode::Value(phase);
2677 }
2678
2679 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2680 // Loading from an InlineType, find the input and make an EncodeP
2681 Node* addr = in(Address);
2682 intptr_t offset;
2683 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
2684 Node* value = see_through_inline_type(phase, this, base, offset);
2685 if (value != nullptr) {
2686 return new EncodePNode(value, type());
2687 }
2688
2689 // Can see the corresponding value, may need to add an EncodeP
2690 value = can_see_stored_value(in(Memory), phase);
2691 if (value != nullptr && phase->type(value)->isa_ptr() && type()->isa_narrowoop()) {
2692 return new EncodePNode(value, type());
2693 }
2694
2695 // Identity call will handle the case where EncodeP is unnecessary
2696 return LoadNode::Ideal(phase, can_reshape);
2697 }
2698
2699 //=============================================================================
2700 //----------------------------LoadKlassNode::make------------------------------
2701 // Polymorphic factory method:
2702 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2703 // sanity check the alias category against the created node type
2704 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2705 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2706 #ifdef _LP64
2707 if (adr_type->is_ptr_to_narrowklass()) {
2708 assert(UseCompressedClassPointers, "no compressed klasses");
2709 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2710 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2711 }
2712 #endif
2713 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2714 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2715 }
2716
2717 //------------------------------Value------------------------------------------
2718 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2752 }
2753 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2754 }
2755 if (!t->is_klass()) {
2756 // a primitive Class (e.g., int.class) has null for a klass field
2757 return TypePtr::NULL_PTR;
2758 }
2759 // Fold up the load of the hidden field
2760 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2761 }
2762 // non-constant mirror, so we can't tell what's going on
2763 }
2764 if (!tinst->is_loaded())
2765 return _type; // Bail out if not loaded
2766 if (offset == oopDesc::klass_offset_in_bytes()) {
2767 return tinst->as_klass_type(true);
2768 }
2769 }
2770
2771 // Check for loading klass from an array
2772 const TypeAryPtr* tary = tp->isa_aryptr();
2773 if (tary != nullptr &&
2774 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2775 return tary->as_klass_type(true)->is_aryklassptr();
2776 }
2777
2778 // Check for loading klass from an array klass
2779 const TypeKlassPtr *tkls = tp->isa_klassptr();
2780 if (tkls != nullptr && !StressReflectiveCode) {
2781 if (!tkls->is_loaded())
2782 return _type; // Bail out if not loaded
2783 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2784 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2785 // // Always returning precise element type is incorrect,
2786 // // e.g., element type could be object and array may contain strings
2787 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2788
2789 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2790 // according to the element type's subclassing.
2791 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2792 }
2793 if (tkls->isa_aryklassptr() != nullptr && tkls->klass_is_exact() &&
2794 !tkls->exact_klass()->is_type_array_klass() &&
2795 tkls->offset() == in_bytes(Klass::super_offset())) {
2796 // We are loading the super klass of a refined array klass, return the non-refined klass pointer
2797 assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2798 return tkls->is_aryklassptr()->with_offset(0)->cast_to_non_refined();
2799 }
2800 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2801 tkls->offset() == in_bytes(Klass::super_offset())) {
2802 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2803 // The field is Klass::_super. Return its (constant) value.
2804 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2805 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2806 }
2807 }
2808
2809 if (tkls != nullptr && !UseSecondarySupersCache
2810 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2811 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2812 return TypePtr::NULL_PTR;
2813 }
2814
2815 // Bailout case
2816 return LoadNode::Value(phase);
2817 }
2818
2819 //------------------------------Identity---------------------------------------
2842 base = bs->step_over_gc_barrier(base);
2843 }
2844
2845 // We can fetch the klass directly through an AllocateNode.
2846 // This works even if the klass is not constant (clone or newArray).
2847 if (offset == oopDesc::klass_offset_in_bytes()) {
2848 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2849 if (allocated_klass != nullptr) {
2850 return allocated_klass;
2851 }
2852 }
2853
2854 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2855 // See inline_native_Class_query for occurrences of these patterns.
2856 // Java Example: x.getClass().isAssignableFrom(y)
2857 //
2858 // This improves reflective code, often making the Class
2859 // mirror go completely dead. (Current exception: Class
2860 // mirrors may appear in debug info, but we could clean them out by
2861 // introducing a new debug info operator for Klass.java_mirror).
2862 //
2863 // This optimization does not apply to arrays because if k is not a
2864 // constant, it was obtained via load_klass which returns the VM type
2865 // and '.java_mirror.as_klass' should return the Java type instead.
2866
2867 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2868 && offset == java_lang_Class::klass_offset()) {
2869 if (base->is_Load()) {
2870 Node* base2 = base->in(MemNode::Address);
2871 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2872 Node* adr2 = base2->in(MemNode::Address);
2873 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2874 if (tkls != nullptr && !tkls->empty()
2875 && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2876 && adr2->is_AddP()) {
2877 int mirror_field = in_bytes(Klass::java_mirror_offset());
2878 if (tkls->offset() == mirror_field) {
2879 #ifdef ASSERT
2880 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2881 assert(tkls2->offset() == 0, "not a load of java_mirror");
2882 #endif
2883 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2884 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2885 return adr2->in(AddPNode::Address);
2886 }
2887 }
2888 }
2889 }
2890 }
2891
2892 return this;
2893 }
2894
2895 LoadNode* LoadNode::clone_pinned() const {
2896 LoadNode* ld = clone()->as_Load();
3022 //---------------------------StoreNode::make-----------------------------------
3023 // Polymorphic factory method:
3024 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
3025 assert((mo == unordered || mo == release), "unexpected");
3026 Compile* C = gvn.C;
3027 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
3028 ctl != nullptr, "raw memory operations should have control edge");
3029
3030 switch (bt) {
3031 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
3032 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
3033 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
3034 case T_CHAR:
3035 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
3036 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3037 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
3038 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3039 case T_METADATA:
3040 case T_ADDRESS:
3041 case T_OBJECT:
3042 case T_ARRAY:
3043 #ifdef _LP64
3044 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3045 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
3046 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
3047 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
3048 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
3049 adr->bottom_type()->isa_rawptr())) {
3050 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
3051 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
3052 }
3053 #endif
3054 {
3055 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
3056 }
3057 default:
3058 guarantee(false, "unexpected basic type %s", type2name(bt));
3059 return (StoreNode*)nullptr;
3060 }
3061 }
3062
3063 //--------------------------bottom_type----------------------------------------
3064 const Type *StoreNode::bottom_type() const {
3065 return Type::MEMORY;
3066 }
3067
3068 //------------------------------hash-------------------------------------------
3069 uint StoreNode::hash() const {
3070 // unroll addition of interesting fields
3071 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
3072
3073 // Since they are not commoned, do not hash them:
3074 return NO_HASH;
3075 }
3076
3077 // Link together multiple stores (B/S/C/I) into a longer one.
3078 //
3700 }
3701 ss.print_cr("[TraceMergeStores]: with");
3702 merged_input_value->dump("\n", false, &ss);
3703 merged_store->dump("\n", false, &ss);
3704 tty->print("%s", ss.as_string());
3705 }
3706 #endif
3707
3708 //------------------------------Ideal------------------------------------------
3709 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3710 // When a store immediately follows a relevant allocation/initialization,
3711 // try to capture it into the initialization, or hoist it above.
3712 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3713 Node* p = MemNode::Ideal_common(phase, can_reshape);
3714 if (p) return (p == NodeSentinel) ? nullptr : p;
3715
3716 Node* mem = in(MemNode::Memory);
3717 Node* address = in(MemNode::Address);
3718 Node* value = in(MemNode::ValueIn);
3719 // Back-to-back stores to same address? Fold em up. Generally
3720 // unsafe if I have intervening uses...
3721 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3722 Node* st = mem;
3723 // If Store 'st' has more than one use, we cannot fold 'st' away.
3724 // For example, 'st' might be the final state at a conditional
3725 // return. Or, 'st' might be used by some node which is live at
3726 // the same time 'st' is live, which might be unschedulable. So,
3727 // require exactly ONE user until such time as we clone 'mem' for
3728 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3729 // true).
3730 while (st->is_Store() && st->outcnt() == 1) {
3731 // Looking at a dead closed cycle of memory?
3732 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3733 assert(Opcode() == st->Opcode() ||
3734 st->Opcode() == Op_StoreVector ||
3735 Opcode() == Op_StoreVector ||
3736 st->Opcode() == Op_StoreVectorScatter ||
3737 Opcode() == Op_StoreVectorScatter ||
3738 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3739 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3740 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3741 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3742 (st->adr_type()->isa_aryptr() && st->adr_type()->is_aryptr()->is_flat()) || // TODO 8343835
3743 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3744 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3745
3746 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3747 st->as_Store()->memory_size() <= this->memory_size()) {
3748 Node* use = st->raw_out(0);
3749 if (phase->is_IterGVN()) {
3750 phase->is_IterGVN()->rehash_node_delayed(use);
3751 }
3752 // It's OK to do this in the parser, since DU info is always accurate,
3753 // and the parser always refers to nodes via SafePointNode maps.
3754 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3755 return this;
3756 }
3757 st = st->in(MemNode::Memory);
3758 }
3759 }
3760
3761
3762 // Capture an unaliased, unconditional, simple store into an initializer.
3863 const StoreVectorNode* store_vector = as_StoreVector();
3864 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3865 const Node* store_indices = store_vector->indices();
3866 const Node* mem_indices = mem_vector->indices();
3867 const Node* store_mask = store_vector->mask();
3868 const Node* mem_mask = mem_vector->mask();
3869 // Ensure types, indices, and masks match
3870 if (store_vector->vect_type() == mem_vector->vect_type() &&
3871 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3872 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3873 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3874 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3875 result = mem;
3876 }
3877 }
3878 }
3879
3880 // Store of zero anywhere into a freshly-allocated object?
3881 // Then the store is useless.
3882 // (It must already have been captured by the InitializeNode.)
3883 if (result == this && ReduceFieldZeroing) {
3884 // a newly allocated object is already all-zeroes everywhere
3885 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3886 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3887 result = mem;
3888 }
3889
3890 if (result == this && phase->type(val)->is_zero_type()) {
3891 // the store may also apply to zero-bits in an earlier object
3892 Node* prev_mem = find_previous_store(phase);
3893 // Steps (a), (b): Walk past independent stores to find an exact match.
3894 if (prev_mem != nullptr) {
3895 if (prev_mem->is_top()) {
3896 // find_previous_store returns top when the access is dead
3897 return prev_mem;
3898 }
3899 Node* prev_val = can_see_stored_value(prev_mem, phase);
3900 if (prev_val != nullptr && prev_val == val) {
3901 // prev_val and val might differ by a cast; it would be good
3902 // to keep the more informative of the two.
3903 result = mem;
3904 }
3905 }
3906 }
3907 }
3908
3909 PhaseIterGVN* igvn = phase->is_IterGVN();
3910 if (result != this && igvn != nullptr) {
4383 // Clearing a short array is faster with stores
4384 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4385 // Already know this is a large node, do not try to ideal it
4386 if (_is_large) return nullptr;
4387
4388 const int unit = BytesPerLong;
4389 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4390 if (!t) return nullptr;
4391 if (!t->is_con()) return nullptr;
4392 intptr_t raw_count = t->get_con();
4393 intptr_t size = raw_count;
4394 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4395 // Clearing nothing uses the Identity call.
4396 // Negative clears are possible on dead ClearArrays
4397 // (see jck test stmt114.stmt11402.val).
4398 if (size <= 0 || size % unit != 0) return nullptr;
4399 intptr_t count = size / unit;
4400 // Length too long; communicate this to matchers and assemblers.
4401 // Assemblers are responsible to produce fast hardware clears for it.
4402 if (size > InitArrayShortSize) {
4403 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4404 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4405 return nullptr;
4406 }
4407 if (!IdealizeClearArrayNode) return nullptr;
4408 Node *mem = in(1);
4409 if( phase->type(mem)==Type::TOP ) return nullptr;
4410 Node *adr = in(3);
4411 const Type* at = phase->type(adr);
4412 if( at==Type::TOP ) return nullptr;
4413 const TypePtr* atp = at->isa_ptr();
4414 // adjust atp to be the correct array element address type
4415 if (atp == nullptr) atp = TypePtr::BOTTOM;
4416 else atp = atp->add_offset(Type::OffsetBot);
4417 // Get base for derived pointer purposes
4418 if( adr->Opcode() != Op_AddP ) Unimplemented();
4419 Node *base = adr->in(1);
4420
4421 Node *val = in(4);
4422 Node *off = phase->MakeConX(BytesPerLong);
4423 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4424 count--;
4425 while( count-- ) {
4426 mem = phase->transform(mem);
4427 adr = phase->transform(new AddPNode(base,adr,off));
4428 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4429 }
4430 return mem;
4431 }
4432
4433 //----------------------------step_through----------------------------------
4434 // Return allocation input memory edge if it is different instance
4435 // or itself if it is the one we are looking for.
4436 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4437 Node* n = *np;
4438 assert(n->is_ClearArray(), "sanity");
4439 intptr_t offset;
4440 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4441 // This method is called only before Allocate nodes are expanded
4442 // during macro nodes expansion. Before that ClearArray nodes are
4443 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4444 // Allocate nodes are expanded) which follows allocations.
4445 assert(alloc != nullptr, "should have allocation");
4446 if (alloc->_idx == instance_id) {
4447 // Can not bypass initialization of the instance we are looking for.
4448 return false;
4451 InitializeNode* init = alloc->initialization();
4452 if (init != nullptr)
4453 *np = init->in(TypeFunc::Memory);
4454 else
4455 *np = alloc->in(TypeFunc::Memory);
4456 return true;
4457 }
4458
4459 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4460 Node* base = dest;
4461 if (raw_base) {
4462 // May be called as part of the initialization of a just allocated object
4463 base = phase->C->top();
4464 }
4465 return phase->transform(new AddPNode(base, dest, offset));
4466 }
4467
4468 //----------------------------clear_memory-------------------------------------
4469 // Generate code to initialize object storage to zero.
4470 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4471 Node* val,
4472 Node* raw_val,
4473 intptr_t start_offset,
4474 Node* end_offset,
4475 bool raw_base,
4476 PhaseGVN* phase) {
4477 intptr_t offset = start_offset;
4478
4479 int unit = BytesPerLong;
4480 if ((offset % unit) != 0) {
4481 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4482 const TypePtr* atp = TypeRawPtr::BOTTOM;
4483 if (val != nullptr) {
4484 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4485 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4486 } else {
4487 assert(raw_val == nullptr, "val may not be null");
4488 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4489 }
4490 mem = phase->transform(mem);
4491 offset += BytesPerInt;
4492 }
4493 assert((offset % unit) == 0, "");
4494
4495 // Initialize the remaining stuff, if any, with a ClearArray.
4496 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, raw_base, phase);
4497 }
4498
4499 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4500 Node* raw_val,
4501 Node* start_offset,
4502 Node* end_offset,
4503 bool raw_base,
4504 PhaseGVN* phase) {
4505 if (start_offset == end_offset) {
4506 // nothing to do
4507 return mem;
4508 }
4509
4510 int unit = BytesPerLong;
4511 Node* zbase = start_offset;
4512 Node* zend = end_offset;
4513
4514 // Scale to the unit required by the CPU:
4515 if (!Matcher::init_array_count_is_in_bytes) {
4516 Node* shift = phase->intcon(exact_log2(unit));
4517 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4518 zend = phase->transform(new URShiftXNode(zend, shift) );
4519 }
4520
4521 // Bulk clear double-words
4522 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4523 Node* adr = make_address(dest, start_offset, raw_base, phase);
4524 if (raw_val == nullptr) {
4525 raw_val = phase->MakeConX(0);
4526 }
4527 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4528 return phase->transform(mem);
4529 }
4530
4531 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4532 Node* val,
4533 Node* raw_val,
4534 intptr_t start_offset,
4535 intptr_t end_offset,
4536 bool raw_base,
4537 PhaseGVN* phase) {
4538 if (start_offset == end_offset) {
4539 // nothing to do
4540 return mem;
4541 }
4542
4543 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4544 intptr_t done_offset = end_offset;
4545 if ((done_offset % BytesPerLong) != 0) {
4546 done_offset -= BytesPerInt;
4547 }
4548 if (done_offset > start_offset) {
4549 mem = clear_memory(ctl, mem, dest, val, raw_val,
4550 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4551 }
4552 if (done_offset < end_offset) { // emit the final 32-bit store
4553 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4554 const TypePtr* atp = TypeRawPtr::BOTTOM;
4555 if (val != nullptr) {
4556 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4557 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4558 } else {
4559 assert(raw_val == nullptr, "val may not be null");
4560 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4561 }
4562 mem = phase->transform(mem);
4563 done_offset += BytesPerInt;
4564 }
4565 assert(done_offset == end_offset, "");
4566 return mem;
4567 }
4568
4569 //=============================================================================
4570 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4571 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4572 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4573 #ifdef ASSERT
4574 , _pair_idx(0)
4575 #endif
4576 {
4577 init_class_id(Class_MemBar);
4578 Node* top = C->top();
4579 init_req(TypeFunc::I_O,top);
4580 init_req(TypeFunc::FramePtr,top);
4581 init_req(TypeFunc::ReturnAdr,top);
4688 PhaseIterGVN* igvn = phase->is_IterGVN();
4689 remove(igvn);
4690 // Must return either the original node (now dead) or a new node
4691 // (Do not return a top here, since that would break the uniqueness of top.)
4692 return new ConINode(TypeInt::ZERO);
4693 }
4694 }
4695 return progress ? this : nullptr;
4696 }
4697
4698 //------------------------------Value------------------------------------------
4699 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4700 if( !in(0) ) return Type::TOP;
4701 if( phase->type(in(0)) == Type::TOP )
4702 return Type::TOP;
4703 return TypeTuple::MEMBAR;
4704 }
4705
4706 //------------------------------match------------------------------------------
4707 // Construct projections for memory.
4708 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4709 switch (proj->_con) {
4710 case TypeFunc::Control:
4711 case TypeFunc::Memory:
4712 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4713 }
4714 ShouldNotReachHere();
4715 return nullptr;
4716 }
4717
4718 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4719 trailing->_kind = TrailingStore;
4720 leading->_kind = LeadingStore;
4721 #ifdef ASSERT
4722 trailing->_pair_idx = leading->_idx;
4723 leading->_pair_idx = leading->_idx;
4724 #endif
4725 }
4726
4727 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4728 trailing->_kind = TrailingLoadStore;
4975 return (req() > RawStores);
4976 }
4977
4978 void InitializeNode::set_complete(PhaseGVN* phase) {
4979 assert(!is_complete(), "caller responsibility");
4980 _is_complete = Complete;
4981
4982 // After this node is complete, it contains a bunch of
4983 // raw-memory initializations. There is no need for
4984 // it to have anything to do with non-raw memory effects.
4985 // Therefore, tell all non-raw users to re-optimize themselves,
4986 // after skipping the memory effects of this initialization.
4987 PhaseIterGVN* igvn = phase->is_IterGVN();
4988 if (igvn) igvn->add_users_to_worklist(this);
4989 }
4990
4991 // convenience function
4992 // return false if the init contains any stores already
4993 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4994 InitializeNode* init = initialization();
4995 if (init == nullptr || init->is_complete()) {
4996 return false;
4997 }
4998 init->remove_extra_zeroes();
4999 // for now, if this allocation has already collected any inits, bail:
5000 if (init->is_non_zero()) return false;
5001 init->set_complete(phase);
5002 return true;
5003 }
5004
5005 void InitializeNode::remove_extra_zeroes() {
5006 if (req() == RawStores) return;
5007 Node* zmem = zero_memory();
5008 uint fill = RawStores;
5009 for (uint i = fill; i < req(); i++) {
5010 Node* n = in(i);
5011 if (n->is_top() || n == zmem) continue; // skip
5012 if (fill < i) set_req(fill, n); // compact
5013 ++fill;
5014 }
5015 // delete any empty spaces created:
5016 while (fill < req()) {
5017 del_req(fill);
5161 // store node that we'd like to capture. We need to check
5162 // the uses of the MergeMemNode.
5163 mems.push(n);
5164 }
5165 } else if (n->is_Mem()) {
5166 Node* other_adr = n->in(MemNode::Address);
5167 if (other_adr == adr) {
5168 failed = true;
5169 break;
5170 } else {
5171 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5172 if (other_t_adr != nullptr) {
5173 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5174 if (other_alias_idx == alias_idx) {
5175 // A load from the same memory slice as the store right
5176 // after the InitializeNode. We check the control of the
5177 // object/array that is loaded from. If it's the same as
5178 // the store control then we cannot capture the store.
5179 assert(!n->is_Store(), "2 stores to same slice on same control?");
5180 Node* base = other_adr;
5181 if (base->is_Phi()) {
5182 // In rare case, base may be a PhiNode and it may read
5183 // the same memory slice between InitializeNode and store.
5184 failed = true;
5185 break;
5186 }
5187 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5188 base = base->in(AddPNode::Base);
5189 if (base != nullptr) {
5190 base = base->uncast();
5191 if (base->is_Proj() && base->in(0) == alloc) {
5192 failed = true;
5193 break;
5194 }
5195 }
5196 }
5197 }
5198 }
5199 } else {
5200 failed = true;
5201 break;
5202 }
5203 }
5204 }
5205 }
5206 if (failed) {
5753 // z's_done 12 16 16 16 12 16 12
5754 // z's_needed 12 16 16 16 16 16 16
5755 // zsize 0 0 0 0 4 0 4
5756 if (next_full_store < 0) {
5757 // Conservative tack: Zero to end of current word.
5758 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5759 } else {
5760 // Zero to beginning of next fully initialized word.
5761 // Or, don't zero at all, if we are already in that word.
5762 assert(next_full_store >= zeroes_needed, "must go forward");
5763 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5764 zeroes_needed = next_full_store;
5765 }
5766 }
5767
5768 if (zeroes_needed > zeroes_done) {
5769 intptr_t zsize = zeroes_needed - zeroes_done;
5770 // Do some incremental zeroing on rawmem, in parallel with inits.
5771 zeroes_done = align_down(zeroes_done, BytesPerInt);
5772 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5773 allocation()->in(AllocateNode::InitValue),
5774 allocation()->in(AllocateNode::RawInitValue),
5775 zeroes_done, zeroes_needed,
5776 true,
5777 phase);
5778 zeroes_done = zeroes_needed;
5779 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5780 do_zeroing = false; // leave the hole, next time
5781 }
5782 }
5783
5784 // Collect the store and move on:
5785 phase->replace_input_of(st, MemNode::Memory, inits);
5786 inits = st; // put it on the linearized chain
5787 set_req(i, zmem); // unhook from previous position
5788
5789 if (zeroes_done == st_off)
5790 zeroes_done = next_init_off;
5791
5792 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5793
5794 #ifdef ASSERT
5815 remove_extra_zeroes(); // clear out all the zmems left over
5816 add_req(inits);
5817
5818 if (!(UseTLAB && ZeroTLAB)) {
5819 // If anything remains to be zeroed, zero it all now.
5820 zeroes_done = align_down(zeroes_done, BytesPerInt);
5821 // if it is the last unused 4 bytes of an instance, forget about it
5822 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5823 if (zeroes_done + BytesPerLong >= size_limit) {
5824 AllocateNode* alloc = allocation();
5825 assert(alloc != nullptr, "must be present");
5826 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5827 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5828 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5829 if (zeroes_done == k->layout_helper())
5830 zeroes_done = size_limit;
5831 }
5832 }
5833 if (zeroes_done < size_limit) {
5834 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5835 allocation()->in(AllocateNode::InitValue),
5836 allocation()->in(AllocateNode::RawInitValue),
5837 zeroes_done, size_in_bytes, true, phase);
5838 }
5839 }
5840
5841 set_complete(phase);
5842 return rawmem;
5843 }
5844
5845 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5846 auto replace_proj = [&](ProjNode* proj) {
5847 C->gvn_replace_by(proj, mem);
5848 return CONTINUE;
5849 };
5850 apply_to_projs(replace_proj, TypeFunc::Memory);
5851 }
5852
5853 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5854 DUIterator_Fast imax, i = fast_outs(imax);
5855 auto replace_proj = [&](ProjNode* proj) {
5856 igvn->replace_node(proj, mem);
|