6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/compile.hpp"
38 #include "opto/connode.hpp"
39 #include "opto/convertnode.hpp"
40 #include "opto/loopnode.hpp"
41 #include "opto/machnode.hpp"
42 #include "opto/matcher.hpp"
43 #include "opto/memnode.hpp"
44 #include "opto/mempointer.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/opcodes.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regalloc.hpp"
50 #include "opto/regmask.hpp"
51 #include "opto/rootnode.hpp"
52 #include "opto/traceMergeStoresTag.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/macros.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #include "utilities/vmError.hpp"
59
60 // Portions of code courtesy of Clifford Click
61
62 // Optimization - Graph Style
63
64 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
65
66 //=============================================================================
67 uint MemNode::size_of() const { return sizeof(*this); }
68
69 const TypePtr *MemNode::adr_type() const {
70 Node* adr = in(Address);
71 if (adr == nullptr) return nullptr; // node is dead
72 const TypePtr* cross_check = nullptr;
73 DEBUG_ONLY(cross_check = _adr_type);
74 return calculate_adr_type(adr->bottom_type(), cross_check);
75 }
124 st->print(", idx=Bot;");
125 else if (atp->index() == Compile::AliasIdxTop)
126 st->print(", idx=Top;");
127 else if (atp->index() == Compile::AliasIdxRaw)
128 st->print(", idx=Raw;");
129 else {
130 ciField* field = atp->field();
131 if (field) {
132 st->print(", name=");
133 field->print_name_on(st);
134 }
135 st->print(", idx=%d;", atp->index());
136 }
137 }
138 }
139
140 extern void print_alias_types();
141
142 #endif
143
144 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
145 assert((t_oop != nullptr), "sanity");
146 bool is_instance = t_oop->is_known_instance_field();
147 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
148 (load != nullptr) && load->is_Load() &&
149 (phase->is_IterGVN() != nullptr);
150 if (!(is_instance || is_boxed_value_load))
151 return mchain; // don't try to optimize non-instance types
152 uint instance_id = t_oop->instance_id();
153 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
154 Node *prev = nullptr;
155 Node *result = mchain;
156 while (prev != result) {
157 prev = result;
158 if (result == start_mem)
159 break; // hit one of our sentinels
160 // skip over a call which does not affect this memory slice
161 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
162 Node *proj_in = result->in(0);
163 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
164 break; // hit one of our sentinels
165 } else if (proj_in->is_Call()) {
166 // ArrayCopyNodes processed here as well
167 CallNode *call = proj_in->as_Call();
168 if (!call->may_modify(t_oop, phase)) { // returns false for instances
169 result = call->in(TypeFunc::Memory);
170 }
171 } else if (proj_in->is_Initialize()) {
172 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
173 // Stop if this is the initialization for the object instance which
174 // contains this memory slice, otherwise skip over it.
175 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
176 break;
177 }
178 if (is_instance) {
179 result = proj_in->in(TypeFunc::Memory);
180 } else if (is_boxed_value_load) {
181 Node* klass = alloc->in(AllocateNode::KlassNode);
182 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
183 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
184 result = proj_in->in(TypeFunc::Memory); // not related allocation
185 }
186 }
187 } else if (proj_in->is_MemBar()) {
188 ArrayCopyNode* ac = nullptr;
189 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
190 break;
191 }
192 result = proj_in->in(TypeFunc::Memory);
193 } else if (proj_in->is_top()) {
194 break; // dead code
195 } else {
196 assert(false, "unexpected projection");
197 }
198 } else if (result->is_ClearArray()) {
199 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
200 // Can not bypass initialization of the instance
201 // we are looking for.
202 break;
203 }
204 // Otherwise skip it (the call updated 'result' value).
205 } else if (result->is_MergeMem()) {
206 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
207 }
208 }
209 return result;
210 }
211
212 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
213 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
214 if (t_oop == nullptr)
215 return mchain; // don't try to optimize non-oop types
216 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
217 bool is_instance = t_oop->is_known_instance_field();
218 PhaseIterGVN *igvn = phase->is_IterGVN();
219 if (is_instance && igvn != nullptr && result->is_Phi()) {
220 PhiNode *mphi = result->as_Phi();
221 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
222 const TypePtr *t = mphi->adr_type();
223 bool do_split = false;
224 // In the following cases, Load memory input can be further optimized based on
225 // its precise address type
226 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
227 do_split = true;
228 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
229 const TypeOopPtr* mem_t =
230 t->is_oopptr()->cast_to_exactness(true)
231 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
232 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
233 if (t_oop->isa_aryptr()) {
234 mem_t = mem_t->is_aryptr()
235 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
236 ->cast_to_size(t_oop->is_aryptr()->size())
237 ->with_offset(t_oop->is_aryptr()->offset())
238 ->is_aryptr();
239 }
240 do_split = mem_t == t_oop;
241 }
242 if (do_split) {
243 // clone the Phi with our address type
244 result = mphi->split_out_instance(t_adr, igvn);
245 } else {
246 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
247 }
248 }
249 return result;
250 }
251
252 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
253 uint alias_idx = phase->C->get_alias_index(tp);
254 Node *mem = mmem;
255 #ifdef ASSERT
256 {
257 // Check that current type is consistent with the alias index used during graph construction
258 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
259 bool consistent = adr_check == nullptr || adr_check->empty() ||
260 phase->C->must_alias(adr_check, alias_idx );
261 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
262 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
263 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
264 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
265 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
266 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
267 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
268 // don't assert if it is dead code.
269 consistent = true;
270 }
271 if( !consistent ) {
272 st->print("alias_idx==%d, adr_check==", alias_idx);
273 if( adr_check == nullptr ) {
274 st->print("null");
275 } else {
276 adr_check->dump();
277 }
278 st->cr();
279 print_alias_types();
280 assert(consistent, "adr_check must match alias idx");
281 }
282 }
283 #endif
996 in_bytes(JavaThread::vthread_offset()),
997 in_bytes(JavaThread::scopedValueCache_offset()),
998 };
999
1000 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1001 if (offset == offsets[i]) {
1002 return true;
1003 }
1004 }
1005 }
1006
1007 return false;
1008 }
1009 #endif
1010
1011 //----------------------------LoadNode::make-----------------------------------
1012 // Polymorphic factory method:
1013 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1014 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1015 Compile* C = gvn.C;
1016 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
1017
1018 // sanity check the alias category against the created node type
1019 assert(!(adr_type->isa_oopptr() &&
1020 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1021 "use LoadKlassNode instead");
1022 assert(!(adr_type->isa_aryptr() &&
1023 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1024 "use LoadRangeNode instead");
1025 // Check control edge of raw loads
1026 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1027 // oop will be recorded in oop map if load crosses safepoint
1028 rt->isa_oopptr() || is_immutable_value(adr),
1029 "raw memory operations should have control edge");
1030 LoadNode* load = nullptr;
1031 switch (bt) {
1032 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1033 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1034 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1035 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1036 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1037 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1038 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1039 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1040 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1041 case T_OBJECT:
1042 case T_NARROWOOP:
1043 #ifdef _LP64
1044 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1045 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1046 } else
1047 #endif
1048 {
1049 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1050 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1051 }
1052 break;
1053 default:
1054 ShouldNotReachHere();
1055 break;
1056 }
1057 assert(load != nullptr, "LoadNode should have been created");
1058 if (unaligned) {
1059 load->set_unaligned_access();
1060 }
1061 if (mismatched) {
1062 load->set_mismatched_access();
1063 }
1064 if (unsafe) {
1065 load->set_unsafe_access();
1066 }
1067 load->set_barrier_data(barrier_data);
1068 if (load->Opcode() == Op_LoadN) {
1069 Node* ld = gvn.transform(load);
1070 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1071 }
1072
1073 return load;
1074 }
1075
1076 //------------------------------hash-------------------------------------------
1077 uint LoadNode::hash() const {
1078 // unroll addition of interesting fields
1079 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1080 }
1081
1082 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1083 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1084 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1085 bool is_stable_ary = FoldStableValues &&
1086 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1087 tp->isa_aryptr()->is_stable();
1088
1089 return (eliminate_boxing && non_volatile) || is_stable_ary;
1090 }
1091
1092 return false;
1093 }
1094
1095 // Is the value loaded previously stored by an arraycopy? If so return
1096 // a load node that reads from the source array so we may be able to
1097 // optimize out the ArrayCopy node later.
1098 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1099 Node* ld_adr = in(MemNode::Address);
1100 intptr_t ld_off = 0;
1101 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1102 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1103 if (ac != nullptr) {
1104 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1105
1106 Node* mem = ac->in(TypeFunc::Memory);
1107 Node* ctl = ac->in(0);
1108 Node* src = ac->in(ArrayCopyNode::Src);
1109
1117 if (ac->as_ArrayCopy()->is_clonebasic()) {
1118 assert(ld_alloc != nullptr, "need an alloc");
1119 assert(addp->is_AddP(), "address must be addp");
1120 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1121 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1122 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1123 addp->set_req(AddPNode::Base, src);
1124 addp->set_req(AddPNode::Address, src);
1125 } else {
1126 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1127 ac->as_ArrayCopy()->is_copyof_validated() ||
1128 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1129 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1130 addp->set_req(AddPNode::Base, src);
1131 addp->set_req(AddPNode::Address, src);
1132
1133 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1134 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1135 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1136
1137 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1138 uint shift = exact_log2(type2aelembytes(ary_elem));
1139
1140 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1141 #ifdef _LP64
1142 diff = phase->transform(new ConvI2LNode(diff));
1143 #endif
1144 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1145
1146 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1147 addp->set_req(AddPNode::Offset, offset);
1148 }
1149 addp = phase->transform(addp);
1150 #ifdef ASSERT
1151 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1152 ld->_adr_type = adr_type;
1153 #endif
1154 ld->set_req(MemNode::Address, addp);
1155 ld->set_req(0, ctl);
1156 ld->set_req(MemNode::Memory, mem);
1157 return ld;
1158 }
1159 return nullptr;
1160 }
1161
1162 // This routine exists to make sure this set of tests is done the same
1163 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1164 // will change the graph shape in a way which makes memory alive twice at the
1165 // same time (uses the Oracle model of aliasing), then some
1166 // LoadXNode::Identity will fold things back to the equivalence-class model
1167 // of aliasing.
1168 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1169 Node* ld_adr = in(MemNode::Address);
1170 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1171 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1172
1173 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1174 uint alias_idx = atp->index();
1175 Node* result = nullptr;
1176 Node* current = st;
1177 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1178 // this load. Stop once any other kind of node is encountered.
1179 //
1180 // In principle, folding a load is moving it up until it meets a matching store.
1181 //
1182 // store(ptr, v); store(ptr, v); store(ptr, v);
1183 // membar1; -> membar1; -> load(ptr);
1184 // membar2; load(ptr); membar1;
1185 // load(ptr); membar2; membar2;
1186 //
1187 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1188 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1189 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1269
1270 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1271 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1272 if (in_vt != out_vt) {
1273 return nullptr;
1274 }
1275 }
1276 return st->in(MemNode::ValueIn);
1277 }
1278
1279 // A load from a freshly-created object always returns zero.
1280 // (This can happen after LoadNode::Ideal resets the load's memory input
1281 // to find_captured_store, which returned InitializeNode::zero_memory.)
1282 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1283 (st->in(0) == ld_alloc) &&
1284 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1285 // return a zero value for the load's basic type
1286 // (This is one of the few places where a generic PhaseTransform
1287 // can create new nodes. Think of it as lazily manifesting
1288 // virtually pre-existing constants.)
1289 if (value_basic_type() != T_VOID) {
1290 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1291 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1292 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1293 // by the ArrayCopyNode.
1294 return phase->zerocon(value_basic_type());
1295 }
1296 } else {
1297 // TODO: materialize all-zero vector constant
1298 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1299 }
1300 }
1301
1302 // A load from an initialization barrier can match a captured store.
1303 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1304 InitializeNode* init = st->in(0)->as_Initialize();
1305 AllocateNode* alloc = init->allocation();
1306 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1307 // examine a captured store value
1308 st = init->find_captured_store(ld_off, memory_size(), phase);
1321 base = bs->step_over_gc_barrier(base);
1322 if (base != nullptr && base->is_Proj() &&
1323 base->as_Proj()->_con == TypeFunc::Parms &&
1324 base->in(0)->is_CallStaticJava() &&
1325 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1326 return base->in(0)->in(TypeFunc::Parms);
1327 }
1328 }
1329
1330 break;
1331 }
1332
1333 return nullptr;
1334 }
1335
1336 //----------------------is_instance_field_load_with_local_phi------------------
1337 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1338 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1339 in(Address)->is_AddP() ) {
1340 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1341 // Only instances and boxed values.
1342 if( t_oop != nullptr &&
1343 (t_oop->is_ptr_to_boxed_value() ||
1344 t_oop->is_known_instance_field()) &&
1345 t_oop->offset() != Type::OffsetBot &&
1346 t_oop->offset() != Type::OffsetTop) {
1347 return true;
1348 }
1349 }
1350 return false;
1351 }
1352
1353 //------------------------------Identity---------------------------------------
1354 // Loads are identity if previous store is to same address
1355 Node* LoadNode::Identity(PhaseGVN* phase) {
1356 // If the previous store-maker is the right kind of Store, and the store is
1357 // to the same address, then we are equal to the value stored.
1358 Node* mem = in(Memory);
1359 Node* value = can_see_stored_value_through_membars(mem, phase);
1360 if( value ) {
1361 // byte, short & char stores truncate naturally.
1362 // A load has to load the truncated value which requires
1363 // some sort of masking operation and that requires an
1364 // Ideal call instead of an Identity call.
1365 if (memory_size() < BytesPerInt) {
1366 // If the input to the store does not fit with the load's result type,
1367 // it must be truncated via an Ideal call.
1368 if (!phase->type(value)->higher_equal(phase->type(this)))
1369 return this;
1370 }
1371 // (This works even when value is a Con, but LoadNode::Value
1372 // usually runs first, producing the singleton type of the Con.)
1373 if (!has_pinned_control_dependency() || value->is_Con()) {
1374 return value;
1375 } else {
1376 return this;
1377 }
1378 }
1379
1380 if (has_pinned_control_dependency()) {
1381 return this;
1382 }
1383 // Search for an existing data phi which was generated before for the same
1384 // instance's field to avoid infinite generation of phis in a loop.
1385 Node *region = mem->in(0);
1386 if (is_instance_field_load_with_local_phi(region)) {
1387 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1388 int this_index = phase->C->get_alias_index(addr_t);
1389 int this_offset = addr_t->offset();
1390 int this_iid = addr_t->instance_id();
1391 if (!addr_t->is_known_instance() &&
1392 addr_t->is_ptr_to_boxed_value()) {
1393 // Use _idx of address base (could be Phi node) for boxed values.
1394 intptr_t ignore = 0;
1395 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1396 if (base == nullptr) {
1397 return this;
1398 }
1399 this_iid = base->_idx;
1400 }
1401 const Type* this_type = bottom_type();
1402 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1403 Node* phi = region->fast_out(i);
1404 if (phi->is_Phi() && phi != mem &&
1405 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1406 return phi;
1407 }
1408 }
1409 }
1410
1411 return this;
1412 }
1413
1948 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1949 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1950
1951 // Skip up past a SafePoint control. Cannot do this for Stores because
1952 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1953 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1954 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1955 !addr_mark &&
1956 (depends_only_on_test() || has_unknown_control_dependency())) {
1957 ctrl = ctrl->in(0);
1958 set_req(MemNode::Control,ctrl);
1959 return this;
1960 }
1961
1962 intptr_t ignore = 0;
1963 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1964 if (base != nullptr
1965 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1966 // Check for useless control edge in some common special cases
1967 if (in(MemNode::Control) != nullptr
1968 && can_remove_control()
1969 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1970 && all_controls_dominate(base, phase->C->start(), phase)) {
1971 // A method-invariant, non-null address (constant or 'this' argument).
1972 set_req(MemNode::Control, nullptr);
1973 return this;
1974 }
1975 }
1976
1977 Node* mem = in(MemNode::Memory);
1978 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1979
1980 if (can_reshape && (addr_t != nullptr)) {
1981 // try to optimize our memory input
1982 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1983 if (opt_mem != mem) {
1984 set_req_X(MemNode::Memory, opt_mem, phase);
1985 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1986 return this;
1987 }
2100 // No match.
2101 return nullptr;
2102 }
2103
2104 //------------------------------Value-----------------------------------------
2105 const Type* LoadNode::Value(PhaseGVN* phase) const {
2106 // Either input is TOP ==> the result is TOP
2107 Node* mem = in(MemNode::Memory);
2108 const Type *t1 = phase->type(mem);
2109 if (t1 == Type::TOP) return Type::TOP;
2110 Node* adr = in(MemNode::Address);
2111 const TypePtr* tp = phase->type(adr)->isa_ptr();
2112 if (tp == nullptr || tp->empty()) return Type::TOP;
2113 int off = tp->offset();
2114 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2115 Compile* C = phase->C;
2116
2117 // If load can see a previous constant store, use that.
2118 Node* value = can_see_stored_value_through_membars(mem, phase);
2119 if (value != nullptr && value->is_Con()) {
2120 assert(value->bottom_type()->higher_equal(_type), "sanity");
2121 return value->bottom_type();
2122 }
2123
2124 // Try to guess loaded type from pointer type
2125 if (tp->isa_aryptr()) {
2126 const TypeAryPtr* ary = tp->is_aryptr();
2127 const Type* t = ary->elem();
2128
2129 // Determine whether the reference is beyond the header or not, by comparing
2130 // the offset against the offset of the start of the array's data.
2131 // Different array types begin at slightly different offsets (12 vs. 16).
2132 // We choose T_BYTE as an example base type that is least restrictive
2133 // as to alignment, which will therefore produce the smallest
2134 // possible base offset.
2135 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2136 const bool off_beyond_header = (off >= min_base_off);
2137
2138 // Try to constant-fold a stable array element.
2139 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2140 // Make sure the reference is not into the header and the offset is constant
2141 ciObject* aobj = ary->const_oop();
2142 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2143 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2144 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off,
2145 stable_dimension,
2146 value_basic_type(), is_unsigned());
2147 if (con_type != nullptr) {
2148 return con_type;
2149 }
2150 }
2151 }
2152
2153 // Don't do this for integer types. There is only potential profit if
2154 // the element type t is lower than _type; that is, for int types, if _type is
2155 // more restrictive than t. This only happens here if one is short and the other
2156 // char (both 16 bits), and in those cases we've made an intentional decision
2157 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2158 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2159 //
2160 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2161 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2162 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2163 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2164 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2165 // In fact, that could have been the original type of p1, and p1 could have
2166 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2167 // expression (LShiftL quux 3) independently optimized to the constant 8.
2168 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2169 && (_type->isa_vect() == nullptr)
2170 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2171 // t might actually be lower than _type, if _type is a unique
2172 // concrete subclass of abstract class t.
2173 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2174 const Type* jt = t->join_speculative(_type);
2175 // In any case, do not allow the join, per se, to empty out the type.
2176 if (jt->empty() && !t->empty()) {
2177 // This can happen if a interface-typed array narrows to a class type.
2178 jt = _type;
2179 }
2180 #ifdef ASSERT
2181 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2182 // The pointers in the autobox arrays are always non-null
2183 Node* base = adr->in(AddPNode::Base);
2184 if ((base != nullptr) && base->is_DecodeN()) {
2185 // Get LoadN node which loads IntegerCache.cache field
2186 base = base->in(1);
2187 }
2188 if ((base != nullptr) && base->is_Con()) {
2189 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2190 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2191 // It could be narrow oop
2192 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2193 }
2194 }
2195 }
2196 #endif
2197 return jt;
2198 }
2199 }
2200 } else if (tp->base() == Type::InstPtr) {
2201 assert( off != Type::OffsetBot ||
2202 // arrays can be cast to Objects
2203 !tp->isa_instptr() ||
2204 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2205 // unsafe field access may not have a constant offset
2206 C->has_unsafe_access(),
2207 "Field accesses must be precise" );
2208 // For oop loads, we expect the _type to be precise.
2209
2210 // Optimize loads from constant fields.
2211 const TypeInstPtr* tinst = tp->is_instptr();
2212 ciObject* const_oop = tinst->const_oop();
2213 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2214 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2215 if (con_type != nullptr) {
2216 return con_type;
2217 }
2218 }
2219 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2220 assert(off != Type::OffsetBot ||
2221 !tp->isa_instklassptr() ||
2222 // arrays can be cast to Objects
2223 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2224 // also allow array-loading from the primary supertype
2225 // array during subtype checks
2226 Opcode() == Op_LoadKlass,
2227 "Field accesses must be precise");
2228 // For klass/static loads, we expect the _type to be precise
2229 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2230 /* With mirrors being an indirect in the Klass*
2231 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2232 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2233 *
2234 * So check the type and klass of the node before the LoadP.
2241 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2242 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2243 return TypeInstPtr::make(klass->java_mirror());
2244 }
2245 }
2246 }
2247
2248 const TypeKlassPtr *tkls = tp->isa_klassptr();
2249 if (tkls != nullptr) {
2250 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2251 ciKlass* klass = tkls->exact_klass();
2252 // We are loading a field from a Klass metaobject whose identity
2253 // is known at compile time (the type is "exact" or "precise").
2254 // Check for fields we know are maintained as constants by the VM.
2255 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2256 // The field is Klass::_super_check_offset. Return its (constant) value.
2257 // (Folds up type checking code.)
2258 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2259 return TypeInt::make(klass->super_check_offset());
2260 }
2261 if (UseCompactObjectHeaders) {
2262 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2263 // The field is Klass::_prototype_header. Return its (constant) value.
2264 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2265 return TypeX::make(klass->prototype_header());
2266 }
2267 }
2268 // Compute index into primary_supers array
2269 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2270 // Check for overflowing; use unsigned compare to handle the negative case.
2271 if( depth < ciKlass::primary_super_limit() ) {
2272 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2273 // (Folds up type checking code.)
2274 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2275 ciKlass *ss = klass->super_of_depth(depth);
2276 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2277 }
2278 const Type* aift = load_array_final_field(tkls, klass);
2279 if (aift != nullptr) return aift;
2280 }
2281
2282 // We can still check if we are loading from the primary_supers array at a
2283 // shallow enough depth. Even though the klass is not exact, entries less
2284 // than or equal to its super depth are correct.
2285 if (tkls->is_loaded()) {
2286 ciKlass* klass = nullptr;
2320 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2321 // The key property of this type is that it folds up tests
2322 // for array-ness, since it proves that the layout_helper is positive.
2323 // Thus, a generic value like the basic object layout helper works fine.
2324 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2325 }
2326 }
2327
2328 // If we are loading from a freshly-allocated object/array, produce a zero.
2329 // Things to check:
2330 // 1. Load is beyond the header: headers are not guaranteed to be zero
2331 // 2. Load is not vectorized: vectors have no zero constant
2332 // 3. Load has no matching store, i.e. the input is the initial memory state
2333 const TypeOopPtr* tinst = tp->isa_oopptr();
2334 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2335 bool is_not_vect = (_type->isa_vect() == nullptr);
2336 if (is_not_header && is_not_vect) {
2337 Node* mem = in(MemNode::Memory);
2338 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2339 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2340 return Type::get_zero_type(_type->basic_type());
2341 }
2342 }
2343
2344 if (!UseCompactObjectHeaders) {
2345 Node* alloc = is_new_object_mark_load();
2346 if (alloc != nullptr) {
2347 return TypeX::make(markWord::prototype().value());
2348 }
2349 }
2350
2351 return _type;
2352 }
2353
2354 //------------------------------match_edge-------------------------------------
2355 // Do we Match on this edge index or not? Match only the address.
2356 uint LoadNode::match_edge(uint idx) const {
2357 return idx == MemNode::Address;
2358 }
2359
2360 //--------------------------LoadBNode::Ideal--------------------------------------
2361 //
2362 // If the previous store is to the same address as this load,
2363 // and the value stored was larger than a byte, replace this load
2364 // with the value stored truncated to a byte. If no truncation is
2365 // needed, the replacement is done in LoadNode::Identity().
2366 //
2367 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2476 }
2477 }
2478 // Identity call will handle the case where truncation is not needed.
2479 return LoadNode::Ideal(phase, can_reshape);
2480 }
2481
2482 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2483 Node* mem = in(MemNode::Memory);
2484 Node* value = can_see_stored_value_through_membars(mem, phase);
2485 if (value != nullptr && value->is_Con() &&
2486 !value->bottom_type()->higher_equal(_type)) {
2487 // If the input to the store does not fit with the load's result type,
2488 // it must be truncated. We can't delay until Ideal call since
2489 // a singleton Value is needed for split_thru_phi optimization.
2490 int con = value->get_int();
2491 return TypeInt::make((con << 16) >> 16);
2492 }
2493 return LoadNode::Value(phase);
2494 }
2495
2496 //=============================================================================
2497 //----------------------------LoadKlassNode::make------------------------------
2498 // Polymorphic factory method:
2499 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2500 // sanity check the alias category against the created node type
2501 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2502 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2503 #ifdef _LP64
2504 if (adr_type->is_ptr_to_narrowklass()) {
2505 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2506 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2507 }
2508 #endif
2509 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2510 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2511 }
2512
2513 //------------------------------Value------------------------------------------
2514 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2515 return klass_value_common(phase);
2548 }
2549 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2550 }
2551 if (!t->is_klass()) {
2552 // a primitive Class (e.g., int.class) has null for a klass field
2553 return TypePtr::NULL_PTR;
2554 }
2555 // Fold up the load of the hidden field
2556 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2557 }
2558 // non-constant mirror, so we can't tell what's going on
2559 }
2560 if (!tinst->is_loaded())
2561 return _type; // Bail out if not loaded
2562 if (offset == oopDesc::klass_offset_in_bytes()) {
2563 return tinst->as_klass_type(true);
2564 }
2565 }
2566
2567 // Check for loading klass from an array
2568 const TypeAryPtr *tary = tp->isa_aryptr();
2569 if (tary != nullptr &&
2570 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2571 return tary->as_klass_type(true);
2572 }
2573
2574 // Check for loading klass from an array klass
2575 const TypeKlassPtr *tkls = tp->isa_klassptr();
2576 if (tkls != nullptr && !StressReflectiveCode) {
2577 if (!tkls->is_loaded())
2578 return _type; // Bail out if not loaded
2579 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2580 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2581 // // Always returning precise element type is incorrect,
2582 // // e.g., element type could be object and array may contain strings
2583 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2584
2585 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2586 // according to the element type's subclassing.
2587 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2588 }
2589 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2590 tkls->offset() == in_bytes(Klass::super_offset())) {
2591 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2592 // The field is Klass::_super. Return its (constant) value.
2593 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2594 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2595 }
2596 }
2597
2598 if (tkls != nullptr && !UseSecondarySupersCache
2599 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2600 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2601 return TypePtr::NULL_PTR;
2602 }
2603
2604 // Bailout case
2605 return LoadNode::Value(phase);
2606 }
2607
2608 //------------------------------Identity---------------------------------------
2631 base = bs->step_over_gc_barrier(base);
2632 }
2633
2634 // We can fetch the klass directly through an AllocateNode.
2635 // This works even if the klass is not constant (clone or newArray).
2636 if (offset == oopDesc::klass_offset_in_bytes()) {
2637 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2638 if (allocated_klass != nullptr) {
2639 return allocated_klass;
2640 }
2641 }
2642
2643 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2644 // See inline_native_Class_query for occurrences of these patterns.
2645 // Java Example: x.getClass().isAssignableFrom(y)
2646 //
2647 // This improves reflective code, often making the Class
2648 // mirror go completely dead. (Current exception: Class
2649 // mirrors may appear in debug info, but we could clean them out by
2650 // introducing a new debug info operator for Klass.java_mirror).
2651
2652 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2653 && offset == java_lang_Class::klass_offset()) {
2654 if (base->is_Load()) {
2655 Node* base2 = base->in(MemNode::Address);
2656 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2657 Node* adr2 = base2->in(MemNode::Address);
2658 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2659 if (tkls != nullptr && !tkls->empty()
2660 && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2661 && adr2->is_AddP()
2662 ) {
2663 int mirror_field = in_bytes(Klass::java_mirror_offset());
2664 if (tkls->offset() == mirror_field) {
2665 #ifdef ASSERT
2666 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2667 assert(tkls2->offset() == 0, "not a load of java_mirror");
2668 #endif
2669 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2670 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2671 return adr2->in(AddPNode::Address);
2672 }
2673 }
2674 }
2675 }
2676 }
2677
2678 return this;
2679 }
2680
2681 LoadNode* LoadNode::clone_pinned() const {
2682 LoadNode* ld = clone()->as_Load();
2809 // Polymorphic factory method:
2810 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2811 assert((mo == unordered || mo == release), "unexpected");
2812 Compile* C = gvn.C;
2813 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
2814 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2815 ctl != nullptr, "raw memory operations should have control edge");
2816
2817 switch (bt) {
2818 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2819 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2820 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2821 case T_CHAR:
2822 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2823 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2824 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2825 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2826 case T_METADATA:
2827 case T_ADDRESS:
2828 case T_OBJECT:
2829 #ifdef _LP64
2830 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2831 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2832 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2833 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2834 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
2835 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2836 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2837 }
2838 #endif
2839 {
2840 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2841 }
2842 default:
2843 ShouldNotReachHere();
2844 return (StoreNode*)nullptr;
2845 }
2846 }
2847
2848 //--------------------------bottom_type----------------------------------------
2849 const Type *StoreNode::bottom_type() const {
2850 return Type::MEMORY;
2851 }
2852
2853 //------------------------------hash-------------------------------------------
2854 uint StoreNode::hash() const {
2855 // unroll addition of interesting fields
2856 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2857
2858 // Since they are not commoned, do not hash them:
2859 return NO_HASH;
2860 }
2861
2862 // Link together multiple stores (B/S/C/I) into a longer one.
2863 //
3485 }
3486 ss.print_cr("[TraceMergeStores]: with");
3487 merged_input_value->dump("\n", false, &ss);
3488 merged_store->dump("\n", false, &ss);
3489 tty->print("%s", ss.as_string());
3490 }
3491 #endif
3492
3493 //------------------------------Ideal------------------------------------------
3494 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3495 // When a store immediately follows a relevant allocation/initialization,
3496 // try to capture it into the initialization, or hoist it above.
3497 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3498 Node* p = MemNode::Ideal_common(phase, can_reshape);
3499 if (p) return (p == NodeSentinel) ? nullptr : p;
3500
3501 Node* mem = in(MemNode::Memory);
3502 Node* address = in(MemNode::Address);
3503 Node* value = in(MemNode::ValueIn);
3504 // Back-to-back stores to same address? Fold em up. Generally
3505 // unsafe if I have intervening uses.
3506 {
3507 Node* st = mem;
3508 // If Store 'st' has more than one use, we cannot fold 'st' away.
3509 // For example, 'st' might be the final state at a conditional
3510 // return. Or, 'st' might be used by some node which is live at
3511 // the same time 'st' is live, which might be unschedulable. So,
3512 // require exactly ONE user until such time as we clone 'mem' for
3513 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3514 // true).
3515 while (st->is_Store() && st->outcnt() == 1) {
3516 // Looking at a dead closed cycle of memory?
3517 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3518 assert(Opcode() == st->Opcode() ||
3519 st->Opcode() == Op_StoreVector ||
3520 Opcode() == Op_StoreVector ||
3521 st->Opcode() == Op_StoreVectorScatter ||
3522 Opcode() == Op_StoreVectorScatter ||
3523 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3524 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3525 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3526 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3527 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3528
3529 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3530 st->as_Store()->memory_size() <= this->memory_size()) {
3531 Node* use = st->raw_out(0);
3532 if (phase->is_IterGVN()) {
3533 phase->is_IterGVN()->rehash_node_delayed(use);
3534 }
3535 // It's OK to do this in the parser, since DU info is always accurate,
3536 // and the parser always refers to nodes via SafePointNode maps.
3537 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3538 return this;
3539 }
3540 st = st->in(MemNode::Memory);
3541 }
3542 }
3543
3544
3545 // Capture an unaliased, unconditional, simple store into an initializer.
3646 const StoreVectorNode* store_vector = as_StoreVector();
3647 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3648 const Node* store_indices = store_vector->indices();
3649 const Node* mem_indices = mem_vector->indices();
3650 const Node* store_mask = store_vector->mask();
3651 const Node* mem_mask = mem_vector->mask();
3652 // Ensure types, indices, and masks match
3653 if (store_vector->vect_type() == mem_vector->vect_type() &&
3654 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3655 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3656 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3657 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3658 result = mem;
3659 }
3660 }
3661 }
3662
3663 // Store of zero anywhere into a freshly-allocated object?
3664 // Then the store is useless.
3665 // (It must already have been captured by the InitializeNode.)
3666 if (result == this &&
3667 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3668 // a newly allocated object is already all-zeroes everywhere
3669 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3670 result = mem;
3671 }
3672
3673 if (result == this) {
3674 // the store may also apply to zero-bits in an earlier object
3675 Node* prev_mem = find_previous_store(phase);
3676 // Steps (a), (b): Walk past independent stores to find an exact match.
3677 if (prev_mem != nullptr) {
3678 if (prev_mem->is_top()) {
3679 // find_previous_store returns top when the access is dead
3680 return prev_mem;
3681 }
3682 Node* prev_val = can_see_stored_value(prev_mem, phase);
3683 if (prev_val != nullptr && prev_val == val) {
3684 // prev_val and val might differ by a cast; it would be good
3685 // to keep the more informative of the two.
3686 result = mem;
3687 }
3688 }
3689 }
3690 }
3691
3692 PhaseIterGVN* igvn = phase->is_IterGVN();
3693 if (result != this && igvn != nullptr) {
4166 // Clearing a short array is faster with stores
4167 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4168 // Already know this is a large node, do not try to ideal it
4169 if (_is_large) return nullptr;
4170
4171 const int unit = BytesPerLong;
4172 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4173 if (!t) return nullptr;
4174 if (!t->is_con()) return nullptr;
4175 intptr_t raw_count = t->get_con();
4176 intptr_t size = raw_count;
4177 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4178 // Clearing nothing uses the Identity call.
4179 // Negative clears are possible on dead ClearArrays
4180 // (see jck test stmt114.stmt11402.val).
4181 if (size <= 0 || size % unit != 0) return nullptr;
4182 intptr_t count = size / unit;
4183 // Length too long; communicate this to matchers and assemblers.
4184 // Assemblers are responsible to produce fast hardware clears for it.
4185 if (size > InitArrayShortSize) {
4186 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4187 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4188 return nullptr;
4189 }
4190 if (!IdealizeClearArrayNode) return nullptr;
4191 Node *mem = in(1);
4192 if( phase->type(mem)==Type::TOP ) return nullptr;
4193 Node *adr = in(3);
4194 const Type* at = phase->type(adr);
4195 if( at==Type::TOP ) return nullptr;
4196 const TypePtr* atp = at->isa_ptr();
4197 // adjust atp to be the correct array element address type
4198 if (atp == nullptr) atp = TypePtr::BOTTOM;
4199 else atp = atp->add_offset(Type::OffsetBot);
4200 // Get base for derived pointer purposes
4201 if( adr->Opcode() != Op_AddP ) Unimplemented();
4202 Node *base = adr->in(1);
4203
4204 Node *zero = phase->makecon(TypeLong::ZERO);
4205 Node *off = phase->MakeConX(BytesPerLong);
4206 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4207 count--;
4208 while (count--) {
4209 mem = phase->transform(mem);
4210 adr = phase->transform(AddPNode::make_with_base(base, adr, off));
4211 mem = new StoreLNode(in(0), mem, adr, atp, zero, MemNode::unordered, false);
4212 }
4213 return mem;
4214 }
4215
4216 //----------------------------step_through----------------------------------
4217 // Return allocation input memory edge if it is different instance
4218 // or itself if it is the one we are looking for.
4219 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4220 Node* n = *np;
4221 assert(n->is_ClearArray(), "sanity");
4222 intptr_t offset;
4223 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4224 // This method is called only before Allocate nodes are expanded
4225 // during macro nodes expansion. Before that ClearArray nodes are
4226 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4227 // Allocate nodes are expanded) which follows allocations.
4228 assert(alloc != nullptr, "should have allocation");
4229 if (alloc->_idx == instance_id) {
4230 // Can not bypass initialization of the instance we are looking for.
4231 return false;
4234 InitializeNode* init = alloc->initialization();
4235 if (init != nullptr)
4236 *np = init->in(TypeFunc::Memory);
4237 else
4238 *np = alloc->in(TypeFunc::Memory);
4239 return true;
4240 }
4241
4242 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4243 Node* base = dest;
4244 if (raw_base) {
4245 // May be called as part of the initialization of a just allocated object
4246 base = phase->C->top();
4247 }
4248 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4249 }
4250
4251 //----------------------------clear_memory-------------------------------------
4252 // Generate code to initialize object storage to zero.
4253 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4254 intptr_t start_offset,
4255 Node* end_offset,
4256 bool raw_base,
4257 PhaseGVN* phase) {
4258 intptr_t offset = start_offset;
4259
4260 int unit = BytesPerLong;
4261 if ((offset % unit) != 0) {
4262 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4263 const TypePtr* atp = TypeRawPtr::BOTTOM;
4264 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4265 mem = phase->transform(mem);
4266 offset += BytesPerInt;
4267 }
4268 assert((offset % unit) == 0, "");
4269
4270 // Initialize the remaining stuff, if any, with a ClearArray.
4271 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, raw_base, phase);
4272 }
4273
4274 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4275 Node* start_offset,
4276 Node* end_offset,
4277 bool raw_base,
4278 PhaseGVN* phase) {
4279 if (start_offset == end_offset) {
4280 // nothing to do
4281 return mem;
4282 }
4283
4284 int unit = BytesPerLong;
4285 Node* zbase = start_offset;
4286 Node* zend = end_offset;
4287
4288 // Scale to the unit required by the CPU:
4289 if (!Matcher::init_array_count_is_in_bytes) {
4290 Node* shift = phase->intcon(exact_log2(unit));
4291 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4292 zend = phase->transform(new URShiftXNode(zend, shift) );
4293 }
4294
4295 // Bulk clear double-words
4296 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4297 Node* adr = make_address(dest, start_offset, raw_base, phase);
4298 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4299 return phase->transform(mem);
4300 }
4301
4302 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4303 intptr_t start_offset,
4304 intptr_t end_offset,
4305 bool raw_base,
4306 PhaseGVN* phase) {
4307 if (start_offset == end_offset) {
4308 // nothing to do
4309 return mem;
4310 }
4311
4312 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4313 intptr_t done_offset = end_offset;
4314 if ((done_offset % BytesPerLong) != 0) {
4315 done_offset -= BytesPerInt;
4316 }
4317 if (done_offset > start_offset) {
4318 mem = clear_memory(ctl, mem, dest,
4319 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4320 }
4321 if (done_offset < end_offset) { // emit the final 32-bit store
4322 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4323 const TypePtr* atp = TypeRawPtr::BOTTOM;
4324 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4325 mem = phase->transform(mem);
4326 done_offset += BytesPerInt;
4327 }
4328 assert(done_offset == end_offset, "");
4329 return mem;
4330 }
4331
4332 //=============================================================================
4333 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4334 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4335 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4336 #ifdef ASSERT
4337 , _pair_idx(0)
4338 #endif
4339 {
4340 init_class_id(Class_MemBar);
4341 Node* top = C->top();
4342 init_req(TypeFunc::I_O,top);
4343 init_req(TypeFunc::FramePtr,top);
4344 init_req(TypeFunc::ReturnAdr,top);
4453 PhaseIterGVN* igvn = phase->is_IterGVN();
4454 remove(igvn);
4455 // Must return either the original node (now dead) or a new node
4456 // (Do not return a top here, since that would break the uniqueness of top.)
4457 return new ConINode(TypeInt::ZERO);
4458 }
4459 }
4460 return progress ? this : nullptr;
4461 }
4462
4463 //------------------------------Value------------------------------------------
4464 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4465 if( !in(0) ) return Type::TOP;
4466 if( phase->type(in(0)) == Type::TOP )
4467 return Type::TOP;
4468 return TypeTuple::MEMBAR;
4469 }
4470
4471 //------------------------------match------------------------------------------
4472 // Construct projections for memory.
4473 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4474 switch (proj->_con) {
4475 case TypeFunc::Control:
4476 case TypeFunc::Memory:
4477 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4478 }
4479 ShouldNotReachHere();
4480 return nullptr;
4481 }
4482
4483 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4484 trailing->_kind = TrailingStore;
4485 leading->_kind = LeadingStore;
4486 #ifdef ASSERT
4487 trailing->_pair_idx = leading->_idx;
4488 leading->_pair_idx = leading->_idx;
4489 #endif
4490 }
4491
4492 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4493 trailing->_kind = TrailingLoadStore;
4740 return (req() > RawStores);
4741 }
4742
4743 void InitializeNode::set_complete(PhaseGVN* phase) {
4744 assert(!is_complete(), "caller responsibility");
4745 _is_complete = Complete;
4746
4747 // After this node is complete, it contains a bunch of
4748 // raw-memory initializations. There is no need for
4749 // it to have anything to do with non-raw memory effects.
4750 // Therefore, tell all non-raw users to re-optimize themselves,
4751 // after skipping the memory effects of this initialization.
4752 PhaseIterGVN* igvn = phase->is_IterGVN();
4753 if (igvn) igvn->add_users_to_worklist(this);
4754 }
4755
4756 // convenience function
4757 // return false if the init contains any stores already
4758 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4759 InitializeNode* init = initialization();
4760 if (init == nullptr || init->is_complete()) return false;
4761 init->remove_extra_zeroes();
4762 // for now, if this allocation has already collected any inits, bail:
4763 if (init->is_non_zero()) return false;
4764 init->set_complete(phase);
4765 return true;
4766 }
4767
4768 void InitializeNode::remove_extra_zeroes() {
4769 if (req() == RawStores) return;
4770 Node* zmem = zero_memory();
4771 uint fill = RawStores;
4772 for (uint i = fill; i < req(); i++) {
4773 Node* n = in(i);
4774 if (n->is_top() || n == zmem) continue; // skip
4775 if (fill < i) set_req(fill, n); // compact
4776 ++fill;
4777 }
4778 // delete any empty spaces created:
4779 while (fill < req()) {
4780 del_req(fill);
4924 // store node that we'd like to capture. We need to check
4925 // the uses of the MergeMemNode.
4926 mems.push(n);
4927 }
4928 } else if (n->is_Mem()) {
4929 Node* other_adr = n->in(MemNode::Address);
4930 if (other_adr == adr) {
4931 failed = true;
4932 break;
4933 } else {
4934 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4935 if (other_t_adr != nullptr) {
4936 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4937 if (other_alias_idx == alias_idx) {
4938 // A load from the same memory slice as the store right
4939 // after the InitializeNode. We check the control of the
4940 // object/array that is loaded from. If it's the same as
4941 // the store control then we cannot capture the store.
4942 assert(!n->is_Store(), "2 stores to same slice on same control?");
4943 Node* base = other_adr;
4944 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4945 base = base->in(AddPNode::Base);
4946 if (base != nullptr) {
4947 base = base->uncast();
4948 if (base->is_Proj() && base->in(0) == alloc) {
4949 failed = true;
4950 break;
4951 }
4952 }
4953 }
4954 }
4955 }
4956 } else {
4957 failed = true;
4958 break;
4959 }
4960 }
4961 }
4962 }
4963 if (failed) {
5509 // z's_done 12 16 16 16 12 16 12
5510 // z's_needed 12 16 16 16 16 16 16
5511 // zsize 0 0 0 0 4 0 4
5512 if (next_full_store < 0) {
5513 // Conservative tack: Zero to end of current word.
5514 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5515 } else {
5516 // Zero to beginning of next fully initialized word.
5517 // Or, don't zero at all, if we are already in that word.
5518 assert(next_full_store >= zeroes_needed, "must go forward");
5519 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5520 zeroes_needed = next_full_store;
5521 }
5522 }
5523
5524 if (zeroes_needed > zeroes_done) {
5525 intptr_t zsize = zeroes_needed - zeroes_done;
5526 // Do some incremental zeroing on rawmem, in parallel with inits.
5527 zeroes_done = align_down(zeroes_done, BytesPerInt);
5528 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5529 zeroes_done, zeroes_needed,
5530 true,
5531 phase);
5532 zeroes_done = zeroes_needed;
5533 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5534 do_zeroing = false; // leave the hole, next time
5535 }
5536 }
5537
5538 // Collect the store and move on:
5539 phase->replace_input_of(st, MemNode::Memory, inits);
5540 inits = st; // put it on the linearized chain
5541 set_req(i, zmem); // unhook from previous position
5542
5543 if (zeroes_done == st_off)
5544 zeroes_done = next_init_off;
5545
5546 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5547
5548 #ifdef ASSERT
5569 remove_extra_zeroes(); // clear out all the zmems left over
5570 add_req(inits);
5571
5572 if (!(UseTLAB && ZeroTLAB)) {
5573 // If anything remains to be zeroed, zero it all now.
5574 zeroes_done = align_down(zeroes_done, BytesPerInt);
5575 // if it is the last unused 4 bytes of an instance, forget about it
5576 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5577 if (zeroes_done + BytesPerLong >= size_limit) {
5578 AllocateNode* alloc = allocation();
5579 assert(alloc != nullptr, "must be present");
5580 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5581 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5582 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5583 if (zeroes_done == k->layout_helper())
5584 zeroes_done = size_limit;
5585 }
5586 }
5587 if (zeroes_done < size_limit) {
5588 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5589 zeroes_done, size_in_bytes, true, phase);
5590 }
5591 }
5592
5593 set_complete(phase);
5594 return rawmem;
5595 }
5596
5597 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5598 auto replace_proj = [&](ProjNode* proj) {
5599 C->gvn_replace_by(proj, mem);
5600 return CONTINUE;
5601 };
5602 apply_to_projs(replace_proj, TypeFunc::Memory);
5603 }
5604
5605 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5606 DUIterator_Fast imax, i = fast_outs(imax);
5607 auto replace_proj = [&](ProjNode* proj) {
5608 igvn->replace_node(proj, mem);
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "ci/ciInlineKlass.hpp"
28 #include "classfile/javaClasses.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "classfile/vmIntrinsics.hpp"
31 #include "compiler/compileLog.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/c2/barrierSetC2.hpp"
34 #include "gc/shared/tlab_globals.hpp"
35 #include "memory/allocation.inline.hpp"
36 #include "memory/resourceArea.hpp"
37 #include "oops/flatArrayKlass.hpp"
38 #include "oops/objArrayKlass.hpp"
39 #include "opto/addnode.hpp"
40 #include "opto/arraycopynode.hpp"
41 #include "opto/callnode.hpp"
42 #include "opto/cfgnode.hpp"
43 #include "opto/compile.hpp"
44 #include "opto/connode.hpp"
45 #include "opto/convertnode.hpp"
46 #include "opto/inlinetypenode.hpp"
47 #include "opto/loopnode.hpp"
48 #include "opto/machnode.hpp"
49 #include "opto/matcher.hpp"
50 #include "opto/memnode.hpp"
51 #include "opto/mempointer.hpp"
52 #include "opto/mulnode.hpp"
53 #include "opto/narrowptrnode.hpp"
54 #include "opto/opcodes.hpp"
55 #include "opto/phaseX.hpp"
56 #include "opto/regalloc.hpp"
57 #include "opto/regmask.hpp"
58 #include "opto/rootnode.hpp"
59 #include "opto/traceMergeStoresTag.hpp"
60 #include "opto/vectornode.hpp"
61 #include "runtime/arguments.hpp"
62 #include "utilities/align.hpp"
63 #include "utilities/copy.hpp"
64 #include "utilities/globalDefinitions.hpp"
65 #include "utilities/macros.hpp"
66 #include "utilities/powerOfTwo.hpp"
67 #include "utilities/vmError.hpp"
68
69 // Portions of code courtesy of Clifford Click
70
71 // Optimization - Graph Style
72
73 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
74
75 //=============================================================================
76 uint MemNode::size_of() const { return sizeof(*this); }
77
78 const TypePtr *MemNode::adr_type() const {
79 Node* adr = in(Address);
80 if (adr == nullptr) return nullptr; // node is dead
81 const TypePtr* cross_check = nullptr;
82 DEBUG_ONLY(cross_check = _adr_type);
83 return calculate_adr_type(adr->bottom_type(), cross_check);
84 }
133 st->print(", idx=Bot;");
134 else if (atp->index() == Compile::AliasIdxTop)
135 st->print(", idx=Top;");
136 else if (atp->index() == Compile::AliasIdxRaw)
137 st->print(", idx=Raw;");
138 else {
139 ciField* field = atp->field();
140 if (field) {
141 st->print(", name=");
142 field->print_name_on(st);
143 }
144 st->print(", idx=%d;", atp->index());
145 }
146 }
147 }
148
149 extern void print_alias_types();
150
151 #endif
152
153 // Find the memory output corresponding to the fall-through path of a call
154 static Node* find_call_fallthrough_mem_output(CallNode* call) {
155 ResourceMark rm;
156 CallProjections* projs = call->extract_projections(false, false);
157 Node* res = projs->fallthrough_memproj;
158 assert(res != nullptr, "must have a fallthrough mem output");
159 return res;
160 }
161
162 // Try to find a better memory input for a load from a strict final field
163 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
164 intptr_t offset = 0;
165 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
166 if (base == nullptr) {
167 return nullptr;
168 }
169
170 Node* base_uncasted = base->uncast();
171 if (base_uncasted->is_Proj()) {
172 Node* multi = base_uncasted->in(0);
173 if (multi->is_top()) {
174 // The pointer dies, make the memory die, too
175 return multi;
176 } else if (multi->is_Allocate()) {
177 base_local = base_uncasted->as_Proj();
178 return nullptr;
179 } else if (multi->is_Call()) {
180 if (!multi->is_CallJava() || multi->as_CallJava()->method() == nullptr || !multi->as_CallJava()->method()->return_value_is_larval()) {
181 // The oop is returned from a call, the memory can be the fallthrough output of the call
182 return find_call_fallthrough_mem_output(multi->as_Call());
183 }
184 } else if (multi->is_Start()) {
185 // The oop is a parameter
186 if (base_uncasted->as_Proj()->_con == TypeFunc::Parms && phase->C->method()->receiver_maybe_larval()) {
187 // The receiver of a constructor is similar to the result of an AllocateNode
188 base_local = base_uncasted->as_Proj();
189 return nullptr;
190 } else {
191 // Use the start memory otherwise
192 return multi->as_Start()->proj_out(TypeFunc::Memory);
193 }
194 }
195 }
196
197 return nullptr;
198 }
199
200 // Whether a call can modify a strict final field, given that the object is allocated inside the
201 // current compilation unit, or is the first parameter when the compilation root is a constructor.
202 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
203 // the target method is a subclass of the class declaring 'field'.
204 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
205 if (!call->is_CallJava()) {
206 return false;
207 }
208
209 ciMethod* target = call->as_CallJava()->method();
210 if (target == nullptr) {
211 return false;
212 } else if (target->intrinsic_id() == vmIntrinsicID::_linkToSpecial) {
213 // linkToSpecial can be used to call a constructor, used in the construction of objects in the
214 // reflection API
215 return true;
216 } else if (!target->is_object_constructor()) {
217 return false;
218 }
219
220 // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
221 // then the field is set inside the constructor, else the field must be set before the
222 // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
223 // while a field Sub.y must be set before Super::<init> is invoked.
224 // We can try to be more heroic and decide if the receiver of the constructor invocation is the
225 // object from which we are loading from. This, however, may be problematic as deciding if 2
226 // nodes are definitely different may not be trivial, especially if the graph is not canonical.
227 // As a result, it is made more conservative for now.
228 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
229 return target->holder()->is_subclass_of(field->holder());
230 }
231
232 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
233 assert(t_oop != nullptr, "sanity");
234 bool is_known_instance = t_oop->is_known_instance_field();
235 bool is_strict_final_load = false;
236
237 // After macro expansion, an allocation may become a call, changing the memory input to the
238 // memory output of that call would be illegal. As a result, disallow this transformation after
239 // macro expansion.
240 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
241 is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
242 #ifdef ASSERT
243 if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
244 assert(is_strict_final_load, "sanity check for basic cases");
245 }
246 #endif // ASSERT
247 }
248
249 if (!is_known_instance && !is_strict_final_load) {
250 return mchain;
251 }
252
253 Node* result = mchain;
254 ProjNode* base_local = nullptr;
255
256 ciField* field = nullptr;
257 if (is_strict_final_load) {
258 field = phase->C->alias_type(t_oop)->field();
259 assert(field != nullptr, "must point to a field");
260
261 Node* adr = load->in(MemNode::Address);
262 assert(phase->type(adr) == t_oop, "inconsistent type");
263 Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
264 if (tmp != nullptr) {
265 result = tmp;
266 }
267 }
268
269 uint instance_id = t_oop->instance_id();
270 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
271 Node* prev = nullptr;
272 while (prev != result) {
273 prev = result;
274 if (result == start_mem) {
275 // start_mem is the earliest memory possible
276 break;
277 }
278
279 // skip over a call which does not affect this memory slice
280 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
281 Node* proj_in = result->in(0);
282 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
283 // This is the allocation that creates the object from which we are loading from
284 break;
285 } else if (proj_in->is_Call()) {
286 // ArrayCopyNodes processed here as well
287 CallNode* call = proj_in->as_Call();
288 if (!call->may_modify(t_oop, phase)) {
289 result = call->in(TypeFunc::Memory);
290 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
291 result = call->in(TypeFunc::Memory);
292 }
293 } else if (proj_in->Opcode() == Op_Tuple) {
294 // The call will be folded, skip over it.
295 break;
296 } else if (proj_in->is_Initialize()) {
297 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
298 // Stop if this is the initialization for the object instance which
299 // contains this memory slice, otherwise skip over it.
300 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
301 break;
302 }
303 if (is_known_instance) {
304 result = proj_in->in(TypeFunc::Memory);
305 } else if (is_strict_final_load) {
306 Node* klass = alloc->in(AllocateNode::KlassNode);
307 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
308 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
309 // Allocation of another type, must be another object
310 result = proj_in->in(TypeFunc::Memory);
311 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
312 // Allocation of another object
313 result = proj_in->in(TypeFunc::Memory);
314 }
315 }
316 } else if (proj_in->is_MemBar()) {
317 ArrayCopyNode* ac = nullptr;
318 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
319 break;
320 }
321 result = proj_in->in(TypeFunc::Memory);
322 } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
323 if (is_strict_final_load) {
324 // LoadFlat and StoreFlat cannot happen to strict final fields
325 result = proj_in->in(TypeFunc::Memory);
326 }
327 } else if (proj_in->is_top()) {
328 break; // dead code
329 } else {
330 assert(false, "unexpected projection of %s", proj_in->Name());
331 }
332 } else if (result->is_ClearArray()) {
333 if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
334 // Can not bypass initialization of the instance
335 // we are looking for.
336 break;
337 }
338 // Otherwise skip it (the call updated 'result' value).
339 } else if (result->is_MergeMem()) {
340 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
341 }
342 }
343 return result;
344 }
345
346 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
347 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
348 if (t_oop == nullptr)
349 return mchain; // don't try to optimize non-oop types
350 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
351 bool is_instance = t_oop->is_known_instance_field();
352 PhaseIterGVN *igvn = phase->is_IterGVN();
353 if (is_instance && igvn != nullptr && result->is_Phi()) {
354 PhiNode *mphi = result->as_Phi();
355 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
356 const TypePtr *t = mphi->adr_type();
357 bool do_split = false;
358 // In the following cases, Load memory input can be further optimized based on
359 // its precise address type
360 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
361 do_split = true;
362 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
363 const TypeOopPtr* mem_t =
364 t->is_oopptr()->cast_to_exactness(true)
365 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
366 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
367 if (t_oop->isa_aryptr()) {
368 mem_t = mem_t->is_aryptr()
369 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
370 ->cast_to_size(t_oop->is_aryptr()->size())
371 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
372 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
373 ->with_offset(t_oop->is_aryptr()->offset())
374 ->is_aryptr();
375 }
376 do_split = mem_t == t_oop;
377 }
378 if (do_split) {
379 // clone the Phi with our address type
380 result = mphi->split_out_instance(t_adr, igvn);
381 } else {
382 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
383 }
384 }
385 return result;
386 }
387
388 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
389 uint alias_idx = phase->C->get_alias_index(tp);
390 Node *mem = mmem;
391 #ifdef ASSERT
392 {
393 // Check that current type is consistent with the alias index used during graph construction
394 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
395 bool consistent = adr_check == nullptr || adr_check->empty() ||
396 phase->C->must_alias(adr_check, alias_idx );
397 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
398 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
399 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
400 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
401 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
402 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
403 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
404 // don't assert if it is dead code.
405 consistent = true;
406 }
407 if( !consistent ) {
408 st->print("alias_idx==%d, adr_check==", alias_idx);
409 if( adr_check == nullptr ) {
410 st->print("null");
411 } else {
412 adr_check->dump();
413 }
414 st->cr();
415 print_alias_types();
416 assert(consistent, "adr_check must match alias idx");
417 }
418 }
419 #endif
1132 in_bytes(JavaThread::vthread_offset()),
1133 in_bytes(JavaThread::scopedValueCache_offset()),
1134 };
1135
1136 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1137 if (offset == offsets[i]) {
1138 return true;
1139 }
1140 }
1141 }
1142
1143 return false;
1144 }
1145 #endif
1146
1147 //----------------------------LoadNode::make-----------------------------------
1148 // Polymorphic factory method:
1149 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1150 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1151 Compile* C = gvn.C;
1152 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr(), true) == C->get_alias_index(adr_type, true), "adr and adr_type must agree");
1153
1154 // sanity check the alias category against the created node type
1155 assert(!(adr_type->isa_oopptr() &&
1156 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1157 "use LoadKlassNode instead");
1158 assert(!(adr_type->isa_aryptr() &&
1159 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1160 "use LoadRangeNode instead");
1161 // Check control edge of raw loads
1162 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1163 // oop will be recorded in oop map if load crosses safepoint
1164 rt->isa_oopptr() || is_immutable_value(adr),
1165 "raw memory operations should have control edge");
1166 LoadNode* load = nullptr;
1167 switch (bt) {
1168 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1169 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1170 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1171 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1172 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1173 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1174 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1175 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1176 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1177 case T_ARRAY:
1178 case T_OBJECT:
1179 case T_NARROWOOP:
1180 #ifdef _LP64
1181 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1182 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1183 } else
1184 #endif
1185 {
1186 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1187 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1188 }
1189 break;
1190 default:
1191 guarantee(false, "unexpected basic type %s", type2name(bt));
1192 break;
1193 }
1194 assert(load != nullptr, "LoadNode should have been created");
1195 if (unaligned) {
1196 load->set_unaligned_access();
1197 }
1198 if (mismatched) {
1199 load->set_mismatched_access();
1200 }
1201 if (unsafe) {
1202 load->set_unsafe_access();
1203 }
1204 load->set_barrier_data(barrier_data);
1205 if (load->Opcode() == Op_LoadN) {
1206 Node* ld = gvn.transform(load);
1207 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1208 }
1209
1210 return load;
1211 }
1212
1213 //------------------------------hash-------------------------------------------
1214 uint LoadNode::hash() const {
1215 // unroll addition of interesting fields
1216 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1217 }
1218
1219 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1220 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1221 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1222 bool is_stable_ary = FoldStableValues &&
1223 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1224 tp->isa_aryptr()->is_stable();
1225
1226 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1227 }
1228
1229 return false;
1230 }
1231
1232 // Is the value loaded previously stored by an arraycopy? If so return
1233 // a load node that reads from the source array so we may be able to
1234 // optimize out the ArrayCopy node later.
1235 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1236 Node* ld_adr = in(MemNode::Address);
1237 intptr_t ld_off = 0;
1238 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1239 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1240 if (ac != nullptr) {
1241 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1242
1243 Node* mem = ac->in(TypeFunc::Memory);
1244 Node* ctl = ac->in(0);
1245 Node* src = ac->in(ArrayCopyNode::Src);
1246
1254 if (ac->as_ArrayCopy()->is_clonebasic()) {
1255 assert(ld_alloc != nullptr, "need an alloc");
1256 assert(addp->is_AddP(), "address must be addp");
1257 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1258 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1259 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1260 addp->set_req(AddPNode::Base, src);
1261 addp->set_req(AddPNode::Address, src);
1262 } else {
1263 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1264 ac->as_ArrayCopy()->is_copyof_validated() ||
1265 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1266 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1267 addp->set_req(AddPNode::Base, src);
1268 addp->set_req(AddPNode::Address, src);
1269
1270 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1271 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1272 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1273
1274 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1275
1276 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1277 #ifdef _LP64
1278 diff = phase->transform(new ConvI2LNode(diff));
1279 #endif
1280 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1281
1282 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1283 addp->set_req(AddPNode::Offset, offset);
1284 }
1285 addp = phase->transform(addp);
1286 #ifdef ASSERT
1287 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1288 ld->_adr_type = adr_type;
1289 #endif
1290 ld->set_req(MemNode::Address, addp);
1291 ld->set_req(0, ctl);
1292 ld->set_req(MemNode::Memory, mem);
1293 return ld;
1294 }
1295 return nullptr;
1296 }
1297
1298 static Node* see_through_inline_type(PhaseValues* phase, const MemNode* load, Node* base, int offset) {
1299 if (!load->is_mismatched_access() && base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1300 InlineTypeNode* vt = base->as_InlineType();
1301 Node* value = vt->field_value_by_offset(offset, true);
1302 assert(value != nullptr, "must see some value");
1303 return value;
1304 }
1305
1306 return nullptr;
1307 }
1308
1309 // This routine exists to make sure this set of tests is done the same
1310 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1311 // will change the graph shape in a way which makes memory alive twice at the
1312 // same time (uses the Oracle model of aliasing), then some
1313 // LoadXNode::Identity will fold things back to the equivalence-class model
1314 // of aliasing.
1315 // This method may find an unencoded node instead of the corresponding encoded one.
1316 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1317 Node* ld_adr = in(MemNode::Address);
1318 intptr_t ld_off = 0;
1319 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1320 // Try to see through an InlineTypeNode
1321 // LoadN is special because the input is not compressed
1322 if (Opcode() != Op_LoadN) {
1323 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1324 if (value != nullptr) {
1325 return value;
1326 }
1327 }
1328
1329 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1330 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1331
1332 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1333 uint alias_idx = atp->index();
1334 Node* result = nullptr;
1335 Node* current = st;
1336 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1337 // this load. Stop once any other kind of node is encountered.
1338 //
1339 // In principle, folding a load is moving it up until it meets a matching store.
1340 //
1341 // store(ptr, v); store(ptr, v); store(ptr, v);
1342 // membar1; -> membar1; -> load(ptr);
1343 // membar2; load(ptr); membar1;
1344 // load(ptr); membar2; membar2;
1345 //
1346 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1347 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1348 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1428
1429 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1430 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1431 if (in_vt != out_vt) {
1432 return nullptr;
1433 }
1434 }
1435 return st->in(MemNode::ValueIn);
1436 }
1437
1438 // A load from a freshly-created object always returns zero.
1439 // (This can happen after LoadNode::Ideal resets the load's memory input
1440 // to find_captured_store, which returned InitializeNode::zero_memory.)
1441 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1442 (st->in(0) == ld_alloc) &&
1443 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1444 // return a zero value for the load's basic type
1445 // (This is one of the few places where a generic PhaseTransform
1446 // can create new nodes. Think of it as lazily manifesting
1447 // virtually pre-existing constants.)
1448 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1449 if (init_value != nullptr) {
1450 const TypeAryPtr* ld_adr_type = phase->type(ld_adr)->isa_aryptr();
1451 if (ld_adr_type == nullptr) {
1452 return nullptr;
1453 }
1454
1455 // We know that this is not a flat array, the load should return the whole oop
1456 if (ld_adr_type->is_not_flat()) {
1457 return init_value;
1458 }
1459
1460 // If this is a flat array, try to see through init_value
1461 if (init_value->is_EncodeP()) {
1462 init_value = init_value->in(1);
1463 }
1464 if (!init_value->is_InlineType() || ld_adr_type->field_offset() == Type::Offset::bottom) {
1465 return nullptr;
1466 }
1467
1468 ciInlineKlass* vk = phase->type(init_value)->inline_klass();
1469 int field_offset_in_payload = ld_adr_type->field_offset().get();
1470 if (field_offset_in_payload == vk->null_marker_offset_in_payload()) {
1471 return init_value->as_InlineType()->get_null_marker();
1472 } else {
1473 return init_value->as_InlineType()->field_value_by_offset(field_offset_in_payload + vk->payload_offset(), true);
1474 }
1475 }
1476 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1477 if (value_basic_type() != T_VOID) {
1478 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1479 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1480 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1481 // by the ArrayCopyNode.
1482 return phase->zerocon(value_basic_type());
1483 }
1484 } else {
1485 // TODO: materialize all-zero vector constant
1486 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1487 }
1488 }
1489
1490 // A load from an initialization barrier can match a captured store.
1491 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1492 InitializeNode* init = st->in(0)->as_Initialize();
1493 AllocateNode* alloc = init->allocation();
1494 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1495 // examine a captured store value
1496 st = init->find_captured_store(ld_off, memory_size(), phase);
1509 base = bs->step_over_gc_barrier(base);
1510 if (base != nullptr && base->is_Proj() &&
1511 base->as_Proj()->_con == TypeFunc::Parms &&
1512 base->in(0)->is_CallStaticJava() &&
1513 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1514 return base->in(0)->in(TypeFunc::Parms);
1515 }
1516 }
1517
1518 break;
1519 }
1520
1521 return nullptr;
1522 }
1523
1524 //----------------------is_instance_field_load_with_local_phi------------------
1525 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1526 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1527 in(Address)->is_AddP() ) {
1528 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1529 // Only known instances and immutable fields
1530 if( t_oop != nullptr &&
1531 (t_oop->is_ptr_to_strict_final_field() ||
1532 t_oop->is_known_instance_field()) &&
1533 t_oop->offset() != Type::OffsetBot &&
1534 t_oop->offset() != Type::OffsetTop) {
1535 return true;
1536 }
1537 }
1538 return false;
1539 }
1540
1541 //------------------------------Identity---------------------------------------
1542 // Loads are identity if previous store is to same address
1543 Node* LoadNode::Identity(PhaseGVN* phase) {
1544 // If the previous store-maker is the right kind of Store, and the store is
1545 // to the same address, then we are equal to the value stored.
1546 Node* mem = in(Memory);
1547 Node* value = can_see_stored_value_through_membars(mem, phase);
1548 if( value ) {
1549 // byte, short & char stores truncate naturally.
1550 // A load has to load the truncated value which requires
1551 // some sort of masking operation and that requires an
1552 // Ideal call instead of an Identity call.
1553 if (memory_size() < BytesPerInt) {
1554 // If the input to the store does not fit with the load's result type,
1555 // it must be truncated via an Ideal call.
1556 if (!phase->type(value)->higher_equal(phase->type(this)))
1557 return this;
1558 }
1559
1560 if (phase->type(value)->isa_ptr() && phase->type(this)->isa_narrowoop()) {
1561 return this;
1562 }
1563 // (This works even when value is a Con, but LoadNode::Value
1564 // usually runs first, producing the singleton type of the Con.)
1565 if (!has_pinned_control_dependency() || value->is_Con()) {
1566 return value;
1567 } else {
1568 return this;
1569 }
1570 }
1571
1572 if (has_pinned_control_dependency()) {
1573 return this;
1574 }
1575 // Search for an existing data phi which was generated before for the same
1576 // instance's field to avoid infinite generation of phis in a loop.
1577 Node *region = mem->in(0);
1578 if (is_instance_field_load_with_local_phi(region)) {
1579 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1580 int this_index = phase->C->get_alias_index(addr_t);
1581 int this_offset = addr_t->offset();
1582 int this_iid = addr_t->instance_id();
1583 if (!addr_t->is_known_instance() &&
1584 addr_t->is_ptr_to_strict_final_field()) {
1585 // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1586 intptr_t ignore = 0;
1587 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1588 if (base == nullptr) {
1589 return this;
1590 }
1591 this_iid = base->_idx;
1592 }
1593 const Type* this_type = bottom_type();
1594 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1595 Node* phi = region->fast_out(i);
1596 if (phi->is_Phi() && phi != mem &&
1597 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1598 return phi;
1599 }
1600 }
1601 }
1602
1603 return this;
1604 }
1605
2140 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2141 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2142
2143 // Skip up past a SafePoint control. Cannot do this for Stores because
2144 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2145 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2146 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2147 !addr_mark &&
2148 (depends_only_on_test() || has_unknown_control_dependency())) {
2149 ctrl = ctrl->in(0);
2150 set_req(MemNode::Control,ctrl);
2151 return this;
2152 }
2153
2154 intptr_t ignore = 0;
2155 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2156 if (base != nullptr
2157 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2158 // Check for useless control edge in some common special cases
2159 if (in(MemNode::Control) != nullptr
2160 // TODO 8350865 Can we re-enable this?
2161 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2162 && can_remove_control()
2163 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2164 && all_controls_dominate(base, phase->C->start(), phase)) {
2165 // A method-invariant, non-null address (constant or 'this' argument).
2166 set_req(MemNode::Control, nullptr);
2167 return this;
2168 }
2169 }
2170
2171 Node* mem = in(MemNode::Memory);
2172 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2173
2174 if (can_reshape && (addr_t != nullptr)) {
2175 // try to optimize our memory input
2176 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2177 if (opt_mem != mem) {
2178 set_req_X(MemNode::Memory, opt_mem, phase);
2179 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2180 return this;
2181 }
2294 // No match.
2295 return nullptr;
2296 }
2297
2298 //------------------------------Value-----------------------------------------
2299 const Type* LoadNode::Value(PhaseGVN* phase) const {
2300 // Either input is TOP ==> the result is TOP
2301 Node* mem = in(MemNode::Memory);
2302 const Type *t1 = phase->type(mem);
2303 if (t1 == Type::TOP) return Type::TOP;
2304 Node* adr = in(MemNode::Address);
2305 const TypePtr* tp = phase->type(adr)->isa_ptr();
2306 if (tp == nullptr || tp->empty()) return Type::TOP;
2307 int off = tp->offset();
2308 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2309 Compile* C = phase->C;
2310
2311 // If load can see a previous constant store, use that.
2312 Node* value = can_see_stored_value_through_membars(mem, phase);
2313 if (value != nullptr && value->is_Con()) {
2314 if (phase->type(value)->isa_ptr() && _type->isa_narrowoop()) {
2315 return phase->type(value)->make_narrowoop();
2316 } else {
2317 assert(value->bottom_type()->higher_equal(_type), "sanity");
2318 return phase->type(value);
2319 }
2320 }
2321 // Try to guess loaded type from pointer type
2322 if (tp->isa_aryptr()) {
2323 const TypeAryPtr* ary = tp->is_aryptr();
2324 const Type* t = ary->elem();
2325
2326 // Determine whether the reference is beyond the header or not, by comparing
2327 // the offset against the offset of the start of the array's data.
2328 // Different array types begin at slightly different offsets (12 vs. 16).
2329 // We choose T_BYTE as an example base type that is least restrictive
2330 // as to alignment, which will therefore produce the smallest
2331 // possible base offset.
2332 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2333 const bool off_beyond_header = (off >= min_base_off);
2334
2335 // Try to constant-fold a stable array element.
2336 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2337 // Make sure the reference is not into the header and the offset is constant
2338 ciObject* aobj = ary->const_oop();
2339 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2340 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2341 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off, ary->field_offset().get(),
2342 stable_dimension,
2343 value_basic_type(), is_unsigned());
2344 if (con_type != nullptr) {
2345 return con_type;
2346 }
2347 }
2348 }
2349
2350 // Don't do this for integer types. There is only potential profit if
2351 // the element type t is lower than _type; that is, for int types, if _type is
2352 // more restrictive than t. This only happens here if one is short and the other
2353 // char (both 16 bits), and in those cases we've made an intentional decision
2354 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2355 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2356 //
2357 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2358 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2359 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2360 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2361 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2362 // In fact, that could have been the original type of p1, and p1 could have
2363 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2364 // expression (LShiftL quux 3) independently optimized to the constant 8.
2365 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2366 && (_type->isa_vect() == nullptr)
2367 && !ary->is_flat()
2368 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2369 // t might actually be lower than _type, if _type is a unique
2370 // concrete subclass of abstract class t.
2371 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2372 const Type* jt = t->join_speculative(_type);
2373 // In any case, do not allow the join, per se, to empty out the type.
2374 if (jt->empty() && !t->empty()) {
2375 // This can happen if a interface-typed array narrows to a class type.
2376 jt = _type;
2377 }
2378 #ifdef ASSERT
2379 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2380 // The pointers in the autobox arrays are always non-null
2381 Node* base = adr->in(AddPNode::Base);
2382 if ((base != nullptr) && base->is_DecodeN()) {
2383 // Get LoadN node which loads IntegerCache.cache field
2384 base = base->in(1);
2385 }
2386 if ((base != nullptr) && base->is_Con()) {
2387 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2388 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2389 // It could be narrow oop
2390 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2391 }
2392 }
2393 }
2394 #endif
2395 return jt;
2396 }
2397 }
2398 } else if (tp->base() == Type::InstPtr) {
2399 assert( off != Type::OffsetBot ||
2400 // arrays can be cast to Objects
2401 !tp->isa_instptr() ||
2402 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2403 // Default value load
2404 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2405 // unsafe field access may not have a constant offset
2406 is_unsafe_access(),
2407 "Field accesses must be precise" );
2408 // For oop loads, we expect the _type to be precise.
2409
2410 const TypeInstPtr* tinst = tp->is_instptr();
2411 BasicType bt = value_basic_type();
2412
2413 // Fold loads of the field map
2414 if (tinst != nullptr) {
2415 ciInstanceKlass* ik = tinst->instance_klass();
2416 int offset = tinst->offset();
2417 if (ik == phase->C->env()->Class_klass()) {
2418 ciType* t = tinst->java_mirror_type();
2419 if (t != nullptr && t->is_inlinetype() && offset == t->as_inline_klass()->field_map_offset()) {
2420 ciConstant map = t->as_inline_klass()->get_field_map();
2421 bool is_narrow_oop = (bt == T_NARROWOOP);
2422 return Type::make_from_constant(map, true, 1, is_narrow_oop);
2423 }
2424 }
2425 }
2426
2427 // Optimize loads from constant fields.
2428 ciObject* const_oop = tinst->const_oop();
2429 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2430 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2431 if (con_type != nullptr) {
2432 return con_type;
2433 }
2434 }
2435 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2436 assert(off != Type::OffsetBot ||
2437 !tp->isa_instklassptr() ||
2438 // arrays can be cast to Objects
2439 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2440 // also allow array-loading from the primary supertype
2441 // array during subtype checks
2442 Opcode() == Op_LoadKlass,
2443 "Field accesses must be precise");
2444 // For klass/static loads, we expect the _type to be precise
2445 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2446 /* With mirrors being an indirect in the Klass*
2447 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2448 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2449 *
2450 * So check the type and klass of the node before the LoadP.
2457 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2458 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2459 return TypeInstPtr::make(klass->java_mirror());
2460 }
2461 }
2462 }
2463
2464 const TypeKlassPtr *tkls = tp->isa_klassptr();
2465 if (tkls != nullptr) {
2466 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2467 ciKlass* klass = tkls->exact_klass();
2468 // We are loading a field from a Klass metaobject whose identity
2469 // is known at compile time (the type is "exact" or "precise").
2470 // Check for fields we know are maintained as constants by the VM.
2471 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2472 // The field is Klass::_super_check_offset. Return its (constant) value.
2473 // (Folds up type checking code.)
2474 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2475 return TypeInt::make(klass->super_check_offset());
2476 }
2477 if (klass->is_inlinetype() && tkls->offset() == in_bytes(InstanceKlass::acmp_maps_offset_offset())) {
2478 return TypeInt::make(klass->as_inline_klass()->field_map_offset());
2479 }
2480 if (klass->is_obj_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset())) {
2481 // Fold loads from LibraryCallKit::load_default_refined_array_klass
2482 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr();
2483 }
2484 if (klass->is_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::properties_offset())) {
2485 assert(klass->is_type_array_klass() || tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2486 return TypeInt::make((jint)klass->as_array_klass()->properties().value());
2487 }
2488 if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2489 assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2490 return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2491 }
2492 if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2493 // The field is Klass::_prototype_header. Return its (constant) value.
2494 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2495 return TypeX::make(klass->prototype_header());
2496 }
2497 // Compute index into primary_supers array
2498 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2499 // Check for overflowing; use unsigned compare to handle the negative case.
2500 if( depth < ciKlass::primary_super_limit() ) {
2501 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2502 // (Folds up type checking code.)
2503 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2504 ciKlass *ss = klass->super_of_depth(depth);
2505 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2506 }
2507 const Type* aift = load_array_final_field(tkls, klass);
2508 if (aift != nullptr) return aift;
2509 }
2510
2511 // We can still check if we are loading from the primary_supers array at a
2512 // shallow enough depth. Even though the klass is not exact, entries less
2513 // than or equal to its super depth are correct.
2514 if (tkls->is_loaded()) {
2515 ciKlass* klass = nullptr;
2549 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2550 // The key property of this type is that it folds up tests
2551 // for array-ness, since it proves that the layout_helper is positive.
2552 // Thus, a generic value like the basic object layout helper works fine.
2553 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2554 }
2555 }
2556
2557 // If we are loading from a freshly-allocated object/array, produce a zero.
2558 // Things to check:
2559 // 1. Load is beyond the header: headers are not guaranteed to be zero
2560 // 2. Load is not vectorized: vectors have no zero constant
2561 // 3. Load has no matching store, i.e. the input is the initial memory state
2562 const TypeOopPtr* tinst = tp->isa_oopptr();
2563 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2564 bool is_not_vect = (_type->isa_vect() == nullptr);
2565 if (is_not_header && is_not_vect) {
2566 Node* mem = in(MemNode::Memory);
2567 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2568 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2569 // TODO 8350865 Scalar replacement does not work well for flat arrays.
2570 // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2571 // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2572 // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2573 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2574 intptr_t offset = 0;
2575 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2576 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2577 if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2578 return _type;
2579 }
2580 }
2581 return Type::get_zero_type(_type->basic_type());
2582 }
2583 }
2584 if (!UseCompactObjectHeaders) {
2585 Node* alloc = is_new_object_mark_load();
2586 if (alloc != nullptr) {
2587 if (Arguments::is_valhalla_enabled()) {
2588 // The mark word may contain property bits (inline, flat, null-free)
2589 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2590 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2591 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2592 return TypeX::make(tkls->exact_klass()->prototype_header());
2593 }
2594 } else {
2595 return TypeX::make(markWord::prototype().value());
2596 }
2597 }
2598 }
2599
2600 return _type;
2601 }
2602
2603 //------------------------------match_edge-------------------------------------
2604 // Do we Match on this edge index or not? Match only the address.
2605 uint LoadNode::match_edge(uint idx) const {
2606 return idx == MemNode::Address;
2607 }
2608
2609 //--------------------------LoadBNode::Ideal--------------------------------------
2610 //
2611 // If the previous store is to the same address as this load,
2612 // and the value stored was larger than a byte, replace this load
2613 // with the value stored truncated to a byte. If no truncation is
2614 // needed, the replacement is done in LoadNode::Identity().
2615 //
2616 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2725 }
2726 }
2727 // Identity call will handle the case where truncation is not needed.
2728 return LoadNode::Ideal(phase, can_reshape);
2729 }
2730
2731 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2732 Node* mem = in(MemNode::Memory);
2733 Node* value = can_see_stored_value_through_membars(mem, phase);
2734 if (value != nullptr && value->is_Con() &&
2735 !value->bottom_type()->higher_equal(_type)) {
2736 // If the input to the store does not fit with the load's result type,
2737 // it must be truncated. We can't delay until Ideal call since
2738 // a singleton Value is needed for split_thru_phi optimization.
2739 int con = value->get_int();
2740 return TypeInt::make((con << 16) >> 16);
2741 }
2742 return LoadNode::Value(phase);
2743 }
2744
2745 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2746 // Loading from an InlineType, find the input and make an EncodeP
2747 Node* addr = in(Address);
2748 intptr_t offset;
2749 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
2750 Node* value = see_through_inline_type(phase, this, base, offset);
2751 if (value != nullptr) {
2752 return new EncodePNode(value, type());
2753 }
2754
2755 // Can see the corresponding value, may need to add an EncodeP
2756 value = can_see_stored_value(in(Memory), phase);
2757 if (value != nullptr && phase->type(value)->isa_ptr() && type()->isa_narrowoop()) {
2758 return new EncodePNode(value, type());
2759 }
2760
2761 // Identity call will handle the case where EncodeP is unnecessary
2762 return LoadNode::Ideal(phase, can_reshape);
2763 }
2764
2765 //=============================================================================
2766 //----------------------------LoadKlassNode::make------------------------------
2767 // Polymorphic factory method:
2768 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2769 // sanity check the alias category against the created node type
2770 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2771 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2772 #ifdef _LP64
2773 if (adr_type->is_ptr_to_narrowklass()) {
2774 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2775 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2776 }
2777 #endif
2778 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2779 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2780 }
2781
2782 //------------------------------Value------------------------------------------
2783 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2784 return klass_value_common(phase);
2817 }
2818 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2819 }
2820 if (!t->is_klass()) {
2821 // a primitive Class (e.g., int.class) has null for a klass field
2822 return TypePtr::NULL_PTR;
2823 }
2824 // Fold up the load of the hidden field
2825 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2826 }
2827 // non-constant mirror, so we can't tell what's going on
2828 }
2829 if (!tinst->is_loaded())
2830 return _type; // Bail out if not loaded
2831 if (offset == oopDesc::klass_offset_in_bytes()) {
2832 return tinst->as_klass_type(true);
2833 }
2834 }
2835
2836 // Check for loading klass from an array
2837 const TypeAryPtr* tary = tp->isa_aryptr();
2838 if (tary != nullptr &&
2839 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2840 return tary->as_klass_type(true)->is_aryklassptr();
2841 }
2842
2843 // Check for loading klass from an array klass
2844 const TypeKlassPtr *tkls = tp->isa_klassptr();
2845 if (tkls != nullptr && !StressReflectiveCode) {
2846 if (!tkls->is_loaded())
2847 return _type; // Bail out if not loaded
2848 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2849 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2850 // // Always returning precise element type is incorrect,
2851 // // e.g., element type could be object and array may contain strings
2852 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2853
2854 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2855 // according to the element type's subclassing.
2856 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2857 }
2858 if (tkls->isa_aryklassptr() != nullptr && tkls->klass_is_exact() &&
2859 !tkls->exact_klass()->is_type_array_klass() &&
2860 tkls->offset() == in_bytes(Klass::super_offset())) {
2861 // We are loading the super klass of a refined array klass, return the non-refined klass pointer
2862 assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2863 return tkls->is_aryklassptr()->with_offset(0)->cast_to_non_refined();
2864 }
2865 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2866 tkls->offset() == in_bytes(Klass::super_offset())) {
2867 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2868 // The field is Klass::_super. Return its (constant) value.
2869 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2870 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2871 }
2872 }
2873
2874 if (tkls != nullptr && !UseSecondarySupersCache
2875 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2876 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2877 return TypePtr::NULL_PTR;
2878 }
2879
2880 // Bailout case
2881 return LoadNode::Value(phase);
2882 }
2883
2884 //------------------------------Identity---------------------------------------
2907 base = bs->step_over_gc_barrier(base);
2908 }
2909
2910 // We can fetch the klass directly through an AllocateNode.
2911 // This works even if the klass is not constant (clone or newArray).
2912 if (offset == oopDesc::klass_offset_in_bytes()) {
2913 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2914 if (allocated_klass != nullptr) {
2915 return allocated_klass;
2916 }
2917 }
2918
2919 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2920 // See inline_native_Class_query for occurrences of these patterns.
2921 // Java Example: x.getClass().isAssignableFrom(y)
2922 //
2923 // This improves reflective code, often making the Class
2924 // mirror go completely dead. (Current exception: Class
2925 // mirrors may appear in debug info, but we could clean them out by
2926 // introducing a new debug info operator for Klass.java_mirror).
2927 //
2928 // This optimization does not apply to arrays because if k is not a
2929 // constant, it was obtained via load_klass which returns the VM type
2930 // and '.java_mirror.as_klass' should return the Java type instead.
2931
2932 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2933 && offset == java_lang_Class::klass_offset()) {
2934 if (base->is_Load()) {
2935 Node* base2 = base->in(MemNode::Address);
2936 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2937 Node* adr2 = base2->in(MemNode::Address);
2938 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2939 if (tkls != nullptr && !tkls->empty()
2940 && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2941 && adr2->is_AddP()) {
2942 int mirror_field = in_bytes(Klass::java_mirror_offset());
2943 if (tkls->offset() == mirror_field) {
2944 #ifdef ASSERT
2945 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2946 assert(tkls2->offset() == 0, "not a load of java_mirror");
2947 #endif
2948 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2949 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2950 return adr2->in(AddPNode::Address);
2951 }
2952 }
2953 }
2954 }
2955 }
2956
2957 return this;
2958 }
2959
2960 LoadNode* LoadNode::clone_pinned() const {
2961 LoadNode* ld = clone()->as_Load();
3088 // Polymorphic factory method:
3089 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
3090 assert((mo == unordered || mo == release), "unexpected");
3091 Compile* C = gvn.C;
3092 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
3093 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
3094 ctl != nullptr, "raw memory operations should have control edge");
3095
3096 switch (bt) {
3097 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
3098 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
3099 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
3100 case T_CHAR:
3101 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
3102 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3103 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
3104 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3105 case T_METADATA:
3106 case T_ADDRESS:
3107 case T_OBJECT:
3108 case T_ARRAY:
3109 #ifdef _LP64
3110 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3111 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
3112 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
3113 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
3114 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
3115 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
3116 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
3117 }
3118 #endif
3119 {
3120 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
3121 }
3122 default:
3123 guarantee(false, "unexpected basic type %s", type2name(bt));
3124 return (StoreNode*)nullptr;
3125 }
3126 }
3127
3128 //--------------------------bottom_type----------------------------------------
3129 const Type *StoreNode::bottom_type() const {
3130 return Type::MEMORY;
3131 }
3132
3133 //------------------------------hash-------------------------------------------
3134 uint StoreNode::hash() const {
3135 // unroll addition of interesting fields
3136 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
3137
3138 // Since they are not commoned, do not hash them:
3139 return NO_HASH;
3140 }
3141
3142 // Link together multiple stores (B/S/C/I) into a longer one.
3143 //
3765 }
3766 ss.print_cr("[TraceMergeStores]: with");
3767 merged_input_value->dump("\n", false, &ss);
3768 merged_store->dump("\n", false, &ss);
3769 tty->print("%s", ss.as_string());
3770 }
3771 #endif
3772
3773 //------------------------------Ideal------------------------------------------
3774 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3775 // When a store immediately follows a relevant allocation/initialization,
3776 // try to capture it into the initialization, or hoist it above.
3777 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3778 Node* p = MemNode::Ideal_common(phase, can_reshape);
3779 if (p) return (p == NodeSentinel) ? nullptr : p;
3780
3781 Node* mem = in(MemNode::Memory);
3782 Node* address = in(MemNode::Address);
3783 Node* value = in(MemNode::ValueIn);
3784 // Back-to-back stores to same address? Fold em up. Generally
3785 // unsafe if I have intervening uses...
3786 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3787 Node* st = mem;
3788 // If Store 'st' has more than one use, we cannot fold 'st' away.
3789 // For example, 'st' might be the final state at a conditional
3790 // return. Or, 'st' might be used by some node which is live at
3791 // the same time 'st' is live, which might be unschedulable. So,
3792 // require exactly ONE user until such time as we clone 'mem' for
3793 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3794 // true).
3795 while (st->is_Store() && st->outcnt() == 1) {
3796 // Looking at a dead closed cycle of memory?
3797 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3798 assert(Opcode() == st->Opcode() ||
3799 st->Opcode() == Op_StoreVector ||
3800 Opcode() == Op_StoreVector ||
3801 st->Opcode() == Op_StoreVectorScatter ||
3802 Opcode() == Op_StoreVectorScatter ||
3803 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3804 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3805 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3806 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3807 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3808 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3809
3810 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3811 st->as_Store()->memory_size() <= this->memory_size()) {
3812 Node* use = st->raw_out(0);
3813 if (phase->is_IterGVN()) {
3814 phase->is_IterGVN()->rehash_node_delayed(use);
3815 }
3816 // It's OK to do this in the parser, since DU info is always accurate,
3817 // and the parser always refers to nodes via SafePointNode maps.
3818 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3819 return this;
3820 }
3821 st = st->in(MemNode::Memory);
3822 }
3823 }
3824
3825
3826 // Capture an unaliased, unconditional, simple store into an initializer.
3927 const StoreVectorNode* store_vector = as_StoreVector();
3928 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3929 const Node* store_indices = store_vector->indices();
3930 const Node* mem_indices = mem_vector->indices();
3931 const Node* store_mask = store_vector->mask();
3932 const Node* mem_mask = mem_vector->mask();
3933 // Ensure types, indices, and masks match
3934 if (store_vector->vect_type() == mem_vector->vect_type() &&
3935 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3936 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3937 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3938 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3939 result = mem;
3940 }
3941 }
3942 }
3943
3944 // Store of zero anywhere into a freshly-allocated object?
3945 // Then the store is useless.
3946 // (It must already have been captured by the InitializeNode.)
3947 if (result == this && ReduceFieldZeroing) {
3948 // a newly allocated object is already all-zeroes everywhere
3949 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3950 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3951 result = mem;
3952 }
3953
3954 if (result == this && phase->type(val)->is_zero_type()) {
3955 // the store may also apply to zero-bits in an earlier object
3956 Node* prev_mem = find_previous_store(phase);
3957 // Steps (a), (b): Walk past independent stores to find an exact match.
3958 if (prev_mem != nullptr) {
3959 if (prev_mem->is_top()) {
3960 // find_previous_store returns top when the access is dead
3961 return prev_mem;
3962 }
3963 Node* prev_val = can_see_stored_value(prev_mem, phase);
3964 if (prev_val != nullptr && prev_val == val) {
3965 // prev_val and val might differ by a cast; it would be good
3966 // to keep the more informative of the two.
3967 result = mem;
3968 }
3969 }
3970 }
3971 }
3972
3973 PhaseIterGVN* igvn = phase->is_IterGVN();
3974 if (result != this && igvn != nullptr) {
4447 // Clearing a short array is faster with stores
4448 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4449 // Already know this is a large node, do not try to ideal it
4450 if (_is_large) return nullptr;
4451
4452 const int unit = BytesPerLong;
4453 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4454 if (!t) return nullptr;
4455 if (!t->is_con()) return nullptr;
4456 intptr_t raw_count = t->get_con();
4457 intptr_t size = raw_count;
4458 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4459 // Clearing nothing uses the Identity call.
4460 // Negative clears are possible on dead ClearArrays
4461 // (see jck test stmt114.stmt11402.val).
4462 if (size <= 0 || size % unit != 0) return nullptr;
4463 intptr_t count = size / unit;
4464 // Length too long; communicate this to matchers and assemblers.
4465 // Assemblers are responsible to produce fast hardware clears for it.
4466 if (size > InitArrayShortSize) {
4467 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4468 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4469 return nullptr;
4470 }
4471 if (!IdealizeClearArrayNode) return nullptr;
4472 Node *mem = in(1);
4473 if( phase->type(mem)==Type::TOP ) return nullptr;
4474 Node *adr = in(3);
4475 const Type* at = phase->type(adr);
4476 if( at==Type::TOP ) return nullptr;
4477 const TypePtr* atp = at->isa_ptr();
4478 // adjust atp to be the correct array element address type
4479 if (atp == nullptr) atp = TypePtr::BOTTOM;
4480 else atp = atp->add_offset(Type::OffsetBot);
4481 // Get base for derived pointer purposes
4482 if( adr->Opcode() != Op_AddP ) Unimplemented();
4483 Node *base = adr->in(1);
4484
4485 Node *val = in(4);
4486 Node *off = phase->MakeConX(BytesPerLong);
4487 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4488 count--;
4489 while (count--) {
4490 mem = phase->transform(mem);
4491 adr = phase->transform(AddPNode::make_with_base(base,adr,off));
4492 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4493 }
4494 return mem;
4495 }
4496
4497 //----------------------------step_through----------------------------------
4498 // Return allocation input memory edge if it is different instance
4499 // or itself if it is the one we are looking for.
4500 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4501 Node* n = *np;
4502 assert(n->is_ClearArray(), "sanity");
4503 intptr_t offset;
4504 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4505 // This method is called only before Allocate nodes are expanded
4506 // during macro nodes expansion. Before that ClearArray nodes are
4507 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4508 // Allocate nodes are expanded) which follows allocations.
4509 assert(alloc != nullptr, "should have allocation");
4510 if (alloc->_idx == instance_id) {
4511 // Can not bypass initialization of the instance we are looking for.
4512 return false;
4515 InitializeNode* init = alloc->initialization();
4516 if (init != nullptr)
4517 *np = init->in(TypeFunc::Memory);
4518 else
4519 *np = alloc->in(TypeFunc::Memory);
4520 return true;
4521 }
4522
4523 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4524 Node* base = dest;
4525 if (raw_base) {
4526 // May be called as part of the initialization of a just allocated object
4527 base = phase->C->top();
4528 }
4529 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4530 }
4531
4532 //----------------------------clear_memory-------------------------------------
4533 // Generate code to initialize object storage to zero.
4534 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4535 Node* val,
4536 Node* raw_val,
4537 intptr_t start_offset,
4538 Node* end_offset,
4539 bool raw_base,
4540 PhaseGVN* phase) {
4541 intptr_t offset = start_offset;
4542
4543 int unit = BytesPerLong;
4544 if ((offset % unit) != 0) {
4545 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4546 const TypePtr* atp = TypeRawPtr::BOTTOM;
4547 if (val != nullptr) {
4548 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4549 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4550 } else {
4551 assert(raw_val == nullptr, "val may not be null");
4552 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4553 }
4554 mem = phase->transform(mem);
4555 offset += BytesPerInt;
4556 }
4557 assert((offset % unit) == 0, "");
4558
4559 // Initialize the remaining stuff, if any, with a ClearArray.
4560 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, raw_base, phase);
4561 }
4562
4563 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4564 Node* raw_val,
4565 Node* start_offset,
4566 Node* end_offset,
4567 bool raw_base,
4568 PhaseGVN* phase) {
4569 if (start_offset == end_offset) {
4570 // nothing to do
4571 return mem;
4572 }
4573
4574 int unit = BytesPerLong;
4575 Node* zbase = start_offset;
4576 Node* zend = end_offset;
4577
4578 // Scale to the unit required by the CPU:
4579 if (!Matcher::init_array_count_is_in_bytes) {
4580 Node* shift = phase->intcon(exact_log2(unit));
4581 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4582 zend = phase->transform(new URShiftXNode(zend, shift) );
4583 }
4584
4585 // Bulk clear double-words
4586 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4587 Node* adr = make_address(dest, start_offset, raw_base, phase);
4588 if (raw_val == nullptr) {
4589 raw_val = phase->MakeConX(0);
4590 }
4591 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4592 return phase->transform(mem);
4593 }
4594
4595 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4596 Node* val,
4597 Node* raw_val,
4598 intptr_t start_offset,
4599 intptr_t end_offset,
4600 bool raw_base,
4601 PhaseGVN* phase) {
4602 if (start_offset == end_offset) {
4603 // nothing to do
4604 return mem;
4605 }
4606
4607 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4608 intptr_t done_offset = end_offset;
4609 if ((done_offset % BytesPerLong) != 0) {
4610 done_offset -= BytesPerInt;
4611 }
4612 if (done_offset > start_offset) {
4613 mem = clear_memory(ctl, mem, dest, val, raw_val,
4614 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4615 }
4616 if (done_offset < end_offset) { // emit the final 32-bit store
4617 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4618 const TypePtr* atp = TypeRawPtr::BOTTOM;
4619 if (val != nullptr) {
4620 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4621 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4622 } else {
4623 assert(raw_val == nullptr, "val may not be null");
4624 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4625 }
4626 mem = phase->transform(mem);
4627 done_offset += BytesPerInt;
4628 }
4629 assert(done_offset == end_offset, "");
4630 return mem;
4631 }
4632
4633 //=============================================================================
4634 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4635 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4636 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4637 #ifdef ASSERT
4638 , _pair_idx(0)
4639 #endif
4640 {
4641 init_class_id(Class_MemBar);
4642 Node* top = C->top();
4643 init_req(TypeFunc::I_O,top);
4644 init_req(TypeFunc::FramePtr,top);
4645 init_req(TypeFunc::ReturnAdr,top);
4754 PhaseIterGVN* igvn = phase->is_IterGVN();
4755 remove(igvn);
4756 // Must return either the original node (now dead) or a new node
4757 // (Do not return a top here, since that would break the uniqueness of top.)
4758 return new ConINode(TypeInt::ZERO);
4759 }
4760 }
4761 return progress ? this : nullptr;
4762 }
4763
4764 //------------------------------Value------------------------------------------
4765 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4766 if( !in(0) ) return Type::TOP;
4767 if( phase->type(in(0)) == Type::TOP )
4768 return Type::TOP;
4769 return TypeTuple::MEMBAR;
4770 }
4771
4772 //------------------------------match------------------------------------------
4773 // Construct projections for memory.
4774 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4775 switch (proj->_con) {
4776 case TypeFunc::Control:
4777 case TypeFunc::Memory:
4778 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4779 }
4780 ShouldNotReachHere();
4781 return nullptr;
4782 }
4783
4784 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4785 trailing->_kind = TrailingStore;
4786 leading->_kind = LeadingStore;
4787 #ifdef ASSERT
4788 trailing->_pair_idx = leading->_idx;
4789 leading->_pair_idx = leading->_idx;
4790 #endif
4791 }
4792
4793 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4794 trailing->_kind = TrailingLoadStore;
5041 return (req() > RawStores);
5042 }
5043
5044 void InitializeNode::set_complete(PhaseGVN* phase) {
5045 assert(!is_complete(), "caller responsibility");
5046 _is_complete = Complete;
5047
5048 // After this node is complete, it contains a bunch of
5049 // raw-memory initializations. There is no need for
5050 // it to have anything to do with non-raw memory effects.
5051 // Therefore, tell all non-raw users to re-optimize themselves,
5052 // after skipping the memory effects of this initialization.
5053 PhaseIterGVN* igvn = phase->is_IterGVN();
5054 if (igvn) igvn->add_users_to_worklist(this);
5055 }
5056
5057 // convenience function
5058 // return false if the init contains any stores already
5059 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
5060 InitializeNode* init = initialization();
5061 if (init == nullptr || init->is_complete()) {
5062 return false;
5063 }
5064 init->remove_extra_zeroes();
5065 // for now, if this allocation has already collected any inits, bail:
5066 if (init->is_non_zero()) return false;
5067 init->set_complete(phase);
5068 return true;
5069 }
5070
5071 void InitializeNode::remove_extra_zeroes() {
5072 if (req() == RawStores) return;
5073 Node* zmem = zero_memory();
5074 uint fill = RawStores;
5075 for (uint i = fill; i < req(); i++) {
5076 Node* n = in(i);
5077 if (n->is_top() || n == zmem) continue; // skip
5078 if (fill < i) set_req(fill, n); // compact
5079 ++fill;
5080 }
5081 // delete any empty spaces created:
5082 while (fill < req()) {
5083 del_req(fill);
5227 // store node that we'd like to capture. We need to check
5228 // the uses of the MergeMemNode.
5229 mems.push(n);
5230 }
5231 } else if (n->is_Mem()) {
5232 Node* other_adr = n->in(MemNode::Address);
5233 if (other_adr == adr) {
5234 failed = true;
5235 break;
5236 } else {
5237 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5238 if (other_t_adr != nullptr) {
5239 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5240 if (other_alias_idx == alias_idx) {
5241 // A load from the same memory slice as the store right
5242 // after the InitializeNode. We check the control of the
5243 // object/array that is loaded from. If it's the same as
5244 // the store control then we cannot capture the store.
5245 assert(!n->is_Store(), "2 stores to same slice on same control?");
5246 Node* base = other_adr;
5247 if (base->is_Phi()) {
5248 // In rare case, base may be a PhiNode and it may read
5249 // the same memory slice between InitializeNode and store.
5250 failed = true;
5251 break;
5252 }
5253 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5254 base = base->in(AddPNode::Base);
5255 if (base != nullptr) {
5256 base = base->uncast();
5257 if (base->is_Proj() && base->in(0) == alloc) {
5258 failed = true;
5259 break;
5260 }
5261 }
5262 }
5263 }
5264 }
5265 } else {
5266 failed = true;
5267 break;
5268 }
5269 }
5270 }
5271 }
5272 if (failed) {
5818 // z's_done 12 16 16 16 12 16 12
5819 // z's_needed 12 16 16 16 16 16 16
5820 // zsize 0 0 0 0 4 0 4
5821 if (next_full_store < 0) {
5822 // Conservative tack: Zero to end of current word.
5823 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5824 } else {
5825 // Zero to beginning of next fully initialized word.
5826 // Or, don't zero at all, if we are already in that word.
5827 assert(next_full_store >= zeroes_needed, "must go forward");
5828 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5829 zeroes_needed = next_full_store;
5830 }
5831 }
5832
5833 if (zeroes_needed > zeroes_done) {
5834 intptr_t zsize = zeroes_needed - zeroes_done;
5835 // Do some incremental zeroing on rawmem, in parallel with inits.
5836 zeroes_done = align_down(zeroes_done, BytesPerInt);
5837 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5838 allocation()->in(AllocateNode::InitValue),
5839 allocation()->in(AllocateNode::RawInitValue),
5840 zeroes_done, zeroes_needed,
5841 true,
5842 phase);
5843 zeroes_done = zeroes_needed;
5844 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5845 do_zeroing = false; // leave the hole, next time
5846 }
5847 }
5848
5849 // Collect the store and move on:
5850 phase->replace_input_of(st, MemNode::Memory, inits);
5851 inits = st; // put it on the linearized chain
5852 set_req(i, zmem); // unhook from previous position
5853
5854 if (zeroes_done == st_off)
5855 zeroes_done = next_init_off;
5856
5857 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5858
5859 #ifdef ASSERT
5880 remove_extra_zeroes(); // clear out all the zmems left over
5881 add_req(inits);
5882
5883 if (!(UseTLAB && ZeroTLAB)) {
5884 // If anything remains to be zeroed, zero it all now.
5885 zeroes_done = align_down(zeroes_done, BytesPerInt);
5886 // if it is the last unused 4 bytes of an instance, forget about it
5887 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5888 if (zeroes_done + BytesPerLong >= size_limit) {
5889 AllocateNode* alloc = allocation();
5890 assert(alloc != nullptr, "must be present");
5891 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5892 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5893 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5894 if (zeroes_done == k->layout_helper())
5895 zeroes_done = size_limit;
5896 }
5897 }
5898 if (zeroes_done < size_limit) {
5899 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5900 allocation()->in(AllocateNode::InitValue),
5901 allocation()->in(AllocateNode::RawInitValue),
5902 zeroes_done, size_in_bytes, true, phase);
5903 }
5904 }
5905
5906 set_complete(phase);
5907 return rawmem;
5908 }
5909
5910 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5911 auto replace_proj = [&](ProjNode* proj) {
5912 C->gvn_replace_by(proj, mem);
5913 return CONTINUE;
5914 };
5915 apply_to_projs(replace_proj, TypeFunc::Memory);
5916 }
5917
5918 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5919 DUIterator_Fast imax, i = fast_outs(imax);
5920 auto replace_proj = [&](ProjNode* proj) {
5921 igvn->replace_node(proj, mem);
|