6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/compile.hpp"
38 #include "opto/connode.hpp"
39 #include "opto/convertnode.hpp"
40 #include "opto/loopnode.hpp"
41 #include "opto/machnode.hpp"
42 #include "opto/matcher.hpp"
43 #include "opto/memnode.hpp"
44 #include "opto/mempointer.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/phaseX.hpp"
48 #include "opto/regalloc.hpp"
49 #include "opto/regmask.hpp"
50 #include "opto/rootnode.hpp"
51 #include "opto/traceMergeStoresTag.hpp"
52 #include "opto/vectornode.hpp"
53 #include "utilities/align.hpp"
54 #include "utilities/copy.hpp"
55 #include "utilities/macros.hpp"
56 #include "utilities/powerOfTwo.hpp"
57 #include "utilities/vmError.hpp"
58
59 // Portions of code courtesy of Clifford Click
60
61 // Optimization - Graph Style
62
63 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
64
65 //=============================================================================
66 uint MemNode::size_of() const { return sizeof(*this); }
67
68 const TypePtr *MemNode::adr_type() const {
69 Node* adr = in(Address);
70 if (adr == nullptr) return nullptr; // node is dead
71 const TypePtr* cross_check = nullptr;
72 DEBUG_ONLY(cross_check = _adr_type);
73 return calculate_adr_type(adr->bottom_type(), cross_check);
74 }
123 st->print(", idx=Bot;");
124 else if (atp->index() == Compile::AliasIdxTop)
125 st->print(", idx=Top;");
126 else if (atp->index() == Compile::AliasIdxRaw)
127 st->print(", idx=Raw;");
128 else {
129 ciField* field = atp->field();
130 if (field) {
131 st->print(", name=");
132 field->print_name_on(st);
133 }
134 st->print(", idx=%d;", atp->index());
135 }
136 }
137 }
138
139 extern void print_alias_types();
140
141 #endif
142
143 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
144 assert((t_oop != nullptr), "sanity");
145 bool is_instance = t_oop->is_known_instance_field();
146 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
147 (load != nullptr) && load->is_Load() &&
148 (phase->is_IterGVN() != nullptr);
149 if (!(is_instance || is_boxed_value_load))
150 return mchain; // don't try to optimize non-instance types
151 uint instance_id = t_oop->instance_id();
152 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
153 Node *prev = nullptr;
154 Node *result = mchain;
155 while (prev != result) {
156 prev = result;
157 if (result == start_mem)
158 break; // hit one of our sentinels
159 // skip over a call which does not affect this memory slice
160 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
161 Node *proj_in = result->in(0);
162 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
163 break; // hit one of our sentinels
164 } else if (proj_in->is_Call()) {
165 // ArrayCopyNodes processed here as well
166 CallNode *call = proj_in->as_Call();
167 if (!call->may_modify(t_oop, phase)) { // returns false for instances
168 result = call->in(TypeFunc::Memory);
169 }
170 } else if (proj_in->is_Initialize()) {
171 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
172 // Stop if this is the initialization for the object instance which
173 // contains this memory slice, otherwise skip over it.
174 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
175 break;
176 }
177 if (is_instance) {
178 result = proj_in->in(TypeFunc::Memory);
179 } else if (is_boxed_value_load) {
180 Node* klass = alloc->in(AllocateNode::KlassNode);
181 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
182 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
183 result = proj_in->in(TypeFunc::Memory); // not related allocation
184 }
185 }
186 } else if (proj_in->is_MemBar()) {
187 ArrayCopyNode* ac = nullptr;
188 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
189 break;
190 }
191 result = proj_in->in(TypeFunc::Memory);
192 } else if (proj_in->is_top()) {
193 break; // dead code
194 } else {
195 assert(false, "unexpected projection");
196 }
197 } else if (result->is_ClearArray()) {
198 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
199 // Can not bypass initialization of the instance
200 // we are looking for.
201 break;
202 }
203 // Otherwise skip it (the call updated 'result' value).
204 } else if (result->is_MergeMem()) {
205 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
206 }
207 }
208 return result;
209 }
210
211 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
212 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
213 if (t_oop == nullptr)
214 return mchain; // don't try to optimize non-oop types
215 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
216 bool is_instance = t_oop->is_known_instance_field();
217 PhaseIterGVN *igvn = phase->is_IterGVN();
218 if (is_instance && igvn != nullptr && result->is_Phi()) {
219 PhiNode *mphi = result->as_Phi();
220 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
221 const TypePtr *t = mphi->adr_type();
222 bool do_split = false;
223 // In the following cases, Load memory input can be further optimized based on
224 // its precise address type
225 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
226 do_split = true;
227 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
228 const TypeOopPtr* mem_t =
229 t->is_oopptr()->cast_to_exactness(true)
230 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
231 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
232 if (t_oop->isa_aryptr()) {
233 mem_t = mem_t->is_aryptr()
234 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
235 ->cast_to_size(t_oop->is_aryptr()->size())
236 ->with_offset(t_oop->is_aryptr()->offset())
237 ->is_aryptr();
238 }
239 do_split = mem_t == t_oop;
240 }
241 if (do_split) {
242 // clone the Phi with our address type
243 result = mphi->split_out_instance(t_adr, igvn);
244 } else {
245 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
246 }
247 }
248 return result;
249 }
250
251 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
252 uint alias_idx = phase->C->get_alias_index(tp);
253 Node *mem = mmem;
254 #ifdef ASSERT
255 {
256 // Check that current type is consistent with the alias index used during graph construction
257 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
258 bool consistent = adr_check == nullptr || adr_check->empty() ||
259 phase->C->must_alias(adr_check, alias_idx );
260 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
261 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
262 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
263 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
264 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
265 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
266 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
267 // don't assert if it is dead code.
268 consistent = true;
269 }
270 if( !consistent ) {
271 st->print("alias_idx==%d, adr_check==", alias_idx);
272 if( adr_check == nullptr ) {
273 st->print("null");
274 } else {
275 adr_check->dump();
276 }
277 st->cr();
278 print_alias_types();
279 assert(consistent, "adr_check must match alias idx");
280 }
281 }
282 #endif
954 "use LoadKlassNode instead");
955 assert(!(adr_type->isa_aryptr() &&
956 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
957 "use LoadRangeNode instead");
958 // Check control edge of raw loads
959 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
960 // oop will be recorded in oop map if load crosses safepoint
961 rt->isa_oopptr() || is_immutable_value(adr),
962 "raw memory operations should have control edge");
963 LoadNode* load = nullptr;
964 switch (bt) {
965 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
966 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
967 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
968 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
969 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
970 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
971 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
972 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
973 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
974 case T_OBJECT:
975 case T_NARROWOOP:
976 #ifdef _LP64
977 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
978 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
979 } else
980 #endif
981 {
982 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
983 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
984 }
985 break;
986 default:
987 ShouldNotReachHere();
988 break;
989 }
990 assert(load != nullptr, "LoadNode should have been created");
991 if (unaligned) {
992 load->set_unaligned_access();
993 }
994 if (mismatched) {
995 load->set_mismatched_access();
996 }
997 if (unsafe) {
998 load->set_unsafe_access();
999 }
1000 load->set_barrier_data(barrier_data);
1001 if (load->Opcode() == Op_LoadN) {
1002 Node* ld = gvn.transform(load);
1003 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1004 }
1005
1006 return load;
1007 }
1008
1009 //------------------------------hash-------------------------------------------
1010 uint LoadNode::hash() const {
1011 // unroll addition of interesting fields
1012 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1013 }
1014
1015 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1016 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1017 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1018 bool is_stable_ary = FoldStableValues &&
1019 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1020 tp->isa_aryptr()->is_stable();
1021
1022 return (eliminate_boxing && non_volatile) || is_stable_ary;
1023 }
1024
1025 return false;
1026 }
1027
1028 // Is the value loaded previously stored by an arraycopy? If so return
1029 // a load node that reads from the source array so we may be able to
1030 // optimize out the ArrayCopy node later.
1031 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1032 Node* ld_adr = in(MemNode::Address);
1033 intptr_t ld_off = 0;
1034 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1035 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1036 if (ac != nullptr) {
1037 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1038
1039 Node* mem = ac->in(TypeFunc::Memory);
1040 Node* ctl = ac->in(0);
1041 Node* src = ac->in(ArrayCopyNode::Src);
1042
1050 if (ac->as_ArrayCopy()->is_clonebasic()) {
1051 assert(ld_alloc != nullptr, "need an alloc");
1052 assert(addp->is_AddP(), "address must be addp");
1053 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1054 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1055 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1056 addp->set_req(AddPNode::Base, src);
1057 addp->set_req(AddPNode::Address, src);
1058 } else {
1059 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1060 ac->as_ArrayCopy()->is_copyof_validated() ||
1061 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1062 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1063 addp->set_req(AddPNode::Base, src);
1064 addp->set_req(AddPNode::Address, src);
1065
1066 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1067 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1068 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1069
1070 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1071 uint shift = exact_log2(type2aelembytes(ary_elem));
1072
1073 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1074 #ifdef _LP64
1075 diff = phase->transform(new ConvI2LNode(diff));
1076 #endif
1077 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1078
1079 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1080 addp->set_req(AddPNode::Offset, offset);
1081 }
1082 addp = phase->transform(addp);
1083 #ifdef ASSERT
1084 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1085 ld->_adr_type = adr_type;
1086 #endif
1087 ld->set_req(MemNode::Address, addp);
1088 ld->set_req(0, ctl);
1089 ld->set_req(MemNode::Memory, mem);
1090 return ld;
1091 }
1092 return nullptr;
1093 }
1094
1095
1096 //---------------------------can_see_stored_value------------------------------
1097 // This routine exists to make sure this set of tests is done the same
1098 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1099 // will change the graph shape in a way which makes memory alive twice at the
1100 // same time (uses the Oracle model of aliasing), then some
1101 // LoadXNode::Identity will fold things back to the equivalence-class model
1102 // of aliasing.
1103 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1104 Node* ld_adr = in(MemNode::Address);
1105 intptr_t ld_off = 0;
1106 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1107 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1108 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1109 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1110 // This is more general than load from boxing objects.
1111 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1112 uint alias_idx = atp->index();
1113 Node* result = nullptr;
1114 Node* current = st;
1115 // Skip through chains of MemBarNodes checking the MergeMems for
1116 // new states for the slice of this load. Stop once any other
1117 // kind of node is encountered. Loads from final memory can skip
1118 // through any kind of MemBar but normal loads shouldn't skip
1119 // through MemBarAcquire since the could allow them to move out of
1120 // a synchronized region. It is not safe to step over MemBarCPUOrder,
1121 // because alias info above them may be inaccurate (e.g., due to
1122 // mixed/mismatched unsafe accesses).
1123 bool is_final_mem = !atp->is_rewritable();
1124 while (current->is_Proj()) {
1125 int opc = current->in(0)->Opcode();
1126 if ((is_final_mem && (opc == Op_MemBarAcquire ||
1170 // Same base, same offset.
1171 // Possible improvement for arrays: check index value instead of absolute offset.
1172
1173 // At this point we have proven something like this setup:
1174 // B = << base >>
1175 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1176 // S = StoreQ(AddP( B , #Off), V)
1177 // (Actually, we haven't yet proven the Q's are the same.)
1178 // In other words, we are loading from a casted version of
1179 // the same pointer-and-offset that we stored to.
1180 // Casted version may carry a dependency and it is respected.
1181 // Thus, we are able to replace L by V.
1182 }
1183 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1184 if (store_Opcode() != st->Opcode()) {
1185 return nullptr;
1186 }
1187 // LoadVector/StoreVector needs additional check to ensure the types match.
1188 if (st->is_StoreVector()) {
1189 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1190 const TypeVect* out_vt = as_LoadVector()->vect_type();
1191 if (in_vt != out_vt) {
1192 return nullptr;
1193 }
1194 }
1195 return st->in(MemNode::ValueIn);
1196 }
1197
1198 // A load from a freshly-created object always returns zero.
1199 // (This can happen after LoadNode::Ideal resets the load's memory input
1200 // to find_captured_store, which returned InitializeNode::zero_memory.)
1201 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1202 (st->in(0) == ld_alloc) &&
1203 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1204 // return a zero value for the load's basic type
1205 // (This is one of the few places where a generic PhaseTransform
1206 // can create new nodes. Think of it as lazily manifesting
1207 // virtually pre-existing constants.)
1208 if (value_basic_type() != T_VOID) {
1209 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1210 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1211 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1212 // by the ArrayCopyNode.
1213 return phase->zerocon(value_basic_type());
1214 }
1215 } else {
1216 // TODO: materialize all-zero vector constant
1217 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1218 }
1219 }
1220
1221 // A load from an initialization barrier can match a captured store.
1222 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1223 InitializeNode* init = st->in(0)->as_Initialize();
1224 AllocateNode* alloc = init->allocation();
1225 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1226 // examine a captured store value
1227 st = init->find_captured_store(ld_off, memory_size(), phase);
1240 base = bs->step_over_gc_barrier(base);
1241 if (base != nullptr && base->is_Proj() &&
1242 base->as_Proj()->_con == TypeFunc::Parms &&
1243 base->in(0)->is_CallStaticJava() &&
1244 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1245 return base->in(0)->in(TypeFunc::Parms);
1246 }
1247 }
1248
1249 break;
1250 }
1251
1252 return nullptr;
1253 }
1254
1255 //----------------------is_instance_field_load_with_local_phi------------------
1256 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1257 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1258 in(Address)->is_AddP() ) {
1259 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1260 // Only instances and boxed values.
1261 if( t_oop != nullptr &&
1262 (t_oop->is_ptr_to_boxed_value() ||
1263 t_oop->is_known_instance_field()) &&
1264 t_oop->offset() != Type::OffsetBot &&
1265 t_oop->offset() != Type::OffsetTop) {
1266 return true;
1267 }
1268 }
1269 return false;
1270 }
1271
1272 //------------------------------Identity---------------------------------------
1273 // Loads are identity if previous store is to same address
1274 Node* LoadNode::Identity(PhaseGVN* phase) {
1275 // If the previous store-maker is the right kind of Store, and the store is
1276 // to the same address, then we are equal to the value stored.
1277 Node* mem = in(Memory);
1278 Node* value = can_see_stored_value(mem, phase);
1279 if( value ) {
1280 // byte, short & char stores truncate naturally.
1281 // A load has to load the truncated value which requires
1282 // some sort of masking operation and that requires an
1283 // Ideal call instead of an Identity call.
1284 if (memory_size() < BytesPerInt) {
1285 // If the input to the store does not fit with the load's result type,
1286 // it must be truncated via an Ideal call.
1287 if (!phase->type(value)->higher_equal(phase->type(this)))
1288 return this;
1289 }
1290 // (This works even when value is a Con, but LoadNode::Value
1291 // usually runs first, producing the singleton type of the Con.)
1292 if (!has_pinned_control_dependency() || value->is_Con()) {
1293 return value;
1294 } else {
1295 return this;
1296 }
1297 }
1298
1299 if (has_pinned_control_dependency()) {
1300 return this;
1301 }
1302 // Search for an existing data phi which was generated before for the same
1303 // instance's field to avoid infinite generation of phis in a loop.
1304 Node *region = mem->in(0);
1305 if (is_instance_field_load_with_local_phi(region)) {
1306 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1307 int this_index = phase->C->get_alias_index(addr_t);
1308 int this_offset = addr_t->offset();
1309 int this_iid = addr_t->instance_id();
1310 if (!addr_t->is_known_instance() &&
1311 addr_t->is_ptr_to_boxed_value()) {
1312 // Use _idx of address base (could be Phi node) for boxed values.
1313 intptr_t ignore = 0;
1314 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1315 if (base == nullptr) {
1316 return this;
1317 }
1318 this_iid = base->_idx;
1319 }
1320 const Type* this_type = bottom_type();
1321 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1322 Node* phi = region->fast_out(i);
1323 if (phi->is_Phi() && phi != mem &&
1324 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1325 return phi;
1326 }
1327 }
1328 }
1329
1330 return this;
1331 }
1332
1848 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1849 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1850
1851 // Skip up past a SafePoint control. Cannot do this for Stores because
1852 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1853 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1854 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1855 !addr_mark &&
1856 (depends_only_on_test() || has_unknown_control_dependency())) {
1857 ctrl = ctrl->in(0);
1858 set_req(MemNode::Control,ctrl);
1859 progress = true;
1860 }
1861
1862 intptr_t ignore = 0;
1863 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1864 if (base != nullptr
1865 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1866 // Check for useless control edge in some common special cases
1867 if (in(MemNode::Control) != nullptr
1868 && can_remove_control()
1869 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1870 && all_controls_dominate(base, phase->C->start())) {
1871 // A method-invariant, non-null address (constant or 'this' argument).
1872 set_req(MemNode::Control, nullptr);
1873 progress = true;
1874 }
1875 }
1876
1877 Node* mem = in(MemNode::Memory);
1878 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1879
1880 if (can_reshape && (addr_t != nullptr)) {
1881 // try to optimize our memory input
1882 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1883 if (opt_mem != mem) {
1884 set_req_X(MemNode::Memory, opt_mem, phase);
1885 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1886 return this;
1887 }
1944 // fold up, do so.
1945 Node* prev_mem = find_previous_store(phase);
1946 if (prev_mem != nullptr) {
1947 Node* value = can_see_arraycopy_value(prev_mem, phase);
1948 if (value != nullptr) {
1949 return value;
1950 }
1951 }
1952 // Steps (a), (b): Walk past independent stores to find an exact match.
1953 if (prev_mem != nullptr && prev_mem != in(MemNode::Memory)) {
1954 // (c) See if we can fold up on the spot, but don't fold up here.
1955 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
1956 // just return a prior value, which is done by Identity calls.
1957 if (can_see_stored_value(prev_mem, phase)) {
1958 // Make ready for step (d):
1959 set_req_X(MemNode::Memory, prev_mem, phase);
1960 return this;
1961 }
1962 }
1963
1964 return progress ? this : nullptr;
1965 }
1966
1967 // Helper to recognize certain Klass fields which are invariant across
1968 // some group of array types (e.g., int[] or all T[] where T < Object).
1969 const Type*
1970 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
1971 ciKlass* klass) const {
1972 assert(!UseCompactObjectHeaders || tkls->offset() != in_bytes(Klass::prototype_header_offset()),
1973 "must not happen");
1974
1975 if (tkls->isa_instklassptr() && tkls->offset() == in_bytes(InstanceKlass::access_flags_offset())) {
1976 // The field is InstanceKlass::_access_flags. Return its (constant) value.
1977 assert(Opcode() == Op_LoadUS, "must load an unsigned short from _access_flags");
1978 ciInstanceKlass* iklass = tkls->is_instklassptr()->instance_klass();
1979 return TypeInt::make(iklass->access_flags());
1980 }
1981 if (tkls->offset() == in_bytes(Klass::misc_flags_offset())) {
1982 // The field is Klass::_misc_flags. Return its (constant) value.
1983 assert(Opcode() == Op_LoadUB, "must load an unsigned byte from _misc_flags");
1984 return TypeInt::make(klass->misc_flags());
1992 // No match.
1993 return nullptr;
1994 }
1995
1996 //------------------------------Value-----------------------------------------
1997 const Type* LoadNode::Value(PhaseGVN* phase) const {
1998 // Either input is TOP ==> the result is TOP
1999 Node* mem = in(MemNode::Memory);
2000 const Type *t1 = phase->type(mem);
2001 if (t1 == Type::TOP) return Type::TOP;
2002 Node* adr = in(MemNode::Address);
2003 const TypePtr* tp = phase->type(adr)->isa_ptr();
2004 if (tp == nullptr || tp->empty()) return Type::TOP;
2005 int off = tp->offset();
2006 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2007 Compile* C = phase->C;
2008
2009 // If load can see a previous constant store, use that.
2010 Node* value = can_see_stored_value(mem, phase);
2011 if (value != nullptr && value->is_Con()) {
2012 assert(value->bottom_type()->higher_equal(_type), "sanity");
2013 return value->bottom_type();
2014 }
2015
2016 // Try to guess loaded type from pointer type
2017 if (tp->isa_aryptr()) {
2018 const TypeAryPtr* ary = tp->is_aryptr();
2019 const Type* t = ary->elem();
2020
2021 // Determine whether the reference is beyond the header or not, by comparing
2022 // the offset against the offset of the start of the array's data.
2023 // Different array types begin at slightly different offsets (12 vs. 16).
2024 // We choose T_BYTE as an example base type that is least restrictive
2025 // as to alignment, which will therefore produce the smallest
2026 // possible base offset.
2027 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2028 const bool off_beyond_header = (off >= min_base_off);
2029
2030 // Try to constant-fold a stable array element.
2031 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2032 // Make sure the reference is not into the header and the offset is constant
2033 ciObject* aobj = ary->const_oop();
2034 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2035 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2036 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off,
2037 stable_dimension,
2038 value_basic_type(), is_unsigned());
2039 if (con_type != nullptr) {
2040 return con_type;
2041 }
2042 }
2043 }
2044
2045 // Don't do this for integer types. There is only potential profit if
2046 // the element type t is lower than _type; that is, for int types, if _type is
2047 // more restrictive than t. This only happens here if one is short and the other
2048 // char (both 16 bits), and in those cases we've made an intentional decision
2049 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2050 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2051 //
2052 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2053 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2054 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2055 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2056 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2057 // In fact, that could have been the original type of p1, and p1 could have
2058 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2059 // expression (LShiftL quux 3) independently optimized to the constant 8.
2060 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2061 && (_type->isa_vect() == nullptr)
2062 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2063 // t might actually be lower than _type, if _type is a unique
2064 // concrete subclass of abstract class t.
2065 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2066 const Type* jt = t->join_speculative(_type);
2067 // In any case, do not allow the join, per se, to empty out the type.
2068 if (jt->empty() && !t->empty()) {
2069 // This can happen if a interface-typed array narrows to a class type.
2070 jt = _type;
2071 }
2072 #ifdef ASSERT
2073 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2074 // The pointers in the autobox arrays are always non-null
2075 Node* base = adr->in(AddPNode::Base);
2076 if ((base != nullptr) && base->is_DecodeN()) {
2077 // Get LoadN node which loads IntegerCache.cache field
2078 base = base->in(1);
2079 }
2080 if ((base != nullptr) && base->is_Con()) {
2081 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2082 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2083 // It could be narrow oop
2084 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2085 }
2086 }
2087 }
2088 #endif
2089 return jt;
2090 }
2091 }
2092 } else if (tp->base() == Type::InstPtr) {
2093 assert( off != Type::OffsetBot ||
2094 // arrays can be cast to Objects
2095 !tp->isa_instptr() ||
2096 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2097 // unsafe field access may not have a constant offset
2098 C->has_unsafe_access(),
2099 "Field accesses must be precise" );
2100 // For oop loads, we expect the _type to be precise.
2101
2102 // Optimize loads from constant fields.
2103 const TypeInstPtr* tinst = tp->is_instptr();
2104 ciObject* const_oop = tinst->const_oop();
2105 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2106 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2107 if (con_type != nullptr) {
2108 return con_type;
2109 }
2110 }
2111 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2112 assert(off != Type::OffsetBot ||
2113 !tp->isa_instklassptr() ||
2114 // arrays can be cast to Objects
2115 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2116 // also allow array-loading from the primary supertype
2117 // array during subtype checks
2118 Opcode() == Op_LoadKlass,
2119 "Field accesses must be precise");
2120 // For klass/static loads, we expect the _type to be precise
2121 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2122 /* With mirrors being an indirect in the Klass*
2123 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2124 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2125 *
2126 * So check the type and klass of the node before the LoadP.
2133 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2134 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2135 return TypeInstPtr::make(klass->java_mirror());
2136 }
2137 }
2138 }
2139
2140 const TypeKlassPtr *tkls = tp->isa_klassptr();
2141 if (tkls != nullptr) {
2142 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2143 ciKlass* klass = tkls->exact_klass();
2144 // We are loading a field from a Klass metaobject whose identity
2145 // is known at compile time (the type is "exact" or "precise").
2146 // Check for fields we know are maintained as constants by the VM.
2147 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2148 // The field is Klass::_super_check_offset. Return its (constant) value.
2149 // (Folds up type checking code.)
2150 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2151 return TypeInt::make(klass->super_check_offset());
2152 }
2153 if (UseCompactObjectHeaders) {
2154 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2155 // The field is Klass::_prototype_header. Return its (constant) value.
2156 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2157 return TypeX::make(klass->prototype_header());
2158 }
2159 }
2160 // Compute index into primary_supers array
2161 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2162 // Check for overflowing; use unsigned compare to handle the negative case.
2163 if( depth < ciKlass::primary_super_limit() ) {
2164 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2165 // (Folds up type checking code.)
2166 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2167 ciKlass *ss = klass->super_of_depth(depth);
2168 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2169 }
2170 const Type* aift = load_array_final_field(tkls, klass);
2171 if (aift != nullptr) return aift;
2172 }
2173
2174 // We can still check if we are loading from the primary_supers array at a
2175 // shallow enough depth. Even though the klass is not exact, entries less
2176 // than or equal to its super depth are correct.
2177 if (tkls->is_loaded()) {
2178 ciKlass* klass = nullptr;
2212 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2213 // The key property of this type is that it folds up tests
2214 // for array-ness, since it proves that the layout_helper is positive.
2215 // Thus, a generic value like the basic object layout helper works fine.
2216 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2217 }
2218 }
2219
2220 // If we are loading from a freshly-allocated object/array, produce a zero.
2221 // Things to check:
2222 // 1. Load is beyond the header: headers are not guaranteed to be zero
2223 // 2. Load is not vectorized: vectors have no zero constant
2224 // 3. Load has no matching store, i.e. the input is the initial memory state
2225 const TypeOopPtr* tinst = tp->isa_oopptr();
2226 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2227 bool is_not_vect = (_type->isa_vect() == nullptr);
2228 if (is_not_header && is_not_vect) {
2229 Node* mem = in(MemNode::Memory);
2230 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2231 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2232 return Type::get_zero_type(_type->basic_type());
2233 }
2234 }
2235
2236 if (!UseCompactObjectHeaders) {
2237 Node* alloc = is_new_object_mark_load();
2238 if (alloc != nullptr) {
2239 return TypeX::make(markWord::prototype().value());
2240 }
2241 }
2242
2243 return _type;
2244 }
2245
2246 //------------------------------match_edge-------------------------------------
2247 // Do we Match on this edge index or not? Match only the address.
2248 uint LoadNode::match_edge(uint idx) const {
2249 return idx == MemNode::Address;
2250 }
2251
2252 //--------------------------LoadBNode::Ideal--------------------------------------
2253 //
2254 // If the previous store is to the same address as this load,
2255 // and the value stored was larger than a byte, replace this load
2256 // with the value stored truncated to a byte. If no truncation is
2257 // needed, the replacement is done in LoadNode::Identity().
2258 //
2259 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2368 }
2369 }
2370 // Identity call will handle the case where truncation is not needed.
2371 return LoadNode::Ideal(phase, can_reshape);
2372 }
2373
2374 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2375 Node* mem = in(MemNode::Memory);
2376 Node* value = can_see_stored_value(mem,phase);
2377 if (value != nullptr && value->is_Con() &&
2378 !value->bottom_type()->higher_equal(_type)) {
2379 // If the input to the store does not fit with the load's result type,
2380 // it must be truncated. We can't delay until Ideal call since
2381 // a singleton Value is needed for split_thru_phi optimization.
2382 int con = value->get_int();
2383 return TypeInt::make((con << 16) >> 16);
2384 }
2385 return LoadNode::Value(phase);
2386 }
2387
2388 //=============================================================================
2389 //----------------------------LoadKlassNode::make------------------------------
2390 // Polymorphic factory method:
2391 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2392 // sanity check the alias category against the created node type
2393 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2394 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2395 #ifdef _LP64
2396 if (adr_type->is_ptr_to_narrowklass()) {
2397 assert(UseCompressedClassPointers, "no compressed klasses");
2398 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2399 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2400 }
2401 #endif
2402 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2403 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2404 }
2405
2406 //------------------------------Value------------------------------------------
2407 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2441 }
2442 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2443 }
2444 if (!t->is_klass()) {
2445 // a primitive Class (e.g., int.class) has null for a klass field
2446 return TypePtr::NULL_PTR;
2447 }
2448 // Fold up the load of the hidden field
2449 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2450 }
2451 // non-constant mirror, so we can't tell what's going on
2452 }
2453 if (!tinst->is_loaded())
2454 return _type; // Bail out if not loaded
2455 if (offset == oopDesc::klass_offset_in_bytes()) {
2456 return tinst->as_klass_type(true);
2457 }
2458 }
2459
2460 // Check for loading klass from an array
2461 const TypeAryPtr *tary = tp->isa_aryptr();
2462 if (tary != nullptr &&
2463 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2464 return tary->as_klass_type(true);
2465 }
2466
2467 // Check for loading klass from an array klass
2468 const TypeKlassPtr *tkls = tp->isa_klassptr();
2469 if (tkls != nullptr && !StressReflectiveCode) {
2470 if (!tkls->is_loaded())
2471 return _type; // Bail out if not loaded
2472 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2473 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2474 // // Always returning precise element type is incorrect,
2475 // // e.g., element type could be object and array may contain strings
2476 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2477
2478 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2479 // according to the element type's subclassing.
2480 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2481 }
2482 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2483 tkls->offset() == in_bytes(Klass::super_offset())) {
2484 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2485 // The field is Klass::_super. Return its (constant) value.
2486 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2487 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2488 }
2489 }
2490
2491 if (tkls != nullptr && !UseSecondarySupersCache
2492 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2493 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2494 return TypePtr::NULL_PTR;
2495 }
2496
2497 // Bailout case
2498 return LoadNode::Value(phase);
2499 }
2500
2501 //------------------------------Identity---------------------------------------
2524 base = bs->step_over_gc_barrier(base);
2525 }
2526
2527 // We can fetch the klass directly through an AllocateNode.
2528 // This works even if the klass is not constant (clone or newArray).
2529 if (offset == oopDesc::klass_offset_in_bytes()) {
2530 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2531 if (allocated_klass != nullptr) {
2532 return allocated_klass;
2533 }
2534 }
2535
2536 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2537 // See inline_native_Class_query for occurrences of these patterns.
2538 // Java Example: x.getClass().isAssignableFrom(y)
2539 //
2540 // This improves reflective code, often making the Class
2541 // mirror go completely dead. (Current exception: Class
2542 // mirrors may appear in debug info, but we could clean them out by
2543 // introducing a new debug info operator for Klass.java_mirror).
2544
2545 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2546 && offset == java_lang_Class::klass_offset()) {
2547 if (base->is_Load()) {
2548 Node* base2 = base->in(MemNode::Address);
2549 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2550 Node* adr2 = base2->in(MemNode::Address);
2551 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2552 if (tkls != nullptr && !tkls->empty()
2553 && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2554 && adr2->is_AddP()
2555 ) {
2556 int mirror_field = in_bytes(Klass::java_mirror_offset());
2557 if (tkls->offset() == mirror_field) {
2558 #ifdef ASSERT
2559 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2560 assert(tkls2->offset() == 0, "not a load of java_mirror");
2561 #endif
2562 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2563 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2564 return adr2->in(AddPNode::Address);
2565 }
2566 }
2567 }
2568 }
2569 }
2570
2571 return this;
2572 }
2573
2574 LoadNode* LoadNode::clone_pinned() const {
2575 LoadNode* ld = clone()->as_Load();
2701 //---------------------------StoreNode::make-----------------------------------
2702 // Polymorphic factory method:
2703 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2704 assert((mo == unordered || mo == release), "unexpected");
2705 Compile* C = gvn.C;
2706 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2707 ctl != nullptr, "raw memory operations should have control edge");
2708
2709 switch (bt) {
2710 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2711 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2712 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2713 case T_CHAR:
2714 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2715 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2716 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2717 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2718 case T_METADATA:
2719 case T_ADDRESS:
2720 case T_OBJECT:
2721 #ifdef _LP64
2722 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2723 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2724 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2725 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2726 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2727 adr->bottom_type()->isa_rawptr())) {
2728 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2729 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2730 }
2731 #endif
2732 {
2733 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2734 }
2735 default:
2736 ShouldNotReachHere();
2737 return (StoreNode*)nullptr;
2738 }
2739 }
2740
2741 //--------------------------bottom_type----------------------------------------
2742 const Type *StoreNode::bottom_type() const {
2743 return Type::MEMORY;
2744 }
2745
2746 //------------------------------hash-------------------------------------------
2747 uint StoreNode::hash() const {
2748 // unroll addition of interesting fields
2749 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2750
2751 // Since they are not commoned, do not hash them:
2752 return NO_HASH;
2753 }
2754
2755 // Link together multiple stores (B/S/C/I) into a longer one.
2756 //
3378 }
3379 ss.print_cr("[TraceMergeStores]: with");
3380 merged_input_value->dump("\n", false, &ss);
3381 merged_store->dump("\n", false, &ss);
3382 tty->print("%s", ss.as_string());
3383 }
3384 #endif
3385
3386 //------------------------------Ideal------------------------------------------
3387 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3388 // When a store immediately follows a relevant allocation/initialization,
3389 // try to capture it into the initialization, or hoist it above.
3390 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3391 Node* p = MemNode::Ideal_common(phase, can_reshape);
3392 if (p) return (p == NodeSentinel) ? nullptr : p;
3393
3394 Node* mem = in(MemNode::Memory);
3395 Node* address = in(MemNode::Address);
3396 Node* value = in(MemNode::ValueIn);
3397 // Back-to-back stores to same address? Fold em up. Generally
3398 // unsafe if I have intervening uses.
3399 {
3400 Node* st = mem;
3401 // If Store 'st' has more than one use, we cannot fold 'st' away.
3402 // For example, 'st' might be the final state at a conditional
3403 // return. Or, 'st' might be used by some node which is live at
3404 // the same time 'st' is live, which might be unschedulable. So,
3405 // require exactly ONE user until such time as we clone 'mem' for
3406 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3407 // true).
3408 while (st->is_Store() && st->outcnt() == 1) {
3409 // Looking at a dead closed cycle of memory?
3410 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3411 assert(Opcode() == st->Opcode() ||
3412 st->Opcode() == Op_StoreVector ||
3413 Opcode() == Op_StoreVector ||
3414 st->Opcode() == Op_StoreVectorScatter ||
3415 Opcode() == Op_StoreVectorScatter ||
3416 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3417 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3418 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3419 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3420 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3421
3422 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3423 st->as_Store()->memory_size() <= this->memory_size()) {
3424 Node* use = st->raw_out(0);
3425 if (phase->is_IterGVN()) {
3426 phase->is_IterGVN()->rehash_node_delayed(use);
3427 }
3428 // It's OK to do this in the parser, since DU info is always accurate,
3429 // and the parser always refers to nodes via SafePointNode maps.
3430 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3431 return this;
3432 }
3433 st = st->in(MemNode::Memory);
3434 }
3435 }
3436
3437
3438 // Capture an unaliased, unconditional, simple store into an initializer.
3536 const StoreVectorNode* store_vector = as_StoreVector();
3537 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3538 const Node* store_indices = store_vector->indices();
3539 const Node* mem_indices = mem_vector->indices();
3540 const Node* store_mask = store_vector->mask();
3541 const Node* mem_mask = mem_vector->mask();
3542 // Ensure types, indices, and masks match
3543 if (store_vector->vect_type() == mem_vector->vect_type() &&
3544 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3545 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3546 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3547 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3548 result = mem;
3549 }
3550 }
3551 }
3552
3553 // Store of zero anywhere into a freshly-allocated object?
3554 // Then the store is useless.
3555 // (It must already have been captured by the InitializeNode.)
3556 if (result == this &&
3557 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3558 // a newly allocated object is already all-zeroes everywhere
3559 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3560 result = mem;
3561 }
3562
3563 if (result == this) {
3564 // the store may also apply to zero-bits in an earlier object
3565 Node* prev_mem = find_previous_store(phase);
3566 // Steps (a), (b): Walk past independent stores to find an exact match.
3567 if (prev_mem != nullptr) {
3568 Node* prev_val = can_see_stored_value(prev_mem, phase);
3569 if (prev_val != nullptr && prev_val == val) {
3570 // prev_val and val might differ by a cast; it would be good
3571 // to keep the more informative of the two.
3572 result = mem;
3573 }
3574 }
3575 }
3576 }
3577
3578 PhaseIterGVN* igvn = phase->is_IterGVN();
3579 if (result != this && igvn != nullptr) {
3580 MemBarNode* trailing = trailing_membar();
3581 if (trailing != nullptr) {
3582 #ifdef ASSERT
3583 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
4052 // Clearing a short array is faster with stores
4053 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4054 // Already know this is a large node, do not try to ideal it
4055 if (_is_large) return nullptr;
4056
4057 const int unit = BytesPerLong;
4058 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4059 if (!t) return nullptr;
4060 if (!t->is_con()) return nullptr;
4061 intptr_t raw_count = t->get_con();
4062 intptr_t size = raw_count;
4063 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4064 // Clearing nothing uses the Identity call.
4065 // Negative clears are possible on dead ClearArrays
4066 // (see jck test stmt114.stmt11402.val).
4067 if (size <= 0 || size % unit != 0) return nullptr;
4068 intptr_t count = size / unit;
4069 // Length too long; communicate this to matchers and assemblers.
4070 // Assemblers are responsible to produce fast hardware clears for it.
4071 if (size > InitArrayShortSize) {
4072 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4073 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4074 return nullptr;
4075 }
4076 if (!IdealizeClearArrayNode) return nullptr;
4077 Node *mem = in(1);
4078 if( phase->type(mem)==Type::TOP ) return nullptr;
4079 Node *adr = in(3);
4080 const Type* at = phase->type(adr);
4081 if( at==Type::TOP ) return nullptr;
4082 const TypePtr* atp = at->isa_ptr();
4083 // adjust atp to be the correct array element address type
4084 if (atp == nullptr) atp = TypePtr::BOTTOM;
4085 else atp = atp->add_offset(Type::OffsetBot);
4086 // Get base for derived pointer purposes
4087 if( adr->Opcode() != Op_AddP ) Unimplemented();
4088 Node *base = adr->in(1);
4089
4090 Node *zero = phase->makecon(TypeLong::ZERO);
4091 Node *off = phase->MakeConX(BytesPerLong);
4092 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4093 count--;
4094 while( count-- ) {
4095 mem = phase->transform(mem);
4096 adr = phase->transform(new AddPNode(base,adr,off));
4097 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4098 }
4099 return mem;
4100 }
4101
4102 //----------------------------step_through----------------------------------
4103 // Return allocation input memory edge if it is different instance
4104 // or itself if it is the one we are looking for.
4105 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4106 Node* n = *np;
4107 assert(n->is_ClearArray(), "sanity");
4108 intptr_t offset;
4109 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4110 // This method is called only before Allocate nodes are expanded
4111 // during macro nodes expansion. Before that ClearArray nodes are
4112 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4113 // Allocate nodes are expanded) which follows allocations.
4114 assert(alloc != nullptr, "should have allocation");
4115 if (alloc->_idx == instance_id) {
4116 // Can not bypass initialization of the instance we are looking for.
4117 return false;
4120 InitializeNode* init = alloc->initialization();
4121 if (init != nullptr)
4122 *np = init->in(TypeFunc::Memory);
4123 else
4124 *np = alloc->in(TypeFunc::Memory);
4125 return true;
4126 }
4127
4128 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4129 Node* base = dest;
4130 if (raw_base) {
4131 // May be called as part of the initialization of a just allocated object
4132 base = phase->C->top();
4133 }
4134 return phase->transform(new AddPNode(base, dest, offset));
4135 }
4136
4137 //----------------------------clear_memory-------------------------------------
4138 // Generate code to initialize object storage to zero.
4139 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4140 intptr_t start_offset,
4141 Node* end_offset,
4142 bool raw_base,
4143 PhaseGVN* phase) {
4144 intptr_t offset = start_offset;
4145
4146 int unit = BytesPerLong;
4147 if ((offset % unit) != 0) {
4148 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4149 const TypePtr* atp = TypeRawPtr::BOTTOM;
4150 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4151 mem = phase->transform(mem);
4152 offset += BytesPerInt;
4153 }
4154 assert((offset % unit) == 0, "");
4155
4156 // Initialize the remaining stuff, if any, with a ClearArray.
4157 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, raw_base, phase);
4158 }
4159
4160 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4161 Node* start_offset,
4162 Node* end_offset,
4163 bool raw_base,
4164 PhaseGVN* phase) {
4165 if (start_offset == end_offset) {
4166 // nothing to do
4167 return mem;
4168 }
4169
4170 int unit = BytesPerLong;
4171 Node* zbase = start_offset;
4172 Node* zend = end_offset;
4173
4174 // Scale to the unit required by the CPU:
4175 if (!Matcher::init_array_count_is_in_bytes) {
4176 Node* shift = phase->intcon(exact_log2(unit));
4177 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4178 zend = phase->transform(new URShiftXNode(zend, shift) );
4179 }
4180
4181 // Bulk clear double-words
4182 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4183 Node* adr = make_address(dest, start_offset, raw_base, phase);
4184 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4185 return phase->transform(mem);
4186 }
4187
4188 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4189 intptr_t start_offset,
4190 intptr_t end_offset,
4191 bool raw_base,
4192 PhaseGVN* phase) {
4193 if (start_offset == end_offset) {
4194 // nothing to do
4195 return mem;
4196 }
4197
4198 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4199 intptr_t done_offset = end_offset;
4200 if ((done_offset % BytesPerLong) != 0) {
4201 done_offset -= BytesPerInt;
4202 }
4203 if (done_offset > start_offset) {
4204 mem = clear_memory(ctl, mem, dest,
4205 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4206 }
4207 if (done_offset < end_offset) { // emit the final 32-bit store
4208 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4209 const TypePtr* atp = TypeRawPtr::BOTTOM;
4210 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4211 mem = phase->transform(mem);
4212 done_offset += BytesPerInt;
4213 }
4214 assert(done_offset == end_offset, "");
4215 return mem;
4216 }
4217
4218 //=============================================================================
4219 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4220 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4221 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4222 #ifdef ASSERT
4223 , _pair_idx(0)
4224 #endif
4225 {
4226 init_class_id(Class_MemBar);
4227 Node* top = C->top();
4228 init_req(TypeFunc::I_O,top);
4229 init_req(TypeFunc::FramePtr,top);
4230 init_req(TypeFunc::ReturnAdr,top);
4337 PhaseIterGVN* igvn = phase->is_IterGVN();
4338 remove(igvn);
4339 // Must return either the original node (now dead) or a new node
4340 // (Do not return a top here, since that would break the uniqueness of top.)
4341 return new ConINode(TypeInt::ZERO);
4342 }
4343 }
4344 return progress ? this : nullptr;
4345 }
4346
4347 //------------------------------Value------------------------------------------
4348 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4349 if( !in(0) ) return Type::TOP;
4350 if( phase->type(in(0)) == Type::TOP )
4351 return Type::TOP;
4352 return TypeTuple::MEMBAR;
4353 }
4354
4355 //------------------------------match------------------------------------------
4356 // Construct projections for memory.
4357 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4358 switch (proj->_con) {
4359 case TypeFunc::Control:
4360 case TypeFunc::Memory:
4361 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4362 }
4363 ShouldNotReachHere();
4364 return nullptr;
4365 }
4366
4367 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4368 trailing->_kind = TrailingStore;
4369 leading->_kind = LeadingStore;
4370 #ifdef ASSERT
4371 trailing->_pair_idx = leading->_idx;
4372 leading->_pair_idx = leading->_idx;
4373 #endif
4374 }
4375
4376 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4377 trailing->_kind = TrailingLoadStore;
4624 return (req() > RawStores);
4625 }
4626
4627 void InitializeNode::set_complete(PhaseGVN* phase) {
4628 assert(!is_complete(), "caller responsibility");
4629 _is_complete = Complete;
4630
4631 // After this node is complete, it contains a bunch of
4632 // raw-memory initializations. There is no need for
4633 // it to have anything to do with non-raw memory effects.
4634 // Therefore, tell all non-raw users to re-optimize themselves,
4635 // after skipping the memory effects of this initialization.
4636 PhaseIterGVN* igvn = phase->is_IterGVN();
4637 if (igvn) igvn->add_users_to_worklist(this);
4638 }
4639
4640 // convenience function
4641 // return false if the init contains any stores already
4642 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4643 InitializeNode* init = initialization();
4644 if (init == nullptr || init->is_complete()) return false;
4645 init->remove_extra_zeroes();
4646 // for now, if this allocation has already collected any inits, bail:
4647 if (init->is_non_zero()) return false;
4648 init->set_complete(phase);
4649 return true;
4650 }
4651
4652 void InitializeNode::remove_extra_zeroes() {
4653 if (req() == RawStores) return;
4654 Node* zmem = zero_memory();
4655 uint fill = RawStores;
4656 for (uint i = fill; i < req(); i++) {
4657 Node* n = in(i);
4658 if (n->is_top() || n == zmem) continue; // skip
4659 if (fill < i) set_req(fill, n); // compact
4660 ++fill;
4661 }
4662 // delete any empty spaces created:
4663 while (fill < req()) {
4664 del_req(fill);
4808 // store node that we'd like to capture. We need to check
4809 // the uses of the MergeMemNode.
4810 mems.push(n);
4811 }
4812 } else if (n->is_Mem()) {
4813 Node* other_adr = n->in(MemNode::Address);
4814 if (other_adr == adr) {
4815 failed = true;
4816 break;
4817 } else {
4818 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4819 if (other_t_adr != nullptr) {
4820 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4821 if (other_alias_idx == alias_idx) {
4822 // A load from the same memory slice as the store right
4823 // after the InitializeNode. We check the control of the
4824 // object/array that is loaded from. If it's the same as
4825 // the store control then we cannot capture the store.
4826 assert(!n->is_Store(), "2 stores to same slice on same control?");
4827 Node* base = other_adr;
4828 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4829 base = base->in(AddPNode::Base);
4830 if (base != nullptr) {
4831 base = base->uncast();
4832 if (base->is_Proj() && base->in(0) == alloc) {
4833 failed = true;
4834 break;
4835 }
4836 }
4837 }
4838 }
4839 }
4840 } else {
4841 failed = true;
4842 break;
4843 }
4844 }
4845 }
4846 }
4847 if (failed) {
5394 // z's_done 12 16 16 16 12 16 12
5395 // z's_needed 12 16 16 16 16 16 16
5396 // zsize 0 0 0 0 4 0 4
5397 if (next_full_store < 0) {
5398 // Conservative tack: Zero to end of current word.
5399 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5400 } else {
5401 // Zero to beginning of next fully initialized word.
5402 // Or, don't zero at all, if we are already in that word.
5403 assert(next_full_store >= zeroes_needed, "must go forward");
5404 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5405 zeroes_needed = next_full_store;
5406 }
5407 }
5408
5409 if (zeroes_needed > zeroes_done) {
5410 intptr_t zsize = zeroes_needed - zeroes_done;
5411 // Do some incremental zeroing on rawmem, in parallel with inits.
5412 zeroes_done = align_down(zeroes_done, BytesPerInt);
5413 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5414 zeroes_done, zeroes_needed,
5415 true,
5416 phase);
5417 zeroes_done = zeroes_needed;
5418 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5419 do_zeroing = false; // leave the hole, next time
5420 }
5421 }
5422
5423 // Collect the store and move on:
5424 phase->replace_input_of(st, MemNode::Memory, inits);
5425 inits = st; // put it on the linearized chain
5426 set_req(i, zmem); // unhook from previous position
5427
5428 if (zeroes_done == st_off)
5429 zeroes_done = next_init_off;
5430
5431 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5432
5433 #ifdef ASSERT
5454 remove_extra_zeroes(); // clear out all the zmems left over
5455 add_req(inits);
5456
5457 if (!(UseTLAB && ZeroTLAB)) {
5458 // If anything remains to be zeroed, zero it all now.
5459 zeroes_done = align_down(zeroes_done, BytesPerInt);
5460 // if it is the last unused 4 bytes of an instance, forget about it
5461 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5462 if (zeroes_done + BytesPerLong >= size_limit) {
5463 AllocateNode* alloc = allocation();
5464 assert(alloc != nullptr, "must be present");
5465 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5466 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5467 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5468 if (zeroes_done == k->layout_helper())
5469 zeroes_done = size_limit;
5470 }
5471 }
5472 if (zeroes_done < size_limit) {
5473 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5474 zeroes_done, size_in_bytes, true, phase);
5475 }
5476 }
5477
5478 set_complete(phase);
5479 return rawmem;
5480 }
5481
5482 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5483 auto replace_proj = [&](ProjNode* proj) {
5484 C->gvn_replace_by(proj, mem);
5485 return CONTINUE;
5486 };
5487 apply_to_projs(replace_proj, TypeFunc::Memory);
5488 }
5489
5490 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5491 DUIterator_Fast imax, i = fast_outs(imax);
5492 auto replace_proj = [&](ProjNode* proj) {
5493 igvn->replace_node(proj, mem);
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "ci/ciInlineKlass.hpp"
28 #include "classfile/javaClasses.hpp"
29 #include "classfile/systemDictionary.hpp"
30 #include "compiler/compileLog.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/c2/barrierSetC2.hpp"
33 #include "gc/shared/tlab_globals.hpp"
34 #include "memory/allocation.inline.hpp"
35 #include "memory/resourceArea.hpp"
36 #include "oops/flatArrayKlass.hpp"
37 #include "oops/objArrayKlass.hpp"
38 #include "opto/addnode.hpp"
39 #include "opto/arraycopynode.hpp"
40 #include "opto/callnode.hpp"
41 #include "opto/cfgnode.hpp"
42 #include "opto/compile.hpp"
43 #include "opto/connode.hpp"
44 #include "opto/convertnode.hpp"
45 #include "opto/inlinetypenode.hpp"
46 #include "opto/loopnode.hpp"
47 #include "opto/machnode.hpp"
48 #include "opto/matcher.hpp"
49 #include "opto/memnode.hpp"
50 #include "opto/mempointer.hpp"
51 #include "opto/mulnode.hpp"
52 #include "opto/narrowptrnode.hpp"
53 #include "opto/phaseX.hpp"
54 #include "opto/regalloc.hpp"
55 #include "opto/regmask.hpp"
56 #include "opto/rootnode.hpp"
57 #include "opto/traceMergeStoresTag.hpp"
58 #include "opto/vectornode.hpp"
59 #include "runtime/arguments.hpp"
60 #include "utilities/align.hpp"
61 #include "utilities/copy.hpp"
62 #include "utilities/globalDefinitions.hpp"
63 #include "utilities/macros.hpp"
64 #include "utilities/powerOfTwo.hpp"
65 #include "utilities/vmError.hpp"
66
67 // Portions of code courtesy of Clifford Click
68
69 // Optimization - Graph Style
70
71 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
72
73 //=============================================================================
74 uint MemNode::size_of() const { return sizeof(*this); }
75
76 const TypePtr *MemNode::adr_type() const {
77 Node* adr = in(Address);
78 if (adr == nullptr) return nullptr; // node is dead
79 const TypePtr* cross_check = nullptr;
80 DEBUG_ONLY(cross_check = _adr_type);
81 return calculate_adr_type(adr->bottom_type(), cross_check);
82 }
131 st->print(", idx=Bot;");
132 else if (atp->index() == Compile::AliasIdxTop)
133 st->print(", idx=Top;");
134 else if (atp->index() == Compile::AliasIdxRaw)
135 st->print(", idx=Raw;");
136 else {
137 ciField* field = atp->field();
138 if (field) {
139 st->print(", name=");
140 field->print_name_on(st);
141 }
142 st->print(", idx=%d;", atp->index());
143 }
144 }
145 }
146
147 extern void print_alias_types();
148
149 #endif
150
151 // Find the memory output corresponding to the fall-through path of a call
152 static Node* find_call_fallthrough_mem_output(CallNode* call) {
153 ResourceMark rm;
154 CallProjections* projs = call->extract_projections(false, false);
155 Node* res = projs->fallthrough_memproj;
156 assert(res != nullptr, "must have a fallthrough mem output");
157 return res;
158 }
159
160 // Try to find a better memory input for a load from a strict final field
161 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
162 intptr_t offset = 0;
163 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
164 if (base == nullptr) {
165 return nullptr;
166 }
167
168 Node* base_uncasted = base->uncast();
169 if (base_uncasted->is_Proj()) {
170 Node* multi = base_uncasted->in(0);
171 if (multi->is_top()) {
172 // The pointer dies, make the memory die, too
173 return multi;
174 } else if (multi->is_Allocate()) {
175 base_local = base_uncasted->as_Proj();
176 return nullptr;
177 } else if (multi->is_Call()) {
178 // The oop is returned from a call, the memory can be the fallthrough output of the call
179 return find_call_fallthrough_mem_output(multi->as_Call());
180 } else if (multi->is_Start()) {
181 // The oop is a parameter
182 if (phase->C->method()->is_object_constructor() && base_uncasted->as_Proj()->_con == TypeFunc::Parms) {
183 // The receiver of a constructor is similar to the result of an AllocateNode
184 base_local = base_uncasted->as_Proj();
185 return nullptr;
186 } else {
187 // Use the start memory otherwise
188 return multi->as_Start()->proj_out(TypeFunc::Memory);
189 }
190 }
191 }
192
193 return nullptr;
194 }
195
196 // Whether a call can modify a strict final field, given that the object is allocated inside the
197 // current compilation unit, or is the first parameter when the compilation root is a constructor.
198 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
199 // the target method is a subclass of the class declaring 'field'.
200 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
201 if (!call->is_CallJava()) {
202 return false;
203 }
204
205 ciMethod* target = call->as_CallJava()->method();
206 if (target == nullptr || !target->is_object_constructor()) {
207 return false;
208 }
209
210 // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
211 // then the field is set inside the constructor, else the field must be set before the
212 // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
213 // while a field Sub.y must be set before Super::<init> is invoked.
214 // We can try to be more heroic and decide if the receiver of the constructor invocation is the
215 // object from which we are loading from. This, however, may be problematic as deciding if 2
216 // nodes are definitely different may not be trivial, especially if the graph is not canonical.
217 // As a result, it is made more conservative for now.
218 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
219 return target->holder()->is_subclass_of(field->holder());
220 }
221
222 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
223 assert(t_oop != nullptr, "sanity");
224 bool is_known_instance = t_oop->is_known_instance_field();
225 bool is_strict_final_load = false;
226
227 // After macro expansion, an allocation may become a call, changing the memory input to the
228 // memory output of that call would be illegal. As a result, disallow this transformation after
229 // macro expansion.
230 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
231 is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
232 #ifdef ASSERT
233 if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
234 assert(is_strict_final_load, "sanity check for basic cases");
235 }
236 #endif // ASSERT
237 }
238
239 if (!is_known_instance && !is_strict_final_load) {
240 return mchain;
241 }
242
243 Node* result = mchain;
244 ProjNode* base_local = nullptr;
245
246 ciField* field = nullptr;
247 if (is_strict_final_load) {
248 field = phase->C->alias_type(t_oop)->field();
249 assert(field != nullptr, "must point to a field");
250
251 Node* adr = load->in(MemNode::Address);
252 assert(phase->type(adr) == t_oop, "inconsistent type");
253 Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
254 if (tmp != nullptr) {
255 result = tmp;
256 }
257 }
258
259 uint instance_id = t_oop->instance_id();
260 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
261 Node* prev = nullptr;
262 while (prev != result) {
263 prev = result;
264 if (result == start_mem) {
265 // start_mem is the earliest memory possible
266 break;
267 }
268
269 // skip over a call which does not affect this memory slice
270 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
271 Node* proj_in = result->in(0);
272 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
273 // This is the allocation that creates the object from which we are loading from
274 break;
275 } else if (proj_in->is_Call()) {
276 // ArrayCopyNodes processed here as well
277 CallNode* call = proj_in->as_Call();
278 if (!call->may_modify(t_oop, phase)) {
279 result = call->in(TypeFunc::Memory);
280 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
281 result = call->in(TypeFunc::Memory);
282 }
283 } else if (proj_in->Opcode() == Op_Tuple) {
284 // The call will be folded, skip over it.
285 break;
286 } else if (proj_in->is_Initialize()) {
287 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
288 // Stop if this is the initialization for the object instance which
289 // contains this memory slice, otherwise skip over it.
290 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
291 break;
292 }
293 if (is_known_instance) {
294 result = proj_in->in(TypeFunc::Memory);
295 } else if (is_strict_final_load) {
296 Node* klass = alloc->in(AllocateNode::KlassNode);
297 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
298 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
299 // Allocation of another type, must be another object
300 result = proj_in->in(TypeFunc::Memory);
301 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
302 // Allocation of another object
303 result = proj_in->in(TypeFunc::Memory);
304 }
305 }
306 } else if (proj_in->is_MemBar()) {
307 ArrayCopyNode* ac = nullptr;
308 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
309 break;
310 }
311 result = proj_in->in(TypeFunc::Memory);
312 } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
313 if (is_strict_final_load) {
314 // LoadFlat and StoreFlat cannot happen to strict final fields
315 result = proj_in->in(TypeFunc::Memory);
316 }
317 } else if (proj_in->is_top()) {
318 break; // dead code
319 } else {
320 assert(false, "unexpected projection of %s", proj_in->Name());
321 }
322 } else if (result->is_ClearArray()) {
323 if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
324 // Can not bypass initialization of the instance
325 // we are looking for.
326 break;
327 }
328 // Otherwise skip it (the call updated 'result' value).
329 } else if (result->is_MergeMem()) {
330 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
331 }
332 }
333 return result;
334 }
335
336 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
337 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
338 if (t_oop == nullptr)
339 return mchain; // don't try to optimize non-oop types
340 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
341 bool is_instance = t_oop->is_known_instance_field();
342 PhaseIterGVN *igvn = phase->is_IterGVN();
343 if (is_instance && igvn != nullptr && result->is_Phi()) {
344 PhiNode *mphi = result->as_Phi();
345 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
346 const TypePtr *t = mphi->adr_type();
347 bool do_split = false;
348 // In the following cases, Load memory input can be further optimized based on
349 // its precise address type
350 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
351 do_split = true;
352 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
353 const TypeOopPtr* mem_t =
354 t->is_oopptr()->cast_to_exactness(true)
355 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
356 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
357 if (t_oop->isa_aryptr()) {
358 mem_t = mem_t->is_aryptr()
359 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
360 ->cast_to_size(t_oop->is_aryptr()->size())
361 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
362 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
363 ->with_offset(t_oop->is_aryptr()->offset())
364 ->is_aryptr();
365 }
366 do_split = mem_t == t_oop;
367 }
368 if (do_split) {
369 // clone the Phi with our address type
370 result = mphi->split_out_instance(t_adr, igvn);
371 } else {
372 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
373 }
374 }
375 return result;
376 }
377
378 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
379 uint alias_idx = phase->C->get_alias_index(tp);
380 Node *mem = mmem;
381 #ifdef ASSERT
382 {
383 // Check that current type is consistent with the alias index used during graph construction
384 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
385 bool consistent = adr_check == nullptr || adr_check->empty() ||
386 phase->C->must_alias(adr_check, alias_idx );
387 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
388 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
389 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
390 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
391 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
392 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
393 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
394 // don't assert if it is dead code.
395 consistent = true;
396 }
397 if( !consistent ) {
398 st->print("alias_idx==%d, adr_check==", alias_idx);
399 if( adr_check == nullptr ) {
400 st->print("null");
401 } else {
402 adr_check->dump();
403 }
404 st->cr();
405 print_alias_types();
406 assert(consistent, "adr_check must match alias idx");
407 }
408 }
409 #endif
1081 "use LoadKlassNode instead");
1082 assert(!(adr_type->isa_aryptr() &&
1083 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1084 "use LoadRangeNode instead");
1085 // Check control edge of raw loads
1086 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1087 // oop will be recorded in oop map if load crosses safepoint
1088 rt->isa_oopptr() || is_immutable_value(adr),
1089 "raw memory operations should have control edge");
1090 LoadNode* load = nullptr;
1091 switch (bt) {
1092 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1093 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1094 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1095 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1096 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1097 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1098 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1099 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1100 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1101 case T_ARRAY:
1102 case T_OBJECT:
1103 case T_NARROWOOP:
1104 #ifdef _LP64
1105 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1106 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1107 } else
1108 #endif
1109 {
1110 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1111 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1112 }
1113 break;
1114 default:
1115 guarantee(false, "unexpected basic type %s", type2name(bt));
1116 break;
1117 }
1118 assert(load != nullptr, "LoadNode should have been created");
1119 if (unaligned) {
1120 load->set_unaligned_access();
1121 }
1122 if (mismatched) {
1123 load->set_mismatched_access();
1124 }
1125 if (unsafe) {
1126 load->set_unsafe_access();
1127 }
1128 load->set_barrier_data(barrier_data);
1129 if (load->Opcode() == Op_LoadN) {
1130 Node* ld = gvn.transform(load);
1131 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1132 }
1133
1134 return load;
1135 }
1136
1137 //------------------------------hash-------------------------------------------
1138 uint LoadNode::hash() const {
1139 // unroll addition of interesting fields
1140 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1141 }
1142
1143 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1144 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1145 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1146 bool is_stable_ary = FoldStableValues &&
1147 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1148 tp->isa_aryptr()->is_stable();
1149
1150 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1151 }
1152
1153 return false;
1154 }
1155
1156 // Is the value loaded previously stored by an arraycopy? If so return
1157 // a load node that reads from the source array so we may be able to
1158 // optimize out the ArrayCopy node later.
1159 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1160 Node* ld_adr = in(MemNode::Address);
1161 intptr_t ld_off = 0;
1162 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1163 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1164 if (ac != nullptr) {
1165 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1166
1167 Node* mem = ac->in(TypeFunc::Memory);
1168 Node* ctl = ac->in(0);
1169 Node* src = ac->in(ArrayCopyNode::Src);
1170
1178 if (ac->as_ArrayCopy()->is_clonebasic()) {
1179 assert(ld_alloc != nullptr, "need an alloc");
1180 assert(addp->is_AddP(), "address must be addp");
1181 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1182 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1183 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1184 addp->set_req(AddPNode::Base, src);
1185 addp->set_req(AddPNode::Address, src);
1186 } else {
1187 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1188 ac->as_ArrayCopy()->is_copyof_validated() ||
1189 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1190 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1191 addp->set_req(AddPNode::Base, src);
1192 addp->set_req(AddPNode::Address, src);
1193
1194 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1195 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1196 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1197
1198 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1199
1200 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1201 #ifdef _LP64
1202 diff = phase->transform(new ConvI2LNode(diff));
1203 #endif
1204 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1205
1206 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1207 addp->set_req(AddPNode::Offset, offset);
1208 }
1209 addp = phase->transform(addp);
1210 #ifdef ASSERT
1211 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1212 ld->_adr_type = adr_type;
1213 #endif
1214 ld->set_req(MemNode::Address, addp);
1215 ld->set_req(0, ctl);
1216 ld->set_req(MemNode::Memory, mem);
1217 return ld;
1218 }
1219 return nullptr;
1220 }
1221
1222 static Node* see_through_inline_type(PhaseValues* phase, const MemNode* load, Node* base, int offset) {
1223 if (!load->is_mismatched_access() && base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1224 InlineTypeNode* vt = base->as_InlineType();
1225 Node* value = vt->field_value_by_offset(offset, true);
1226 assert(value != nullptr, "must see some value");
1227 return value;
1228 }
1229
1230 return nullptr;
1231 }
1232
1233 //---------------------------can_see_stored_value------------------------------
1234 // This routine exists to make sure this set of tests is done the same
1235 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1236 // will change the graph shape in a way which makes memory alive twice at the
1237 // same time (uses the Oracle model of aliasing), then some
1238 // LoadXNode::Identity will fold things back to the equivalence-class model
1239 // of aliasing.
1240 // This method may find an unencoded node instead of the corresponding encoded one.
1241 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1242 Node* ld_adr = in(MemNode::Address);
1243 intptr_t ld_off = 0;
1244 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1245 // Try to see through an InlineTypeNode
1246 // LoadN is special because the input is not compressed
1247 if (Opcode() != Op_LoadN) {
1248 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1249 if (value != nullptr) {
1250 return value;
1251 }
1252 }
1253
1254 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1255 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1256 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1257 // This is more general than load from boxing objects.
1258 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1259 uint alias_idx = atp->index();
1260 Node* result = nullptr;
1261 Node* current = st;
1262 // Skip through chains of MemBarNodes checking the MergeMems for
1263 // new states for the slice of this load. Stop once any other
1264 // kind of node is encountered. Loads from final memory can skip
1265 // through any kind of MemBar but normal loads shouldn't skip
1266 // through MemBarAcquire since the could allow them to move out of
1267 // a synchronized region. It is not safe to step over MemBarCPUOrder,
1268 // because alias info above them may be inaccurate (e.g., due to
1269 // mixed/mismatched unsafe accesses).
1270 bool is_final_mem = !atp->is_rewritable();
1271 while (current->is_Proj()) {
1272 int opc = current->in(0)->Opcode();
1273 if ((is_final_mem && (opc == Op_MemBarAcquire ||
1317 // Same base, same offset.
1318 // Possible improvement for arrays: check index value instead of absolute offset.
1319
1320 // At this point we have proven something like this setup:
1321 // B = << base >>
1322 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1323 // S = StoreQ(AddP( B , #Off), V)
1324 // (Actually, we haven't yet proven the Q's are the same.)
1325 // In other words, we are loading from a casted version of
1326 // the same pointer-and-offset that we stored to.
1327 // Casted version may carry a dependency and it is respected.
1328 // Thus, we are able to replace L by V.
1329 }
1330 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1331 if (store_Opcode() != st->Opcode()) {
1332 return nullptr;
1333 }
1334 // LoadVector/StoreVector needs additional check to ensure the types match.
1335 if (st->is_StoreVector()) {
1336 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1337 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1338 if (in_vt != out_vt) {
1339 return nullptr;
1340 }
1341 }
1342 return st->in(MemNode::ValueIn);
1343 }
1344
1345 // A load from a freshly-created object always returns zero.
1346 // (This can happen after LoadNode::Ideal resets the load's memory input
1347 // to find_captured_store, which returned InitializeNode::zero_memory.)
1348 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1349 (st->in(0) == ld_alloc) &&
1350 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1351 // return a zero value for the load's basic type
1352 // (This is one of the few places where a generic PhaseTransform
1353 // can create new nodes. Think of it as lazily manifesting
1354 // virtually pre-existing constants.)
1355 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1356 if (init_value != nullptr) {
1357 const TypeAryPtr* ld_adr_type = phase->type(ld_adr)->isa_aryptr();
1358 if (ld_adr_type == nullptr) {
1359 return nullptr;
1360 }
1361
1362 // We know that this is not a flat array, the load should return the whole oop
1363 if (ld_adr_type->is_not_flat()) {
1364 return init_value;
1365 }
1366
1367 // If this is a flat array, try to see through init_value
1368 if (init_value->is_EncodeP()) {
1369 init_value = init_value->in(1);
1370 }
1371 if (!init_value->is_InlineType() || ld_adr_type->field_offset() == Type::Offset::bottom) {
1372 return nullptr;
1373 }
1374
1375 ciInlineKlass* vk = phase->type(init_value)->inline_klass();
1376 int field_offset_in_payload = ld_adr_type->field_offset().get();
1377 if (field_offset_in_payload == vk->null_marker_offset_in_payload()) {
1378 return init_value->as_InlineType()->get_null_marker();
1379 } else {
1380 return init_value->as_InlineType()->field_value_by_offset(field_offset_in_payload + vk->payload_offset(), true);
1381 }
1382 }
1383 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1384 if (value_basic_type() != T_VOID) {
1385 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1386 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1387 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1388 // by the ArrayCopyNode.
1389 return phase->zerocon(value_basic_type());
1390 }
1391 } else {
1392 // TODO: materialize all-zero vector constant
1393 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1394 }
1395 }
1396
1397 // A load from an initialization barrier can match a captured store.
1398 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1399 InitializeNode* init = st->in(0)->as_Initialize();
1400 AllocateNode* alloc = init->allocation();
1401 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1402 // examine a captured store value
1403 st = init->find_captured_store(ld_off, memory_size(), phase);
1416 base = bs->step_over_gc_barrier(base);
1417 if (base != nullptr && base->is_Proj() &&
1418 base->as_Proj()->_con == TypeFunc::Parms &&
1419 base->in(0)->is_CallStaticJava() &&
1420 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1421 return base->in(0)->in(TypeFunc::Parms);
1422 }
1423 }
1424
1425 break;
1426 }
1427
1428 return nullptr;
1429 }
1430
1431 //----------------------is_instance_field_load_with_local_phi------------------
1432 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1433 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1434 in(Address)->is_AddP() ) {
1435 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1436 // Only known instances and immutable fields
1437 if( t_oop != nullptr &&
1438 (t_oop->is_ptr_to_strict_final_field() ||
1439 t_oop->is_known_instance_field()) &&
1440 t_oop->offset() != Type::OffsetBot &&
1441 t_oop->offset() != Type::OffsetTop) {
1442 return true;
1443 }
1444 }
1445 return false;
1446 }
1447
1448 //------------------------------Identity---------------------------------------
1449 // Loads are identity if previous store is to same address
1450 Node* LoadNode::Identity(PhaseGVN* phase) {
1451 // If the previous store-maker is the right kind of Store, and the store is
1452 // to the same address, then we are equal to the value stored.
1453 Node* mem = in(Memory);
1454 Node* value = can_see_stored_value(mem, phase);
1455 if( value ) {
1456 // byte, short & char stores truncate naturally.
1457 // A load has to load the truncated value which requires
1458 // some sort of masking operation and that requires an
1459 // Ideal call instead of an Identity call.
1460 if (memory_size() < BytesPerInt) {
1461 // If the input to the store does not fit with the load's result type,
1462 // it must be truncated via an Ideal call.
1463 if (!phase->type(value)->higher_equal(phase->type(this)))
1464 return this;
1465 }
1466
1467 if (phase->type(value)->isa_ptr() && phase->type(this)->isa_narrowoop()) {
1468 return this;
1469 }
1470 // (This works even when value is a Con, but LoadNode::Value
1471 // usually runs first, producing the singleton type of the Con.)
1472 if (!has_pinned_control_dependency() || value->is_Con()) {
1473 return value;
1474 } else {
1475 return this;
1476 }
1477 }
1478
1479 if (has_pinned_control_dependency()) {
1480 return this;
1481 }
1482 // Search for an existing data phi which was generated before for the same
1483 // instance's field to avoid infinite generation of phis in a loop.
1484 Node *region = mem->in(0);
1485 if (is_instance_field_load_with_local_phi(region)) {
1486 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1487 int this_index = phase->C->get_alias_index(addr_t);
1488 int this_offset = addr_t->offset();
1489 int this_iid = addr_t->instance_id();
1490 if (!addr_t->is_known_instance() &&
1491 addr_t->is_ptr_to_strict_final_field()) {
1492 // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1493 intptr_t ignore = 0;
1494 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1495 if (base == nullptr) {
1496 return this;
1497 }
1498 this_iid = base->_idx;
1499 }
1500 const Type* this_type = bottom_type();
1501 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1502 Node* phi = region->fast_out(i);
1503 if (phi->is_Phi() && phi != mem &&
1504 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1505 return phi;
1506 }
1507 }
1508 }
1509
1510 return this;
1511 }
1512
2028 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2029 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2030
2031 // Skip up past a SafePoint control. Cannot do this for Stores because
2032 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2033 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2034 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2035 !addr_mark &&
2036 (depends_only_on_test() || has_unknown_control_dependency())) {
2037 ctrl = ctrl->in(0);
2038 set_req(MemNode::Control,ctrl);
2039 progress = true;
2040 }
2041
2042 intptr_t ignore = 0;
2043 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2044 if (base != nullptr
2045 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2046 // Check for useless control edge in some common special cases
2047 if (in(MemNode::Control) != nullptr
2048 // TODO 8350865 Can we re-enable this?
2049 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2050 && can_remove_control()
2051 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2052 && all_controls_dominate(base, phase->C->start())) {
2053 // A method-invariant, non-null address (constant or 'this' argument).
2054 set_req(MemNode::Control, nullptr);
2055 progress = true;
2056 }
2057 }
2058
2059 Node* mem = in(MemNode::Memory);
2060 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2061
2062 if (can_reshape && (addr_t != nullptr)) {
2063 // try to optimize our memory input
2064 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2065 if (opt_mem != mem) {
2066 set_req_X(MemNode::Memory, opt_mem, phase);
2067 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2068 return this;
2069 }
2126 // fold up, do so.
2127 Node* prev_mem = find_previous_store(phase);
2128 if (prev_mem != nullptr) {
2129 Node* value = can_see_arraycopy_value(prev_mem, phase);
2130 if (value != nullptr) {
2131 return value;
2132 }
2133 }
2134 // Steps (a), (b): Walk past independent stores to find an exact match.
2135 if (prev_mem != nullptr && prev_mem != in(MemNode::Memory)) {
2136 // (c) See if we can fold up on the spot, but don't fold up here.
2137 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
2138 // just return a prior value, which is done by Identity calls.
2139 if (can_see_stored_value(prev_mem, phase)) {
2140 // Make ready for step (d):
2141 set_req_X(MemNode::Memory, prev_mem, phase);
2142 return this;
2143 }
2144 }
2145
2146 if (progress) {
2147 return this;
2148 }
2149
2150 if (!can_reshape) {
2151 phase->record_for_igvn(this);
2152 }
2153 return nullptr;
2154 }
2155
2156 // Helper to recognize certain Klass fields which are invariant across
2157 // some group of array types (e.g., int[] or all T[] where T < Object).
2158 const Type*
2159 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
2160 ciKlass* klass) const {
2161 assert(!UseCompactObjectHeaders || tkls->offset() != in_bytes(Klass::prototype_header_offset()),
2162 "must not happen");
2163
2164 if (tkls->isa_instklassptr() && tkls->offset() == in_bytes(InstanceKlass::access_flags_offset())) {
2165 // The field is InstanceKlass::_access_flags. Return its (constant) value.
2166 assert(Opcode() == Op_LoadUS, "must load an unsigned short from _access_flags");
2167 ciInstanceKlass* iklass = tkls->is_instklassptr()->instance_klass();
2168 return TypeInt::make(iklass->access_flags());
2169 }
2170 if (tkls->offset() == in_bytes(Klass::misc_flags_offset())) {
2171 // The field is Klass::_misc_flags. Return its (constant) value.
2172 assert(Opcode() == Op_LoadUB, "must load an unsigned byte from _misc_flags");
2173 return TypeInt::make(klass->misc_flags());
2181 // No match.
2182 return nullptr;
2183 }
2184
2185 //------------------------------Value-----------------------------------------
2186 const Type* LoadNode::Value(PhaseGVN* phase) const {
2187 // Either input is TOP ==> the result is TOP
2188 Node* mem = in(MemNode::Memory);
2189 const Type *t1 = phase->type(mem);
2190 if (t1 == Type::TOP) return Type::TOP;
2191 Node* adr = in(MemNode::Address);
2192 const TypePtr* tp = phase->type(adr)->isa_ptr();
2193 if (tp == nullptr || tp->empty()) return Type::TOP;
2194 int off = tp->offset();
2195 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2196 Compile* C = phase->C;
2197
2198 // If load can see a previous constant store, use that.
2199 Node* value = can_see_stored_value(mem, phase);
2200 if (value != nullptr && value->is_Con()) {
2201 if (phase->type(value)->isa_ptr() && _type->isa_narrowoop()) {
2202 return phase->type(value)->make_narrowoop();
2203 } else {
2204 assert(value->bottom_type()->higher_equal(_type), "sanity");
2205 return phase->type(value);
2206 }
2207 }
2208 // Try to guess loaded type from pointer type
2209 if (tp->isa_aryptr()) {
2210 const TypeAryPtr* ary = tp->is_aryptr();
2211 const Type* t = ary->elem();
2212
2213 // Determine whether the reference is beyond the header or not, by comparing
2214 // the offset against the offset of the start of the array's data.
2215 // Different array types begin at slightly different offsets (12 vs. 16).
2216 // We choose T_BYTE as an example base type that is least restrictive
2217 // as to alignment, which will therefore produce the smallest
2218 // possible base offset.
2219 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2220 const bool off_beyond_header = (off >= min_base_off);
2221
2222 // Try to constant-fold a stable array element.
2223 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2224 // Make sure the reference is not into the header and the offset is constant
2225 ciObject* aobj = ary->const_oop();
2226 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2227 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2228 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off, ary->field_offset().get(),
2229 stable_dimension,
2230 value_basic_type(), is_unsigned());
2231 if (con_type != nullptr) {
2232 return con_type;
2233 }
2234 }
2235 }
2236
2237 // Don't do this for integer types. There is only potential profit if
2238 // the element type t is lower than _type; that is, for int types, if _type is
2239 // more restrictive than t. This only happens here if one is short and the other
2240 // char (both 16 bits), and in those cases we've made an intentional decision
2241 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2242 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2243 //
2244 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2245 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2246 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2247 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2248 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2249 // In fact, that could have been the original type of p1, and p1 could have
2250 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2251 // expression (LShiftL quux 3) independently optimized to the constant 8.
2252 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2253 && (_type->isa_vect() == nullptr)
2254 && !ary->is_flat()
2255 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2256 // t might actually be lower than _type, if _type is a unique
2257 // concrete subclass of abstract class t.
2258 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2259 const Type* jt = t->join_speculative(_type);
2260 // In any case, do not allow the join, per se, to empty out the type.
2261 if (jt->empty() && !t->empty()) {
2262 // This can happen if a interface-typed array narrows to a class type.
2263 jt = _type;
2264 }
2265 #ifdef ASSERT
2266 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2267 // The pointers in the autobox arrays are always non-null
2268 Node* base = adr->in(AddPNode::Base);
2269 if ((base != nullptr) && base->is_DecodeN()) {
2270 // Get LoadN node which loads IntegerCache.cache field
2271 base = base->in(1);
2272 }
2273 if ((base != nullptr) && base->is_Con()) {
2274 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2275 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2276 // It could be narrow oop
2277 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2278 }
2279 }
2280 }
2281 #endif
2282 return jt;
2283 }
2284 }
2285 } else if (tp->base() == Type::InstPtr) {
2286 assert( off != Type::OffsetBot ||
2287 // arrays can be cast to Objects
2288 !tp->isa_instptr() ||
2289 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2290 // Default value load
2291 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2292 // unsafe field access may not have a constant offset
2293 C->has_unsafe_access(),
2294 "Field accesses must be precise" );
2295 // For oop loads, we expect the _type to be precise.
2296
2297 const TypeInstPtr* tinst = tp->is_instptr();
2298 BasicType bt = value_basic_type();
2299
2300 // Fold loads of the field map
2301 if (tinst != nullptr) {
2302 ciInstanceKlass* ik = tinst->instance_klass();
2303 int offset = tinst->offset();
2304 if (ik == phase->C->env()->Class_klass()) {
2305 ciType* t = tinst->java_mirror_type();
2306 if (t != nullptr && t->is_inlinetype() && offset == t->as_inline_klass()->field_map_offset()) {
2307 ciConstant map = t->as_inline_klass()->get_field_map();
2308 bool is_narrow_oop = (bt == T_NARROWOOP);
2309 return Type::make_from_constant(map, true, 1, is_narrow_oop);
2310 }
2311 }
2312 }
2313
2314 // Optimize loads from constant fields.
2315 ciObject* const_oop = tinst->const_oop();
2316 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2317 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2318 if (con_type != nullptr) {
2319 return con_type;
2320 }
2321 }
2322 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2323 assert(off != Type::OffsetBot ||
2324 !tp->isa_instklassptr() ||
2325 // arrays can be cast to Objects
2326 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2327 // also allow array-loading from the primary supertype
2328 // array during subtype checks
2329 Opcode() == Op_LoadKlass,
2330 "Field accesses must be precise");
2331 // For klass/static loads, we expect the _type to be precise
2332 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2333 /* With mirrors being an indirect in the Klass*
2334 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2335 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2336 *
2337 * So check the type and klass of the node before the LoadP.
2344 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2345 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2346 return TypeInstPtr::make(klass->java_mirror());
2347 }
2348 }
2349 }
2350
2351 const TypeKlassPtr *tkls = tp->isa_klassptr();
2352 if (tkls != nullptr) {
2353 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2354 ciKlass* klass = tkls->exact_klass();
2355 // We are loading a field from a Klass metaobject whose identity
2356 // is known at compile time (the type is "exact" or "precise").
2357 // Check for fields we know are maintained as constants by the VM.
2358 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2359 // The field is Klass::_super_check_offset. Return its (constant) value.
2360 // (Folds up type checking code.)
2361 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2362 return TypeInt::make(klass->super_check_offset());
2363 }
2364 if (klass->is_inlinetype() && tkls->offset() == in_bytes(InstanceKlass::acmp_maps_offset_offset())) {
2365 return TypeInt::make(klass->as_inline_klass()->field_map_offset());
2366 }
2367 if (klass->is_obj_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset())) {
2368 // Fold loads from LibraryCallKit::load_default_refined_array_klass
2369 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr();
2370 }
2371 if (klass->is_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::properties_offset())) {
2372 assert(klass->is_type_array_klass() || tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2373 return TypeInt::make((jint)klass->as_array_klass()->properties().value());
2374 }
2375 if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2376 assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2377 return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2378 }
2379 if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2380 // The field is Klass::_prototype_header. Return its (constant) value.
2381 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2382 return TypeX::make(klass->prototype_header());
2383 }
2384 // Compute index into primary_supers array
2385 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2386 // Check for overflowing; use unsigned compare to handle the negative case.
2387 if( depth < ciKlass::primary_super_limit() ) {
2388 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2389 // (Folds up type checking code.)
2390 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2391 ciKlass *ss = klass->super_of_depth(depth);
2392 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2393 }
2394 const Type* aift = load_array_final_field(tkls, klass);
2395 if (aift != nullptr) return aift;
2396 }
2397
2398 // We can still check if we are loading from the primary_supers array at a
2399 // shallow enough depth. Even though the klass is not exact, entries less
2400 // than or equal to its super depth are correct.
2401 if (tkls->is_loaded()) {
2402 ciKlass* klass = nullptr;
2436 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2437 // The key property of this type is that it folds up tests
2438 // for array-ness, since it proves that the layout_helper is positive.
2439 // Thus, a generic value like the basic object layout helper works fine.
2440 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2441 }
2442 }
2443
2444 // If we are loading from a freshly-allocated object/array, produce a zero.
2445 // Things to check:
2446 // 1. Load is beyond the header: headers are not guaranteed to be zero
2447 // 2. Load is not vectorized: vectors have no zero constant
2448 // 3. Load has no matching store, i.e. the input is the initial memory state
2449 const TypeOopPtr* tinst = tp->isa_oopptr();
2450 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2451 bool is_not_vect = (_type->isa_vect() == nullptr);
2452 if (is_not_header && is_not_vect) {
2453 Node* mem = in(MemNode::Memory);
2454 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2455 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2456 // TODO 8350865 Scalar replacement does not work well for flat arrays.
2457 // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2458 // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2459 // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2460 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2461 intptr_t offset = 0;
2462 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2463 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2464 if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2465 return _type;
2466 }
2467 }
2468 return Type::get_zero_type(_type->basic_type());
2469 }
2470 }
2471 if (!UseCompactObjectHeaders) {
2472 Node* alloc = is_new_object_mark_load();
2473 if (alloc != nullptr) {
2474 if (Arguments::is_valhalla_enabled()) {
2475 // The mark word may contain property bits (inline, flat, null-free)
2476 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2477 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2478 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2479 return TypeX::make(tkls->exact_klass()->prototype_header());
2480 }
2481 } else {
2482 return TypeX::make(markWord::prototype().value());
2483 }
2484 }
2485 }
2486
2487 return _type;
2488 }
2489
2490 //------------------------------match_edge-------------------------------------
2491 // Do we Match on this edge index or not? Match only the address.
2492 uint LoadNode::match_edge(uint idx) const {
2493 return idx == MemNode::Address;
2494 }
2495
2496 //--------------------------LoadBNode::Ideal--------------------------------------
2497 //
2498 // If the previous store is to the same address as this load,
2499 // and the value stored was larger than a byte, replace this load
2500 // with the value stored truncated to a byte. If no truncation is
2501 // needed, the replacement is done in LoadNode::Identity().
2502 //
2503 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2612 }
2613 }
2614 // Identity call will handle the case where truncation is not needed.
2615 return LoadNode::Ideal(phase, can_reshape);
2616 }
2617
2618 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2619 Node* mem = in(MemNode::Memory);
2620 Node* value = can_see_stored_value(mem,phase);
2621 if (value != nullptr && value->is_Con() &&
2622 !value->bottom_type()->higher_equal(_type)) {
2623 // If the input to the store does not fit with the load's result type,
2624 // it must be truncated. We can't delay until Ideal call since
2625 // a singleton Value is needed for split_thru_phi optimization.
2626 int con = value->get_int();
2627 return TypeInt::make((con << 16) >> 16);
2628 }
2629 return LoadNode::Value(phase);
2630 }
2631
2632 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2633 // Loading from an InlineType, find the input and make an EncodeP
2634 Node* addr = in(Address);
2635 intptr_t offset;
2636 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
2637 Node* value = see_through_inline_type(phase, this, base, offset);
2638 if (value != nullptr) {
2639 return new EncodePNode(value, type());
2640 }
2641
2642 // Can see the corresponding value, may need to add an EncodeP
2643 value = can_see_stored_value(in(Memory), phase);
2644 if (value != nullptr && phase->type(value)->isa_ptr() && type()->isa_narrowoop()) {
2645 return new EncodePNode(value, type());
2646 }
2647
2648 // Identity call will handle the case where EncodeP is unnecessary
2649 return LoadNode::Ideal(phase, can_reshape);
2650 }
2651
2652 //=============================================================================
2653 //----------------------------LoadKlassNode::make------------------------------
2654 // Polymorphic factory method:
2655 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2656 // sanity check the alias category against the created node type
2657 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2658 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2659 #ifdef _LP64
2660 if (adr_type->is_ptr_to_narrowklass()) {
2661 assert(UseCompressedClassPointers, "no compressed klasses");
2662 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2663 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2664 }
2665 #endif
2666 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2667 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2668 }
2669
2670 //------------------------------Value------------------------------------------
2671 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2705 }
2706 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2707 }
2708 if (!t->is_klass()) {
2709 // a primitive Class (e.g., int.class) has null for a klass field
2710 return TypePtr::NULL_PTR;
2711 }
2712 // Fold up the load of the hidden field
2713 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2714 }
2715 // non-constant mirror, so we can't tell what's going on
2716 }
2717 if (!tinst->is_loaded())
2718 return _type; // Bail out if not loaded
2719 if (offset == oopDesc::klass_offset_in_bytes()) {
2720 return tinst->as_klass_type(true);
2721 }
2722 }
2723
2724 // Check for loading klass from an array
2725 const TypeAryPtr* tary = tp->isa_aryptr();
2726 if (tary != nullptr &&
2727 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2728 return tary->as_klass_type(true)->is_aryklassptr();
2729 }
2730
2731 // Check for loading klass from an array klass
2732 const TypeKlassPtr *tkls = tp->isa_klassptr();
2733 if (tkls != nullptr && !StressReflectiveCode) {
2734 if (!tkls->is_loaded())
2735 return _type; // Bail out if not loaded
2736 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2737 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2738 // // Always returning precise element type is incorrect,
2739 // // e.g., element type could be object and array may contain strings
2740 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2741
2742 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2743 // according to the element type's subclassing.
2744 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2745 }
2746 if (tkls->isa_aryklassptr() != nullptr && tkls->klass_is_exact() &&
2747 !tkls->exact_klass()->is_type_array_klass() &&
2748 tkls->offset() == in_bytes(Klass::super_offset())) {
2749 // We are loading the super klass of a refined array klass, return the non-refined klass pointer
2750 assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2751 return tkls->is_aryklassptr()->with_offset(0)->cast_to_non_refined();
2752 }
2753 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2754 tkls->offset() == in_bytes(Klass::super_offset())) {
2755 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2756 // The field is Klass::_super. Return its (constant) value.
2757 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2758 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2759 }
2760 }
2761
2762 if (tkls != nullptr && !UseSecondarySupersCache
2763 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2764 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2765 return TypePtr::NULL_PTR;
2766 }
2767
2768 // Bailout case
2769 return LoadNode::Value(phase);
2770 }
2771
2772 //------------------------------Identity---------------------------------------
2795 base = bs->step_over_gc_barrier(base);
2796 }
2797
2798 // We can fetch the klass directly through an AllocateNode.
2799 // This works even if the klass is not constant (clone or newArray).
2800 if (offset == oopDesc::klass_offset_in_bytes()) {
2801 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2802 if (allocated_klass != nullptr) {
2803 return allocated_klass;
2804 }
2805 }
2806
2807 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2808 // See inline_native_Class_query for occurrences of these patterns.
2809 // Java Example: x.getClass().isAssignableFrom(y)
2810 //
2811 // This improves reflective code, often making the Class
2812 // mirror go completely dead. (Current exception: Class
2813 // mirrors may appear in debug info, but we could clean them out by
2814 // introducing a new debug info operator for Klass.java_mirror).
2815 //
2816 // This optimization does not apply to arrays because if k is not a
2817 // constant, it was obtained via load_klass which returns the VM type
2818 // and '.java_mirror.as_klass' should return the Java type instead.
2819
2820 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2821 && offset == java_lang_Class::klass_offset()) {
2822 if (base->is_Load()) {
2823 Node* base2 = base->in(MemNode::Address);
2824 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2825 Node* adr2 = base2->in(MemNode::Address);
2826 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2827 if (tkls != nullptr && !tkls->empty()
2828 && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2829 && adr2->is_AddP()) {
2830 int mirror_field = in_bytes(Klass::java_mirror_offset());
2831 if (tkls->offset() == mirror_field) {
2832 #ifdef ASSERT
2833 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2834 assert(tkls2->offset() == 0, "not a load of java_mirror");
2835 #endif
2836 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2837 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2838 return adr2->in(AddPNode::Address);
2839 }
2840 }
2841 }
2842 }
2843 }
2844
2845 return this;
2846 }
2847
2848 LoadNode* LoadNode::clone_pinned() const {
2849 LoadNode* ld = clone()->as_Load();
2975 //---------------------------StoreNode::make-----------------------------------
2976 // Polymorphic factory method:
2977 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2978 assert((mo == unordered || mo == release), "unexpected");
2979 Compile* C = gvn.C;
2980 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2981 ctl != nullptr, "raw memory operations should have control edge");
2982
2983 switch (bt) {
2984 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2985 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2986 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2987 case T_CHAR:
2988 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2989 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2990 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2991 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2992 case T_METADATA:
2993 case T_ADDRESS:
2994 case T_OBJECT:
2995 case T_ARRAY:
2996 #ifdef _LP64
2997 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2998 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2999 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
3000 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
3001 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
3002 adr->bottom_type()->isa_rawptr())) {
3003 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
3004 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
3005 }
3006 #endif
3007 {
3008 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
3009 }
3010 default:
3011 guarantee(false, "unexpected basic type %s", type2name(bt));
3012 return (StoreNode*)nullptr;
3013 }
3014 }
3015
3016 //--------------------------bottom_type----------------------------------------
3017 const Type *StoreNode::bottom_type() const {
3018 return Type::MEMORY;
3019 }
3020
3021 //------------------------------hash-------------------------------------------
3022 uint StoreNode::hash() const {
3023 // unroll addition of interesting fields
3024 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
3025
3026 // Since they are not commoned, do not hash them:
3027 return NO_HASH;
3028 }
3029
3030 // Link together multiple stores (B/S/C/I) into a longer one.
3031 //
3653 }
3654 ss.print_cr("[TraceMergeStores]: with");
3655 merged_input_value->dump("\n", false, &ss);
3656 merged_store->dump("\n", false, &ss);
3657 tty->print("%s", ss.as_string());
3658 }
3659 #endif
3660
3661 //------------------------------Ideal------------------------------------------
3662 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3663 // When a store immediately follows a relevant allocation/initialization,
3664 // try to capture it into the initialization, or hoist it above.
3665 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3666 Node* p = MemNode::Ideal_common(phase, can_reshape);
3667 if (p) return (p == NodeSentinel) ? nullptr : p;
3668
3669 Node* mem = in(MemNode::Memory);
3670 Node* address = in(MemNode::Address);
3671 Node* value = in(MemNode::ValueIn);
3672 // Back-to-back stores to same address? Fold em up. Generally
3673 // unsafe if I have intervening uses...
3674 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3675 Node* st = mem;
3676 // If Store 'st' has more than one use, we cannot fold 'st' away.
3677 // For example, 'st' might be the final state at a conditional
3678 // return. Or, 'st' might be used by some node which is live at
3679 // the same time 'st' is live, which might be unschedulable. So,
3680 // require exactly ONE user until such time as we clone 'mem' for
3681 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3682 // true).
3683 while (st->is_Store() && st->outcnt() == 1) {
3684 // Looking at a dead closed cycle of memory?
3685 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3686 assert(Opcode() == st->Opcode() ||
3687 st->Opcode() == Op_StoreVector ||
3688 Opcode() == Op_StoreVector ||
3689 st->Opcode() == Op_StoreVectorScatter ||
3690 Opcode() == Op_StoreVectorScatter ||
3691 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3692 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3693 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3694 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3695 (st->adr_type()->isa_aryptr() && st->adr_type()->is_aryptr()->is_flat()) || // TODO 8343835
3696 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3697 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3698
3699 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3700 st->as_Store()->memory_size() <= this->memory_size()) {
3701 Node* use = st->raw_out(0);
3702 if (phase->is_IterGVN()) {
3703 phase->is_IterGVN()->rehash_node_delayed(use);
3704 }
3705 // It's OK to do this in the parser, since DU info is always accurate,
3706 // and the parser always refers to nodes via SafePointNode maps.
3707 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3708 return this;
3709 }
3710 st = st->in(MemNode::Memory);
3711 }
3712 }
3713
3714
3715 // Capture an unaliased, unconditional, simple store into an initializer.
3813 const StoreVectorNode* store_vector = as_StoreVector();
3814 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3815 const Node* store_indices = store_vector->indices();
3816 const Node* mem_indices = mem_vector->indices();
3817 const Node* store_mask = store_vector->mask();
3818 const Node* mem_mask = mem_vector->mask();
3819 // Ensure types, indices, and masks match
3820 if (store_vector->vect_type() == mem_vector->vect_type() &&
3821 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3822 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3823 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3824 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3825 result = mem;
3826 }
3827 }
3828 }
3829
3830 // Store of zero anywhere into a freshly-allocated object?
3831 // Then the store is useless.
3832 // (It must already have been captured by the InitializeNode.)
3833 if (result == this && ReduceFieldZeroing) {
3834 // a newly allocated object is already all-zeroes everywhere
3835 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3836 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3837 result = mem;
3838 }
3839
3840 if (result == this && phase->type(val)->is_zero_type()) {
3841 // the store may also apply to zero-bits in an earlier object
3842 Node* prev_mem = find_previous_store(phase);
3843 // Steps (a), (b): Walk past independent stores to find an exact match.
3844 if (prev_mem != nullptr) {
3845 Node* prev_val = can_see_stored_value(prev_mem, phase);
3846 if (prev_val != nullptr && prev_val == val) {
3847 // prev_val and val might differ by a cast; it would be good
3848 // to keep the more informative of the two.
3849 result = mem;
3850 }
3851 }
3852 }
3853 }
3854
3855 PhaseIterGVN* igvn = phase->is_IterGVN();
3856 if (result != this && igvn != nullptr) {
3857 MemBarNode* trailing = trailing_membar();
3858 if (trailing != nullptr) {
3859 #ifdef ASSERT
3860 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
4329 // Clearing a short array is faster with stores
4330 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4331 // Already know this is a large node, do not try to ideal it
4332 if (_is_large) return nullptr;
4333
4334 const int unit = BytesPerLong;
4335 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4336 if (!t) return nullptr;
4337 if (!t->is_con()) return nullptr;
4338 intptr_t raw_count = t->get_con();
4339 intptr_t size = raw_count;
4340 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4341 // Clearing nothing uses the Identity call.
4342 // Negative clears are possible on dead ClearArrays
4343 // (see jck test stmt114.stmt11402.val).
4344 if (size <= 0 || size % unit != 0) return nullptr;
4345 intptr_t count = size / unit;
4346 // Length too long; communicate this to matchers and assemblers.
4347 // Assemblers are responsible to produce fast hardware clears for it.
4348 if (size > InitArrayShortSize) {
4349 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4350 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4351 return nullptr;
4352 }
4353 if (!IdealizeClearArrayNode) return nullptr;
4354 Node *mem = in(1);
4355 if( phase->type(mem)==Type::TOP ) return nullptr;
4356 Node *adr = in(3);
4357 const Type* at = phase->type(adr);
4358 if( at==Type::TOP ) return nullptr;
4359 const TypePtr* atp = at->isa_ptr();
4360 // adjust atp to be the correct array element address type
4361 if (atp == nullptr) atp = TypePtr::BOTTOM;
4362 else atp = atp->add_offset(Type::OffsetBot);
4363 // Get base for derived pointer purposes
4364 if( adr->Opcode() != Op_AddP ) Unimplemented();
4365 Node *base = adr->in(1);
4366
4367 Node *val = in(4);
4368 Node *off = phase->MakeConX(BytesPerLong);
4369 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4370 count--;
4371 while( count-- ) {
4372 mem = phase->transform(mem);
4373 adr = phase->transform(new AddPNode(base,adr,off));
4374 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4375 }
4376 return mem;
4377 }
4378
4379 //----------------------------step_through----------------------------------
4380 // Return allocation input memory edge if it is different instance
4381 // or itself if it is the one we are looking for.
4382 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4383 Node* n = *np;
4384 assert(n->is_ClearArray(), "sanity");
4385 intptr_t offset;
4386 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4387 // This method is called only before Allocate nodes are expanded
4388 // during macro nodes expansion. Before that ClearArray nodes are
4389 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4390 // Allocate nodes are expanded) which follows allocations.
4391 assert(alloc != nullptr, "should have allocation");
4392 if (alloc->_idx == instance_id) {
4393 // Can not bypass initialization of the instance we are looking for.
4394 return false;
4397 InitializeNode* init = alloc->initialization();
4398 if (init != nullptr)
4399 *np = init->in(TypeFunc::Memory);
4400 else
4401 *np = alloc->in(TypeFunc::Memory);
4402 return true;
4403 }
4404
4405 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4406 Node* base = dest;
4407 if (raw_base) {
4408 // May be called as part of the initialization of a just allocated object
4409 base = phase->C->top();
4410 }
4411 return phase->transform(new AddPNode(base, dest, offset));
4412 }
4413
4414 //----------------------------clear_memory-------------------------------------
4415 // Generate code to initialize object storage to zero.
4416 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4417 Node* val,
4418 Node* raw_val,
4419 intptr_t start_offset,
4420 Node* end_offset,
4421 bool raw_base,
4422 PhaseGVN* phase) {
4423 intptr_t offset = start_offset;
4424
4425 int unit = BytesPerLong;
4426 if ((offset % unit) != 0) {
4427 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4428 const TypePtr* atp = TypeRawPtr::BOTTOM;
4429 if (val != nullptr) {
4430 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4431 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4432 } else {
4433 assert(raw_val == nullptr, "val may not be null");
4434 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4435 }
4436 mem = phase->transform(mem);
4437 offset += BytesPerInt;
4438 }
4439 assert((offset % unit) == 0, "");
4440
4441 // Initialize the remaining stuff, if any, with a ClearArray.
4442 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, raw_base, phase);
4443 }
4444
4445 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4446 Node* raw_val,
4447 Node* start_offset,
4448 Node* end_offset,
4449 bool raw_base,
4450 PhaseGVN* phase) {
4451 if (start_offset == end_offset) {
4452 // nothing to do
4453 return mem;
4454 }
4455
4456 int unit = BytesPerLong;
4457 Node* zbase = start_offset;
4458 Node* zend = end_offset;
4459
4460 // Scale to the unit required by the CPU:
4461 if (!Matcher::init_array_count_is_in_bytes) {
4462 Node* shift = phase->intcon(exact_log2(unit));
4463 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4464 zend = phase->transform(new URShiftXNode(zend, shift) );
4465 }
4466
4467 // Bulk clear double-words
4468 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4469 Node* adr = make_address(dest, start_offset, raw_base, phase);
4470 if (raw_val == nullptr) {
4471 raw_val = phase->MakeConX(0);
4472 }
4473 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4474 return phase->transform(mem);
4475 }
4476
4477 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4478 Node* val,
4479 Node* raw_val,
4480 intptr_t start_offset,
4481 intptr_t end_offset,
4482 bool raw_base,
4483 PhaseGVN* phase) {
4484 if (start_offset == end_offset) {
4485 // nothing to do
4486 return mem;
4487 }
4488
4489 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4490 intptr_t done_offset = end_offset;
4491 if ((done_offset % BytesPerLong) != 0) {
4492 done_offset -= BytesPerInt;
4493 }
4494 if (done_offset > start_offset) {
4495 mem = clear_memory(ctl, mem, dest, val, raw_val,
4496 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4497 }
4498 if (done_offset < end_offset) { // emit the final 32-bit store
4499 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4500 const TypePtr* atp = TypeRawPtr::BOTTOM;
4501 if (val != nullptr) {
4502 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4503 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4504 } else {
4505 assert(raw_val == nullptr, "val may not be null");
4506 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4507 }
4508 mem = phase->transform(mem);
4509 done_offset += BytesPerInt;
4510 }
4511 assert(done_offset == end_offset, "");
4512 return mem;
4513 }
4514
4515 //=============================================================================
4516 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4517 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4518 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4519 #ifdef ASSERT
4520 , _pair_idx(0)
4521 #endif
4522 {
4523 init_class_id(Class_MemBar);
4524 Node* top = C->top();
4525 init_req(TypeFunc::I_O,top);
4526 init_req(TypeFunc::FramePtr,top);
4527 init_req(TypeFunc::ReturnAdr,top);
4634 PhaseIterGVN* igvn = phase->is_IterGVN();
4635 remove(igvn);
4636 // Must return either the original node (now dead) or a new node
4637 // (Do not return a top here, since that would break the uniqueness of top.)
4638 return new ConINode(TypeInt::ZERO);
4639 }
4640 }
4641 return progress ? this : nullptr;
4642 }
4643
4644 //------------------------------Value------------------------------------------
4645 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4646 if( !in(0) ) return Type::TOP;
4647 if( phase->type(in(0)) == Type::TOP )
4648 return Type::TOP;
4649 return TypeTuple::MEMBAR;
4650 }
4651
4652 //------------------------------match------------------------------------------
4653 // Construct projections for memory.
4654 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4655 switch (proj->_con) {
4656 case TypeFunc::Control:
4657 case TypeFunc::Memory:
4658 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4659 }
4660 ShouldNotReachHere();
4661 return nullptr;
4662 }
4663
4664 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4665 trailing->_kind = TrailingStore;
4666 leading->_kind = LeadingStore;
4667 #ifdef ASSERT
4668 trailing->_pair_idx = leading->_idx;
4669 leading->_pair_idx = leading->_idx;
4670 #endif
4671 }
4672
4673 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4674 trailing->_kind = TrailingLoadStore;
4921 return (req() > RawStores);
4922 }
4923
4924 void InitializeNode::set_complete(PhaseGVN* phase) {
4925 assert(!is_complete(), "caller responsibility");
4926 _is_complete = Complete;
4927
4928 // After this node is complete, it contains a bunch of
4929 // raw-memory initializations. There is no need for
4930 // it to have anything to do with non-raw memory effects.
4931 // Therefore, tell all non-raw users to re-optimize themselves,
4932 // after skipping the memory effects of this initialization.
4933 PhaseIterGVN* igvn = phase->is_IterGVN();
4934 if (igvn) igvn->add_users_to_worklist(this);
4935 }
4936
4937 // convenience function
4938 // return false if the init contains any stores already
4939 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4940 InitializeNode* init = initialization();
4941 if (init == nullptr || init->is_complete()) {
4942 return false;
4943 }
4944 init->remove_extra_zeroes();
4945 // for now, if this allocation has already collected any inits, bail:
4946 if (init->is_non_zero()) return false;
4947 init->set_complete(phase);
4948 return true;
4949 }
4950
4951 void InitializeNode::remove_extra_zeroes() {
4952 if (req() == RawStores) return;
4953 Node* zmem = zero_memory();
4954 uint fill = RawStores;
4955 for (uint i = fill; i < req(); i++) {
4956 Node* n = in(i);
4957 if (n->is_top() || n == zmem) continue; // skip
4958 if (fill < i) set_req(fill, n); // compact
4959 ++fill;
4960 }
4961 // delete any empty spaces created:
4962 while (fill < req()) {
4963 del_req(fill);
5107 // store node that we'd like to capture. We need to check
5108 // the uses of the MergeMemNode.
5109 mems.push(n);
5110 }
5111 } else if (n->is_Mem()) {
5112 Node* other_adr = n->in(MemNode::Address);
5113 if (other_adr == adr) {
5114 failed = true;
5115 break;
5116 } else {
5117 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5118 if (other_t_adr != nullptr) {
5119 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5120 if (other_alias_idx == alias_idx) {
5121 // A load from the same memory slice as the store right
5122 // after the InitializeNode. We check the control of the
5123 // object/array that is loaded from. If it's the same as
5124 // the store control then we cannot capture the store.
5125 assert(!n->is_Store(), "2 stores to same slice on same control?");
5126 Node* base = other_adr;
5127 if (base->is_Phi()) {
5128 // In rare case, base may be a PhiNode and it may read
5129 // the same memory slice between InitializeNode and store.
5130 failed = true;
5131 break;
5132 }
5133 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5134 base = base->in(AddPNode::Base);
5135 if (base != nullptr) {
5136 base = base->uncast();
5137 if (base->is_Proj() && base->in(0) == alloc) {
5138 failed = true;
5139 break;
5140 }
5141 }
5142 }
5143 }
5144 }
5145 } else {
5146 failed = true;
5147 break;
5148 }
5149 }
5150 }
5151 }
5152 if (failed) {
5699 // z's_done 12 16 16 16 12 16 12
5700 // z's_needed 12 16 16 16 16 16 16
5701 // zsize 0 0 0 0 4 0 4
5702 if (next_full_store < 0) {
5703 // Conservative tack: Zero to end of current word.
5704 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5705 } else {
5706 // Zero to beginning of next fully initialized word.
5707 // Or, don't zero at all, if we are already in that word.
5708 assert(next_full_store >= zeroes_needed, "must go forward");
5709 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5710 zeroes_needed = next_full_store;
5711 }
5712 }
5713
5714 if (zeroes_needed > zeroes_done) {
5715 intptr_t zsize = zeroes_needed - zeroes_done;
5716 // Do some incremental zeroing on rawmem, in parallel with inits.
5717 zeroes_done = align_down(zeroes_done, BytesPerInt);
5718 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5719 allocation()->in(AllocateNode::InitValue),
5720 allocation()->in(AllocateNode::RawInitValue),
5721 zeroes_done, zeroes_needed,
5722 true,
5723 phase);
5724 zeroes_done = zeroes_needed;
5725 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5726 do_zeroing = false; // leave the hole, next time
5727 }
5728 }
5729
5730 // Collect the store and move on:
5731 phase->replace_input_of(st, MemNode::Memory, inits);
5732 inits = st; // put it on the linearized chain
5733 set_req(i, zmem); // unhook from previous position
5734
5735 if (zeroes_done == st_off)
5736 zeroes_done = next_init_off;
5737
5738 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5739
5740 #ifdef ASSERT
5761 remove_extra_zeroes(); // clear out all the zmems left over
5762 add_req(inits);
5763
5764 if (!(UseTLAB && ZeroTLAB)) {
5765 // If anything remains to be zeroed, zero it all now.
5766 zeroes_done = align_down(zeroes_done, BytesPerInt);
5767 // if it is the last unused 4 bytes of an instance, forget about it
5768 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5769 if (zeroes_done + BytesPerLong >= size_limit) {
5770 AllocateNode* alloc = allocation();
5771 assert(alloc != nullptr, "must be present");
5772 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5773 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5774 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5775 if (zeroes_done == k->layout_helper())
5776 zeroes_done = size_limit;
5777 }
5778 }
5779 if (zeroes_done < size_limit) {
5780 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5781 allocation()->in(AllocateNode::InitValue),
5782 allocation()->in(AllocateNode::RawInitValue),
5783 zeroes_done, size_in_bytes, true, phase);
5784 }
5785 }
5786
5787 set_complete(phase);
5788 return rawmem;
5789 }
5790
5791 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5792 auto replace_proj = [&](ProjNode* proj) {
5793 C->gvn_replace_by(proj, mem);
5794 return CONTINUE;
5795 };
5796 apply_to_projs(replace_proj, TypeFunc::Memory);
5797 }
5798
5799 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5800 DUIterator_Fast imax, i = fast_outs(imax);
5801 auto replace_proj = [&](ProjNode* proj) {
5802 igvn->replace_node(proj, mem);
|