6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/compile.hpp"
38 #include "opto/connode.hpp"
39 #include "opto/convertnode.hpp"
40 #include "opto/loopnode.hpp"
41 #include "opto/machnode.hpp"
42 #include "opto/matcher.hpp"
43 #include "opto/memnode.hpp"
44 #include "opto/mempointer.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/phaseX.hpp"
48 #include "opto/regalloc.hpp"
49 #include "opto/regmask.hpp"
50 #include "opto/rootnode.hpp"
51 #include "opto/traceMergeStoresTag.hpp"
52 #include "opto/vectornode.hpp"
53 #include "utilities/align.hpp"
54 #include "utilities/copy.hpp"
55 #include "utilities/macros.hpp"
56 #include "utilities/powerOfTwo.hpp"
57 #include "utilities/vmError.hpp"
58
59 // Portions of code courtesy of Clifford Click
60
61 // Optimization - Graph Style
62
63 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
64
65 //=============================================================================
66 uint MemNode::size_of() const { return sizeof(*this); }
67
68 const TypePtr *MemNode::adr_type() const {
69 Node* adr = in(Address);
70 if (adr == nullptr) return nullptr; // node is dead
71 const TypePtr* cross_check = nullptr;
72 DEBUG_ONLY(cross_check = _adr_type);
73 return calculate_adr_type(adr->bottom_type(), cross_check);
74 }
123 st->print(", idx=Bot;");
124 else if (atp->index() == Compile::AliasIdxTop)
125 st->print(", idx=Top;");
126 else if (atp->index() == Compile::AliasIdxRaw)
127 st->print(", idx=Raw;");
128 else {
129 ciField* field = atp->field();
130 if (field) {
131 st->print(", name=");
132 field->print_name_on(st);
133 }
134 st->print(", idx=%d;", atp->index());
135 }
136 }
137 }
138
139 extern void print_alias_types();
140
141 #endif
142
143 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
144 assert((t_oop != nullptr), "sanity");
145 bool is_instance = t_oop->is_known_instance_field();
146 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
147 (load != nullptr) && load->is_Load() &&
148 (phase->is_IterGVN() != nullptr);
149 if (!(is_instance || is_boxed_value_load))
150 return mchain; // don't try to optimize non-instance types
151 uint instance_id = t_oop->instance_id();
152 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
153 Node *prev = nullptr;
154 Node *result = mchain;
155 while (prev != result) {
156 prev = result;
157 if (result == start_mem)
158 break; // hit one of our sentinels
159 // skip over a call which does not affect this memory slice
160 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
161 Node *proj_in = result->in(0);
162 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
163 break; // hit one of our sentinels
164 } else if (proj_in->is_Call()) {
165 // ArrayCopyNodes processed here as well
166 CallNode *call = proj_in->as_Call();
167 if (!call->may_modify(t_oop, phase)) { // returns false for instances
168 result = call->in(TypeFunc::Memory);
169 }
170 } else if (proj_in->is_Initialize()) {
171 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
172 // Stop if this is the initialization for the object instance which
173 // contains this memory slice, otherwise skip over it.
174 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
175 break;
176 }
177 if (is_instance) {
178 result = proj_in->in(TypeFunc::Memory);
179 } else if (is_boxed_value_load) {
180 Node* klass = alloc->in(AllocateNode::KlassNode);
181 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
182 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
183 result = proj_in->in(TypeFunc::Memory); // not related allocation
184 }
185 }
186 } else if (proj_in->is_MemBar()) {
187 ArrayCopyNode* ac = nullptr;
188 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
189 break;
190 }
191 result = proj_in->in(TypeFunc::Memory);
192 } else if (proj_in->is_top()) {
193 break; // dead code
194 } else {
195 assert(false, "unexpected projection");
196 }
197 } else if (result->is_ClearArray()) {
198 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
199 // Can not bypass initialization of the instance
200 // we are looking for.
201 break;
202 }
203 // Otherwise skip it (the call updated 'result' value).
204 } else if (result->is_MergeMem()) {
205 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
206 }
207 }
208 return result;
209 }
210
211 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
212 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
213 if (t_oop == nullptr)
214 return mchain; // don't try to optimize non-oop types
215 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
216 bool is_instance = t_oop->is_known_instance_field();
217 PhaseIterGVN *igvn = phase->is_IterGVN();
218 if (is_instance && igvn != nullptr && result->is_Phi()) {
219 PhiNode *mphi = result->as_Phi();
220 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
221 const TypePtr *t = mphi->adr_type();
222 bool do_split = false;
223 // In the following cases, Load memory input can be further optimized based on
224 // its precise address type
225 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
226 do_split = true;
227 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
228 const TypeOopPtr* mem_t =
229 t->is_oopptr()->cast_to_exactness(true)
230 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
231 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
232 if (t_oop->isa_aryptr()) {
233 mem_t = mem_t->is_aryptr()
234 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
235 ->cast_to_size(t_oop->is_aryptr()->size())
236 ->with_offset(t_oop->is_aryptr()->offset())
237 ->is_aryptr();
238 }
239 do_split = mem_t == t_oop;
240 }
241 if (do_split) {
242 // clone the Phi with our address type
243 result = mphi->split_out_instance(t_adr, igvn);
244 } else {
245 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
246 }
247 }
248 return result;
249 }
250
251 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
252 uint alias_idx = phase->C->get_alias_index(tp);
253 Node *mem = mmem;
254 #ifdef ASSERT
255 {
256 // Check that current type is consistent with the alias index used during graph construction
257 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
258 bool consistent = adr_check == nullptr || adr_check->empty() ||
259 phase->C->must_alias(adr_check, alias_idx );
260 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
261 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
262 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
263 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
264 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
265 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
266 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
267 // don't assert if it is dead code.
268 consistent = true;
269 }
270 if( !consistent ) {
271 st->print("alias_idx==%d, adr_check==", alias_idx);
272 if( adr_check == nullptr ) {
273 st->print("null");
274 } else {
275 adr_check->dump();
276 }
277 st->cr();
278 print_alias_types();
279 assert(consistent, "adr_check must match alias idx");
280 }
281 }
282 #endif
954 "use LoadKlassNode instead");
955 assert(!(adr_type->isa_aryptr() &&
956 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
957 "use LoadRangeNode instead");
958 // Check control edge of raw loads
959 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
960 // oop will be recorded in oop map if load crosses safepoint
961 rt->isa_oopptr() || is_immutable_value(adr),
962 "raw memory operations should have control edge");
963 LoadNode* load = nullptr;
964 switch (bt) {
965 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
966 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
967 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
968 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
969 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
970 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
971 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
972 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
973 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
974 case T_OBJECT:
975 case T_NARROWOOP:
976 #ifdef _LP64
977 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
978 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
979 } else
980 #endif
981 {
982 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
983 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
984 }
985 break;
986 default:
987 ShouldNotReachHere();
988 break;
989 }
990 assert(load != nullptr, "LoadNode should have been created");
991 if (unaligned) {
992 load->set_unaligned_access();
993 }
994 if (mismatched) {
995 load->set_mismatched_access();
996 }
997 if (unsafe) {
998 load->set_unsafe_access();
999 }
1000 load->set_barrier_data(barrier_data);
1001 if (load->Opcode() == Op_LoadN) {
1002 Node* ld = gvn.transform(load);
1003 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1004 }
1005
1006 return load;
1007 }
1008
1009 //------------------------------hash-------------------------------------------
1010 uint LoadNode::hash() const {
1011 // unroll addition of interesting fields
1012 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1013 }
1014
1015 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1016 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1017 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1018 bool is_stable_ary = FoldStableValues &&
1019 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1020 tp->isa_aryptr()->is_stable();
1021
1022 return (eliminate_boxing && non_volatile) || is_stable_ary;
1023 }
1024
1025 return false;
1026 }
1027
1028 LoadNode* LoadNode::pin_array_access_node() const {
1029 const TypePtr* adr_type = this->adr_type();
1030 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1031 return clone_pinned();
1032 }
1033 return nullptr;
1034 }
1035
1036 // Is the value loaded previously stored by an arraycopy? If so return
1037 // a load node that reads from the source array so we may be able to
1038 // optimize out the ArrayCopy node later.
1039 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1040 Node* ld_adr = in(MemNode::Address);
1041 intptr_t ld_off = 0;
1042 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1058 if (ac->as_ArrayCopy()->is_clonebasic()) {
1059 assert(ld_alloc != nullptr, "need an alloc");
1060 assert(addp->is_AddP(), "address must be addp");
1061 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1062 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1063 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1064 addp->set_req(AddPNode::Base, src);
1065 addp->set_req(AddPNode::Address, src);
1066 } else {
1067 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1068 ac->as_ArrayCopy()->is_copyof_validated() ||
1069 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1070 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1071 addp->set_req(AddPNode::Base, src);
1072 addp->set_req(AddPNode::Address, src);
1073
1074 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1075 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1076 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1077
1078 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1079 uint shift = exact_log2(type2aelembytes(ary_elem));
1080
1081 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1082 #ifdef _LP64
1083 diff = phase->transform(new ConvI2LNode(diff));
1084 #endif
1085 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1086
1087 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1088 addp->set_req(AddPNode::Offset, offset);
1089 }
1090 addp = phase->transform(addp);
1091 #ifdef ASSERT
1092 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1093 ld->_adr_type = adr_type;
1094 #endif
1095 ld->set_req(MemNode::Address, addp);
1096 ld->set_req(0, ctl);
1097 ld->set_req(MemNode::Memory, mem);
1098 return ld;
1099 }
1100 return nullptr;
1101 }
1102
1103
1104 //---------------------------can_see_stored_value------------------------------
1105 // This routine exists to make sure this set of tests is done the same
1106 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1107 // will change the graph shape in a way which makes memory alive twice at the
1108 // same time (uses the Oracle model of aliasing), then some
1109 // LoadXNode::Identity will fold things back to the equivalence-class model
1110 // of aliasing.
1111 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1112 Node* ld_adr = in(MemNode::Address);
1113 intptr_t ld_off = 0;
1114 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1115 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1116 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1117 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1118 // This is more general than load from boxing objects.
1119 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1120 uint alias_idx = atp->index();
1121 Node* result = nullptr;
1122 Node* current = st;
1123 // Skip through chains of MemBarNodes checking the MergeMems for
1124 // new states for the slice of this load. Stop once any other
1125 // kind of node is encountered. Loads from final memory can skip
1126 // through any kind of MemBar but normal loads shouldn't skip
1127 // through MemBarAcquire since the could allow them to move out of
1128 // a synchronized region. It is not safe to step over MemBarCPUOrder,
1129 // because alias info above them may be inaccurate (e.g., due to
1130 // mixed/mismatched unsafe accesses).
1131 bool is_final_mem = !atp->is_rewritable();
1132 while (current->is_Proj()) {
1133 int opc = current->in(0)->Opcode();
1134 if ((is_final_mem && (opc == Op_MemBarAcquire ||
1178 // Same base, same offset.
1179 // Possible improvement for arrays: check index value instead of absolute offset.
1180
1181 // At this point we have proven something like this setup:
1182 // B = << base >>
1183 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1184 // S = StoreQ(AddP( B , #Off), V)
1185 // (Actually, we haven't yet proven the Q's are the same.)
1186 // In other words, we are loading from a casted version of
1187 // the same pointer-and-offset that we stored to.
1188 // Casted version may carry a dependency and it is respected.
1189 // Thus, we are able to replace L by V.
1190 }
1191 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1192 if (store_Opcode() != st->Opcode()) {
1193 return nullptr;
1194 }
1195 // LoadVector/StoreVector needs additional check to ensure the types match.
1196 if (st->is_StoreVector()) {
1197 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1198 const TypeVect* out_vt = as_LoadVector()->vect_type();
1199 if (in_vt != out_vt) {
1200 return nullptr;
1201 }
1202 }
1203 return st->in(MemNode::ValueIn);
1204 }
1205
1206 // A load from a freshly-created object always returns zero.
1207 // (This can happen after LoadNode::Ideal resets the load's memory input
1208 // to find_captured_store, which returned InitializeNode::zero_memory.)
1209 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1210 (st->in(0) == ld_alloc) &&
1211 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1212 // return a zero value for the load's basic type
1213 // (This is one of the few places where a generic PhaseTransform
1214 // can create new nodes. Think of it as lazily manifesting
1215 // virtually pre-existing constants.)
1216 if (value_basic_type() != T_VOID) {
1217 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1218 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1219 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1220 // by the ArrayCopyNode.
1221 return phase->zerocon(value_basic_type());
1222 }
1223 } else {
1224 // TODO: materialize all-zero vector constant
1225 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1226 }
1227 }
1228
1229 // A load from an initialization barrier can match a captured store.
1230 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1231 InitializeNode* init = st->in(0)->as_Initialize();
1232 AllocateNode* alloc = init->allocation();
1233 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1234 // examine a captured store value
1235 st = init->find_captured_store(ld_off, memory_size(), phase);
1248 base = bs->step_over_gc_barrier(base);
1249 if (base != nullptr && base->is_Proj() &&
1250 base->as_Proj()->_con == TypeFunc::Parms &&
1251 base->in(0)->is_CallStaticJava() &&
1252 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1253 return base->in(0)->in(TypeFunc::Parms);
1254 }
1255 }
1256
1257 break;
1258 }
1259
1260 return nullptr;
1261 }
1262
1263 //----------------------is_instance_field_load_with_local_phi------------------
1264 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1265 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1266 in(Address)->is_AddP() ) {
1267 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1268 // Only instances and boxed values.
1269 if( t_oop != nullptr &&
1270 (t_oop->is_ptr_to_boxed_value() ||
1271 t_oop->is_known_instance_field()) &&
1272 t_oop->offset() != Type::OffsetBot &&
1273 t_oop->offset() != Type::OffsetTop) {
1274 return true;
1275 }
1276 }
1277 return false;
1278 }
1279
1280 //------------------------------Identity---------------------------------------
1281 // Loads are identity if previous store is to same address
1282 Node* LoadNode::Identity(PhaseGVN* phase) {
1283 // If the previous store-maker is the right kind of Store, and the store is
1284 // to the same address, then we are equal to the value stored.
1285 Node* mem = in(Memory);
1286 Node* value = can_see_stored_value(mem, phase);
1287 if( value ) {
1288 // byte, short & char stores truncate naturally.
1289 // A load has to load the truncated value which requires
1290 // some sort of masking operation and that requires an
1299 // usually runs first, producing the singleton type of the Con.)
1300 if (!has_pinned_control_dependency() || value->is_Con()) {
1301 return value;
1302 } else {
1303 return this;
1304 }
1305 }
1306
1307 if (has_pinned_control_dependency()) {
1308 return this;
1309 }
1310 // Search for an existing data phi which was generated before for the same
1311 // instance's field to avoid infinite generation of phis in a loop.
1312 Node *region = mem->in(0);
1313 if (is_instance_field_load_with_local_phi(region)) {
1314 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1315 int this_index = phase->C->get_alias_index(addr_t);
1316 int this_offset = addr_t->offset();
1317 int this_iid = addr_t->instance_id();
1318 if (!addr_t->is_known_instance() &&
1319 addr_t->is_ptr_to_boxed_value()) {
1320 // Use _idx of address base (could be Phi node) for boxed values.
1321 intptr_t ignore = 0;
1322 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1323 if (base == nullptr) {
1324 return this;
1325 }
1326 this_iid = base->_idx;
1327 }
1328 const Type* this_type = bottom_type();
1329 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1330 Node* phi = region->fast_out(i);
1331 if (phi->is_Phi() && phi != mem &&
1332 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1333 return phi;
1334 }
1335 }
1336 }
1337
1338 return this;
1339 }
1340
1856 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1857 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1858
1859 // Skip up past a SafePoint control. Cannot do this for Stores because
1860 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1861 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1862 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1863 !addr_mark &&
1864 (depends_only_on_test() || has_unknown_control_dependency())) {
1865 ctrl = ctrl->in(0);
1866 set_req(MemNode::Control,ctrl);
1867 progress = true;
1868 }
1869
1870 intptr_t ignore = 0;
1871 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1872 if (base != nullptr
1873 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1874 // Check for useless control edge in some common special cases
1875 if (in(MemNode::Control) != nullptr
1876 && can_remove_control()
1877 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1878 && all_controls_dominate(base, phase->C->start())) {
1879 // A method-invariant, non-null address (constant or 'this' argument).
1880 set_req(MemNode::Control, nullptr);
1881 progress = true;
1882 }
1883 }
1884
1885 Node* mem = in(MemNode::Memory);
1886 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1887
1888 if (can_reshape && (addr_t != nullptr)) {
1889 // try to optimize our memory input
1890 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1891 if (opt_mem != mem) {
1892 set_req_X(MemNode::Memory, opt_mem, phase);
1893 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1894 return this;
1895 }
1952 // fold up, do so.
1953 Node* prev_mem = find_previous_store(phase);
1954 if (prev_mem != nullptr) {
1955 Node* value = can_see_arraycopy_value(prev_mem, phase);
1956 if (value != nullptr) {
1957 return value;
1958 }
1959 }
1960 // Steps (a), (b): Walk past independent stores to find an exact match.
1961 if (prev_mem != nullptr && prev_mem != in(MemNode::Memory)) {
1962 // (c) See if we can fold up on the spot, but don't fold up here.
1963 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
1964 // just return a prior value, which is done by Identity calls.
1965 if (can_see_stored_value(prev_mem, phase)) {
1966 // Make ready for step (d):
1967 set_req_X(MemNode::Memory, prev_mem, phase);
1968 return this;
1969 }
1970 }
1971
1972 return progress ? this : nullptr;
1973 }
1974
1975 // Helper to recognize certain Klass fields which are invariant across
1976 // some group of array types (e.g., int[] or all T[] where T < Object).
1977 const Type*
1978 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
1979 ciKlass* klass) const {
1980 assert(!UseCompactObjectHeaders || tkls->offset() != in_bytes(Klass::prototype_header_offset()),
1981 "must not happen");
1982 if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
1983 // The field is Klass::_access_flags. Return its (constant) value.
1984 assert(Opcode() == Op_LoadUS, "must load an unsigned short from _access_flags");
1985 return TypeInt::make(klass->access_flags());
1986 }
1987 if (tkls->offset() == in_bytes(Klass::misc_flags_offset())) {
1988 // The field is Klass::_misc_flags. Return its (constant) value.
1989 assert(Opcode() == Op_LoadUB, "must load an unsigned byte from _misc_flags");
1990 return TypeInt::make(klass->misc_flags());
1991 }
1992 if (tkls->offset() == in_bytes(Klass::layout_helper_offset())) {
2052 }
2053 }
2054
2055 // Don't do this for integer types. There is only potential profit if
2056 // the element type t is lower than _type; that is, for int types, if _type is
2057 // more restrictive than t. This only happens here if one is short and the other
2058 // char (both 16 bits), and in those cases we've made an intentional decision
2059 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2060 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2061 //
2062 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2063 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2064 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2065 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2066 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2067 // In fact, that could have been the original type of p1, and p1 could have
2068 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2069 // expression (LShiftL quux 3) independently optimized to the constant 8.
2070 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2071 && (_type->isa_vect() == nullptr)
2072 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2073 // t might actually be lower than _type, if _type is a unique
2074 // concrete subclass of abstract class t.
2075 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2076 const Type* jt = t->join_speculative(_type);
2077 // In any case, do not allow the join, per se, to empty out the type.
2078 if (jt->empty() && !t->empty()) {
2079 // This can happen if a interface-typed array narrows to a class type.
2080 jt = _type;
2081 }
2082 #ifdef ASSERT
2083 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2084 // The pointers in the autobox arrays are always non-null
2085 Node* base = adr->in(AddPNode::Base);
2086 if ((base != nullptr) && base->is_DecodeN()) {
2087 // Get LoadN node which loads IntegerCache.cache field
2088 base = base->in(1);
2089 }
2090 if ((base != nullptr) && base->is_Con()) {
2091 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2092 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2093 // It could be narrow oop
2094 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2095 }
2096 }
2097 }
2098 #endif
2099 return jt;
2100 }
2101 }
2102 } else if (tp->base() == Type::InstPtr) {
2103 assert( off != Type::OffsetBot ||
2104 // arrays can be cast to Objects
2105 !tp->isa_instptr() ||
2106 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2107 // unsafe field access may not have a constant offset
2108 C->has_unsafe_access(),
2109 "Field accesses must be precise" );
2110 // For oop loads, we expect the _type to be precise.
2111
2112 // Optimize loads from constant fields.
2113 const TypeInstPtr* tinst = tp->is_instptr();
2114 ciObject* const_oop = tinst->const_oop();
2115 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2116 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2117 if (con_type != nullptr) {
2118 return con_type;
2119 }
2120 }
2121 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2122 assert(off != Type::OffsetBot ||
2123 !tp->isa_instklassptr() ||
2124 // arrays can be cast to Objects
2125 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2126 // also allow array-loading from the primary supertype
2127 // array during subtype checks
2128 Opcode() == Op_LoadKlass,
2129 "Field accesses must be precise");
2130 // For klass/static loads, we expect the _type to be precise
2131 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2132 /* With mirrors being an indirect in the Klass*
2133 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2134 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2135 *
2136 * So check the type and klass of the node before the LoadP.
2143 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2144 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2145 return TypeInstPtr::make(klass->java_mirror());
2146 }
2147 }
2148 }
2149
2150 const TypeKlassPtr *tkls = tp->isa_klassptr();
2151 if (tkls != nullptr) {
2152 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2153 ciKlass* klass = tkls->exact_klass();
2154 // We are loading a field from a Klass metaobject whose identity
2155 // is known at compile time (the type is "exact" or "precise").
2156 // Check for fields we know are maintained as constants by the VM.
2157 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2158 // The field is Klass::_super_check_offset. Return its (constant) value.
2159 // (Folds up type checking code.)
2160 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2161 return TypeInt::make(klass->super_check_offset());
2162 }
2163 if (UseCompactObjectHeaders) {
2164 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2165 // The field is Klass::_prototype_header. Return its (constant) value.
2166 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2167 return TypeX::make(klass->prototype_header());
2168 }
2169 }
2170 // Compute index into primary_supers array
2171 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2172 // Check for overflowing; use unsigned compare to handle the negative case.
2173 if( depth < ciKlass::primary_super_limit() ) {
2174 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2175 // (Folds up type checking code.)
2176 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2177 ciKlass *ss = klass->super_of_depth(depth);
2178 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2179 }
2180 const Type* aift = load_array_final_field(tkls, klass);
2181 if (aift != nullptr) return aift;
2182 }
2183
2184 // We can still check if we are loading from the primary_supers array at a
2185 // shallow enough depth. Even though the klass is not exact, entries less
2186 // than or equal to its super depth are correct.
2187 if (tkls->is_loaded()) {
2188 ciKlass* klass = nullptr;
2219 !tkls->is_instklassptr()->might_be_an_array() // not the supertype of all T[] (java.lang.Object) or has an interface that is not Serializable or Cloneable
2220 ) {
2221 assert(Opcode() == Op_LoadI, "must load an int from _layout_helper");
2222 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2223 // The key property of this type is that it folds up tests
2224 // for array-ness, since it proves that the layout_helper is positive.
2225 // Thus, a generic value like the basic object layout helper works fine.
2226 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2227 }
2228 }
2229
2230 bool is_vect = (_type->isa_vect() != nullptr);
2231 if (is_instance && !is_vect) {
2232 // If we have an instance type and our memory input is the
2233 // programs's initial memory state, there is no matching store,
2234 // so just return a zero of the appropriate type -
2235 // except if it is vectorized - then we have no zero constant.
2236 Node *mem = in(MemNode::Memory);
2237 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2238 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2239 return Type::get_zero_type(_type->basic_type());
2240 }
2241 }
2242
2243 if (!UseCompactObjectHeaders) {
2244 Node* alloc = is_new_object_mark_load();
2245 if (alloc != nullptr) {
2246 return TypeX::make(markWord::prototype().value());
2247 }
2248 }
2249
2250 return _type;
2251 }
2252
2253 //------------------------------match_edge-------------------------------------
2254 // Do we Match on this edge index or not? Match only the address.
2255 uint LoadNode::match_edge(uint idx) const {
2256 return idx == MemNode::Address;
2257 }
2258
2259 //--------------------------LoadBNode::Ideal--------------------------------------
2260 //
2261 // If the previous store is to the same address as this load,
2262 // and the value stored was larger than a byte, replace this load
2263 // with the value stored truncated to a byte. If no truncation is
2264 // needed, the replacement is done in LoadNode::Identity().
2265 //
2266 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2375 }
2376 }
2377 // Identity call will handle the case where truncation is not needed.
2378 return LoadNode::Ideal(phase, can_reshape);
2379 }
2380
2381 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2382 Node* mem = in(MemNode::Memory);
2383 Node* value = can_see_stored_value(mem,phase);
2384 if (value != nullptr && value->is_Con() &&
2385 !value->bottom_type()->higher_equal(_type)) {
2386 // If the input to the store does not fit with the load's result type,
2387 // it must be truncated. We can't delay until Ideal call since
2388 // a singleton Value is needed for split_thru_phi optimization.
2389 int con = value->get_int();
2390 return TypeInt::make((con << 16) >> 16);
2391 }
2392 return LoadNode::Value(phase);
2393 }
2394
2395 //=============================================================================
2396 //----------------------------LoadKlassNode::make------------------------------
2397 // Polymorphic factory method:
2398 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2399 // sanity check the alias category against the created node type
2400 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2401 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2402 #ifdef _LP64
2403 if (adr_type->is_ptr_to_narrowklass()) {
2404 assert(UseCompressedClassPointers, "no compressed klasses");
2405 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2406 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2407 }
2408 #endif
2409 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2410 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2411 }
2412
2413 //------------------------------Value------------------------------------------
2414 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2448 }
2449 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2450 }
2451 if (!t->is_klass()) {
2452 // a primitive Class (e.g., int.class) has null for a klass field
2453 return TypePtr::NULL_PTR;
2454 }
2455 // Fold up the load of the hidden field
2456 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2457 }
2458 // non-constant mirror, so we can't tell what's going on
2459 }
2460 if (!tinst->is_loaded())
2461 return _type; // Bail out if not loaded
2462 if (offset == oopDesc::klass_offset_in_bytes()) {
2463 return tinst->as_klass_type(true);
2464 }
2465 }
2466
2467 // Check for loading klass from an array
2468 const TypeAryPtr *tary = tp->isa_aryptr();
2469 if (tary != nullptr &&
2470 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2471 return tary->as_klass_type(true);
2472 }
2473
2474 // Check for loading klass from an array klass
2475 const TypeKlassPtr *tkls = tp->isa_klassptr();
2476 if (tkls != nullptr && !StressReflectiveCode) {
2477 if (!tkls->is_loaded())
2478 return _type; // Bail out if not loaded
2479 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2480 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2481 // // Always returning precise element type is incorrect,
2482 // // e.g., element type could be object and array may contain strings
2483 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2484
2485 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2486 // according to the element type's subclassing.
2487 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2488 }
2489 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2490 tkls->offset() == in_bytes(Klass::super_offset())) {
2491 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2492 // The field is Klass::_super. Return its (constant) value.
2493 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2494 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2495 }
2496 }
2497
2498 if (tkls != nullptr && !UseSecondarySupersCache
2499 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2500 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2501 return TypePtr::NULL_PTR;
2502 }
2503
2504 // Bailout case
2505 return LoadNode::Value(phase);
2506 }
2507
2508 //------------------------------Identity---------------------------------------
2531 base = bs->step_over_gc_barrier(base);
2532 }
2533
2534 // We can fetch the klass directly through an AllocateNode.
2535 // This works even if the klass is not constant (clone or newArray).
2536 if (offset == oopDesc::klass_offset_in_bytes()) {
2537 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2538 if (allocated_klass != nullptr) {
2539 return allocated_klass;
2540 }
2541 }
2542
2543 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2544 // See inline_native_Class_query for occurrences of these patterns.
2545 // Java Example: x.getClass().isAssignableFrom(y)
2546 //
2547 // This improves reflective code, often making the Class
2548 // mirror go completely dead. (Current exception: Class
2549 // mirrors may appear in debug info, but we could clean them out by
2550 // introducing a new debug info operator for Klass.java_mirror).
2551
2552 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2553 && offset == java_lang_Class::klass_offset()) {
2554 if (base->is_Load()) {
2555 Node* base2 = base->in(MemNode::Address);
2556 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2557 Node* adr2 = base2->in(MemNode::Address);
2558 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2559 if (tkls != nullptr && !tkls->empty()
2560 && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2561 && adr2->is_AddP()
2562 ) {
2563 int mirror_field = in_bytes(Klass::java_mirror_offset());
2564 if (tkls->offset() == mirror_field) {
2565 return adr2->in(AddPNode::Base);
2566 }
2567 }
2568 }
2569 }
2570 }
2571
2572 return this;
2573 }
2574
2575 LoadNode* LoadNode::clone_pinned() const {
2576 LoadNode* ld = clone()->as_Load();
2577 ld->_control_dependency = UnknownControl;
2578 return ld;
2579 }
2580
2581
2582 //------------------------------Value------------------------------------------
2687 //---------------------------StoreNode::make-----------------------------------
2688 // Polymorphic factory method:
2689 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2690 assert((mo == unordered || mo == release), "unexpected");
2691 Compile* C = gvn.C;
2692 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2693 ctl != nullptr, "raw memory operations should have control edge");
2694
2695 switch (bt) {
2696 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2697 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2698 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2699 case T_CHAR:
2700 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2701 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2702 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2703 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2704 case T_METADATA:
2705 case T_ADDRESS:
2706 case T_OBJECT:
2707 #ifdef _LP64
2708 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2709 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2710 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2711 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2712 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2713 adr->bottom_type()->isa_rawptr())) {
2714 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2715 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2716 }
2717 #endif
2718 {
2719 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2720 }
2721 default:
2722 ShouldNotReachHere();
2723 return (StoreNode*)nullptr;
2724 }
2725 }
2726
2727 //--------------------------bottom_type----------------------------------------
2728 const Type *StoreNode::bottom_type() const {
2729 return Type::MEMORY;
2730 }
2731
2732 //------------------------------hash-------------------------------------------
2733 uint StoreNode::hash() const {
2734 // unroll addition of interesting fields
2735 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2736
2737 // Since they are not commoned, do not hash them:
2738 return NO_HASH;
2739 }
2740
2741 // Link together multiple stores (B/S/C/I) into a longer one.
2742 //
3364 }
3365 ss.print_cr("[TraceMergeStores]: with");
3366 merged_input_value->dump("\n", false, &ss);
3367 merged_store->dump("\n", false, &ss);
3368 tty->print("%s", ss.as_string());
3369 }
3370 #endif
3371
3372 //------------------------------Ideal------------------------------------------
3373 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3374 // When a store immediately follows a relevant allocation/initialization,
3375 // try to capture it into the initialization, or hoist it above.
3376 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3377 Node* p = MemNode::Ideal_common(phase, can_reshape);
3378 if (p) return (p == NodeSentinel) ? nullptr : p;
3379
3380 Node* mem = in(MemNode::Memory);
3381 Node* address = in(MemNode::Address);
3382 Node* value = in(MemNode::ValueIn);
3383 // Back-to-back stores to same address? Fold em up. Generally
3384 // unsafe if I have intervening uses.
3385 {
3386 Node* st = mem;
3387 // If Store 'st' has more than one use, we cannot fold 'st' away.
3388 // For example, 'st' might be the final state at a conditional
3389 // return. Or, 'st' might be used by some node which is live at
3390 // the same time 'st' is live, which might be unschedulable. So,
3391 // require exactly ONE user until such time as we clone 'mem' for
3392 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3393 // true).
3394 while (st->is_Store() && st->outcnt() == 1) {
3395 // Looking at a dead closed cycle of memory?
3396 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3397 assert(Opcode() == st->Opcode() ||
3398 st->Opcode() == Op_StoreVector ||
3399 Opcode() == Op_StoreVector ||
3400 st->Opcode() == Op_StoreVectorScatter ||
3401 Opcode() == Op_StoreVectorScatter ||
3402 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3403 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3404 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3405 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3406 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3407
3408 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3409 st->as_Store()->memory_size() <= this->memory_size()) {
3410 Node* use = st->raw_out(0);
3411 if (phase->is_IterGVN()) {
3412 phase->is_IterGVN()->rehash_node_delayed(use);
3413 }
3414 // It's OK to do this in the parser, since DU info is always accurate,
3415 // and the parser always refers to nodes via SafePointNode maps.
3416 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3417 return this;
3418 }
3419 st = st->in(MemNode::Memory);
3420 }
3421 }
3422
3423
3424 // Capture an unaliased, unconditional, simple store into an initializer.
3522 const StoreVectorNode* store_vector = as_StoreVector();
3523 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3524 const Node* store_indices = store_vector->indices();
3525 const Node* mem_indices = mem_vector->indices();
3526 const Node* store_mask = store_vector->mask();
3527 const Node* mem_mask = mem_vector->mask();
3528 // Ensure types, indices, and masks match
3529 if (store_vector->vect_type() == mem_vector->vect_type() &&
3530 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3531 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3532 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3533 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3534 result = mem;
3535 }
3536 }
3537 }
3538
3539 // Store of zero anywhere into a freshly-allocated object?
3540 // Then the store is useless.
3541 // (It must already have been captured by the InitializeNode.)
3542 if (result == this &&
3543 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3544 // a newly allocated object is already all-zeroes everywhere
3545 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3546 result = mem;
3547 }
3548
3549 if (result == this) {
3550 // the store may also apply to zero-bits in an earlier object
3551 Node* prev_mem = find_previous_store(phase);
3552 // Steps (a), (b): Walk past independent stores to find an exact match.
3553 if (prev_mem != nullptr) {
3554 Node* prev_val = can_see_stored_value(prev_mem, phase);
3555 if (prev_val != nullptr && prev_val == val) {
3556 // prev_val and val might differ by a cast; it would be good
3557 // to keep the more informative of the two.
3558 result = mem;
3559 }
3560 }
3561 }
3562 }
3563
3564 PhaseIterGVN* igvn = phase->is_IterGVN();
3565 if (result != this && igvn != nullptr) {
3566 MemBarNode* trailing = trailing_membar();
3567 if (trailing != nullptr) {
3568 #ifdef ASSERT
3569 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
4033 // Clearing a short array is faster with stores
4034 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4035 // Already know this is a large node, do not try to ideal it
4036 if (_is_large) return nullptr;
4037
4038 const int unit = BytesPerLong;
4039 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4040 if (!t) return nullptr;
4041 if (!t->is_con()) return nullptr;
4042 intptr_t raw_count = t->get_con();
4043 intptr_t size = raw_count;
4044 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4045 // Clearing nothing uses the Identity call.
4046 // Negative clears are possible on dead ClearArrays
4047 // (see jck test stmt114.stmt11402.val).
4048 if (size <= 0 || size % unit != 0) return nullptr;
4049 intptr_t count = size / unit;
4050 // Length too long; communicate this to matchers and assemblers.
4051 // Assemblers are responsible to produce fast hardware clears for it.
4052 if (size > InitArrayShortSize) {
4053 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4054 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4055 return nullptr;
4056 }
4057 if (!IdealizeClearArrayNode) return nullptr;
4058 Node *mem = in(1);
4059 if( phase->type(mem)==Type::TOP ) return nullptr;
4060 Node *adr = in(3);
4061 const Type* at = phase->type(adr);
4062 if( at==Type::TOP ) return nullptr;
4063 const TypePtr* atp = at->isa_ptr();
4064 // adjust atp to be the correct array element address type
4065 if (atp == nullptr) atp = TypePtr::BOTTOM;
4066 else atp = atp->add_offset(Type::OffsetBot);
4067 // Get base for derived pointer purposes
4068 if( adr->Opcode() != Op_AddP ) Unimplemented();
4069 Node *base = adr->in(1);
4070
4071 Node *zero = phase->makecon(TypeLong::ZERO);
4072 Node *off = phase->MakeConX(BytesPerLong);
4073 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4074 count--;
4075 while( count-- ) {
4076 mem = phase->transform(mem);
4077 adr = phase->transform(new AddPNode(base,adr,off));
4078 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4079 }
4080 return mem;
4081 }
4082
4083 //----------------------------step_through----------------------------------
4084 // Return allocation input memory edge if it is different instance
4085 // or itself if it is the one we are looking for.
4086 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4087 Node* n = *np;
4088 assert(n->is_ClearArray(), "sanity");
4089 intptr_t offset;
4090 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4091 // This method is called only before Allocate nodes are expanded
4092 // during macro nodes expansion. Before that ClearArray nodes are
4093 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4094 // Allocate nodes are expanded) which follows allocations.
4095 assert(alloc != nullptr, "should have allocation");
4096 if (alloc->_idx == instance_id) {
4097 // Can not bypass initialization of the instance we are looking for.
4098 return false;
4099 }
4100 // Otherwise skip it.
4101 InitializeNode* init = alloc->initialization();
4102 if (init != nullptr)
4103 *np = init->in(TypeFunc::Memory);
4104 else
4105 *np = alloc->in(TypeFunc::Memory);
4106 return true;
4107 }
4108
4109 //----------------------------clear_memory-------------------------------------
4110 // Generate code to initialize object storage to zero.
4111 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4112 intptr_t start_offset,
4113 Node* end_offset,
4114 PhaseGVN* phase) {
4115 intptr_t offset = start_offset;
4116
4117 int unit = BytesPerLong;
4118 if ((offset % unit) != 0) {
4119 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
4120 adr = phase->transform(adr);
4121 const TypePtr* atp = TypeRawPtr::BOTTOM;
4122 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4123 mem = phase->transform(mem);
4124 offset += BytesPerInt;
4125 }
4126 assert((offset % unit) == 0, "");
4127
4128 // Initialize the remaining stuff, if any, with a ClearArray.
4129 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
4130 }
4131
4132 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4133 Node* start_offset,
4134 Node* end_offset,
4135 PhaseGVN* phase) {
4136 if (start_offset == end_offset) {
4137 // nothing to do
4138 return mem;
4139 }
4140
4141 int unit = BytesPerLong;
4142 Node* zbase = start_offset;
4143 Node* zend = end_offset;
4144
4145 // Scale to the unit required by the CPU:
4146 if (!Matcher::init_array_count_is_in_bytes) {
4147 Node* shift = phase->intcon(exact_log2(unit));
4148 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4149 zend = phase->transform(new URShiftXNode(zend, shift) );
4150 }
4151
4152 // Bulk clear double-words
4153 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4154 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
4155 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4156 return phase->transform(mem);
4157 }
4158
4159 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4160 intptr_t start_offset,
4161 intptr_t end_offset,
4162 PhaseGVN* phase) {
4163 if (start_offset == end_offset) {
4164 // nothing to do
4165 return mem;
4166 }
4167
4168 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4169 intptr_t done_offset = end_offset;
4170 if ((done_offset % BytesPerLong) != 0) {
4171 done_offset -= BytesPerInt;
4172 }
4173 if (done_offset > start_offset) {
4174 mem = clear_memory(ctl, mem, dest,
4175 start_offset, phase->MakeConX(done_offset), phase);
4176 }
4177 if (done_offset < end_offset) { // emit the final 32-bit store
4178 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
4179 adr = phase->transform(adr);
4180 const TypePtr* atp = TypeRawPtr::BOTTOM;
4181 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4182 mem = phase->transform(mem);
4183 done_offset += BytesPerInt;
4184 }
4185 assert(done_offset == end_offset, "");
4186 return mem;
4187 }
4188
4189 //=============================================================================
4190 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4191 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4192 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4193 #ifdef ASSERT
4194 , _pair_idx(0)
4195 #endif
4196 {
4197 init_class_id(Class_MemBar);
4198 Node* top = C->top();
4199 init_req(TypeFunc::I_O,top);
4200 init_req(TypeFunc::FramePtr,top);
4201 init_req(TypeFunc::ReturnAdr,top);
4304 PhaseIterGVN* igvn = phase->is_IterGVN();
4305 remove(igvn);
4306 // Must return either the original node (now dead) or a new node
4307 // (Do not return a top here, since that would break the uniqueness of top.)
4308 return new ConINode(TypeInt::ZERO);
4309 }
4310 }
4311 return progress ? this : nullptr;
4312 }
4313
4314 //------------------------------Value------------------------------------------
4315 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4316 if( !in(0) ) return Type::TOP;
4317 if( phase->type(in(0)) == Type::TOP )
4318 return Type::TOP;
4319 return TypeTuple::MEMBAR;
4320 }
4321
4322 //------------------------------match------------------------------------------
4323 // Construct projections for memory.
4324 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4325 switch (proj->_con) {
4326 case TypeFunc::Control:
4327 case TypeFunc::Memory:
4328 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4329 }
4330 ShouldNotReachHere();
4331 return nullptr;
4332 }
4333
4334 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4335 trailing->_kind = TrailingStore;
4336 leading->_kind = LeadingStore;
4337 #ifdef ASSERT
4338 trailing->_pair_idx = leading->_idx;
4339 leading->_pair_idx = leading->_idx;
4340 #endif
4341 }
4342
4343 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4344 trailing->_kind = TrailingLoadStore;
4591 return (req() > RawStores);
4592 }
4593
4594 void InitializeNode::set_complete(PhaseGVN* phase) {
4595 assert(!is_complete(), "caller responsibility");
4596 _is_complete = Complete;
4597
4598 // After this node is complete, it contains a bunch of
4599 // raw-memory initializations. There is no need for
4600 // it to have anything to do with non-raw memory effects.
4601 // Therefore, tell all non-raw users to re-optimize themselves,
4602 // after skipping the memory effects of this initialization.
4603 PhaseIterGVN* igvn = phase->is_IterGVN();
4604 if (igvn) igvn->add_users_to_worklist(this);
4605 }
4606
4607 // convenience function
4608 // return false if the init contains any stores already
4609 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4610 InitializeNode* init = initialization();
4611 if (init == nullptr || init->is_complete()) return false;
4612 init->remove_extra_zeroes();
4613 // for now, if this allocation has already collected any inits, bail:
4614 if (init->is_non_zero()) return false;
4615 init->set_complete(phase);
4616 return true;
4617 }
4618
4619 void InitializeNode::remove_extra_zeroes() {
4620 if (req() == RawStores) return;
4621 Node* zmem = zero_memory();
4622 uint fill = RawStores;
4623 for (uint i = fill; i < req(); i++) {
4624 Node* n = in(i);
4625 if (n->is_top() || n == zmem) continue; // skip
4626 if (fill < i) set_req(fill, n); // compact
4627 ++fill;
4628 }
4629 // delete any empty spaces created:
4630 while (fill < req()) {
4631 del_req(fill);
4775 // store node that we'd like to capture. We need to check
4776 // the uses of the MergeMemNode.
4777 mems.push(n);
4778 }
4779 } else if (n->is_Mem()) {
4780 Node* other_adr = n->in(MemNode::Address);
4781 if (other_adr == adr) {
4782 failed = true;
4783 break;
4784 } else {
4785 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4786 if (other_t_adr != nullptr) {
4787 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4788 if (other_alias_idx == alias_idx) {
4789 // A load from the same memory slice as the store right
4790 // after the InitializeNode. We check the control of the
4791 // object/array that is loaded from. If it's the same as
4792 // the store control then we cannot capture the store.
4793 assert(!n->is_Store(), "2 stores to same slice on same control?");
4794 Node* base = other_adr;
4795 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4796 base = base->in(AddPNode::Base);
4797 if (base != nullptr) {
4798 base = base->uncast();
4799 if (base->is_Proj() && base->in(0) == alloc) {
4800 failed = true;
4801 break;
4802 }
4803 }
4804 }
4805 }
4806 }
4807 } else {
4808 failed = true;
4809 break;
4810 }
4811 }
4812 }
4813 }
4814 if (failed) {
5361 // z's_done 12 16 16 16 12 16 12
5362 // z's_needed 12 16 16 16 16 16 16
5363 // zsize 0 0 0 0 4 0 4
5364 if (next_full_store < 0) {
5365 // Conservative tack: Zero to end of current word.
5366 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5367 } else {
5368 // Zero to beginning of next fully initialized word.
5369 // Or, don't zero at all, if we are already in that word.
5370 assert(next_full_store >= zeroes_needed, "must go forward");
5371 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5372 zeroes_needed = next_full_store;
5373 }
5374 }
5375
5376 if (zeroes_needed > zeroes_done) {
5377 intptr_t zsize = zeroes_needed - zeroes_done;
5378 // Do some incremental zeroing on rawmem, in parallel with inits.
5379 zeroes_done = align_down(zeroes_done, BytesPerInt);
5380 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5381 zeroes_done, zeroes_needed,
5382 phase);
5383 zeroes_done = zeroes_needed;
5384 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5385 do_zeroing = false; // leave the hole, next time
5386 }
5387 }
5388
5389 // Collect the store and move on:
5390 phase->replace_input_of(st, MemNode::Memory, inits);
5391 inits = st; // put it on the linearized chain
5392 set_req(i, zmem); // unhook from previous position
5393
5394 if (zeroes_done == st_off)
5395 zeroes_done = next_init_off;
5396
5397 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5398
5399 #ifdef ASSERT
5400 // Various order invariants. Weaker than stores_are_sane because
5420 remove_extra_zeroes(); // clear out all the zmems left over
5421 add_req(inits);
5422
5423 if (!(UseTLAB && ZeroTLAB)) {
5424 // If anything remains to be zeroed, zero it all now.
5425 zeroes_done = align_down(zeroes_done, BytesPerInt);
5426 // if it is the last unused 4 bytes of an instance, forget about it
5427 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5428 if (zeroes_done + BytesPerLong >= size_limit) {
5429 AllocateNode* alloc = allocation();
5430 assert(alloc != nullptr, "must be present");
5431 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5432 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5433 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5434 if (zeroes_done == k->layout_helper())
5435 zeroes_done = size_limit;
5436 }
5437 }
5438 if (zeroes_done < size_limit) {
5439 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5440 zeroes_done, size_in_bytes, phase);
5441 }
5442 }
5443
5444 set_complete(phase);
5445 return rawmem;
5446 }
5447
5448
5449 #ifdef ASSERT
5450 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5451 if (is_complete())
5452 return true; // stores could be anything at this point
5453 assert(allocation() != nullptr, "must be present");
5454 intptr_t last_off = allocation()->minimum_header_size();
5455 for (uint i = InitializeNode::RawStores; i < req(); i++) {
5456 Node* st = in(i);
5457 intptr_t st_off = get_store_offset(st, phase);
5458 if (st_off < 0) continue; // ignore dead garbage
5459 if (last_off > st_off) {
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "compiler/compileLog.hpp"
30 #include "gc/shared/barrierSet.hpp"
31 #include "gc/shared/c2/barrierSetC2.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "memory/allocation.inline.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "oops/flatArrayKlass.hpp"
36 #include "oops/objArrayKlass.hpp"
37 #include "opto/addnode.hpp"
38 #include "opto/arraycopynode.hpp"
39 #include "opto/cfgnode.hpp"
40 #include "opto/compile.hpp"
41 #include "opto/connode.hpp"
42 #include "opto/convertnode.hpp"
43 #include "opto/inlinetypenode.hpp"
44 #include "opto/loopnode.hpp"
45 #include "opto/machnode.hpp"
46 #include "opto/matcher.hpp"
47 #include "opto/memnode.hpp"
48 #include "opto/mempointer.hpp"
49 #include "opto/mulnode.hpp"
50 #include "opto/narrowptrnode.hpp"
51 #include "opto/phaseX.hpp"
52 #include "opto/regalloc.hpp"
53 #include "opto/regmask.hpp"
54 #include "opto/rootnode.hpp"
55 #include "opto/traceMergeStoresTag.hpp"
56 #include "opto/vectornode.hpp"
57 #include "utilities/align.hpp"
58 #include "utilities/copy.hpp"
59 #include "utilities/globalDefinitions.hpp"
60 #include "utilities/macros.hpp"
61 #include "utilities/powerOfTwo.hpp"
62 #include "utilities/vmError.hpp"
63
64 // Portions of code courtesy of Clifford Click
65
66 // Optimization - Graph Style
67
68 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
69
70 //=============================================================================
71 uint MemNode::size_of() const { return sizeof(*this); }
72
73 const TypePtr *MemNode::adr_type() const {
74 Node* adr = in(Address);
75 if (adr == nullptr) return nullptr; // node is dead
76 const TypePtr* cross_check = nullptr;
77 DEBUG_ONLY(cross_check = _adr_type);
78 return calculate_adr_type(adr->bottom_type(), cross_check);
79 }
128 st->print(", idx=Bot;");
129 else if (atp->index() == Compile::AliasIdxTop)
130 st->print(", idx=Top;");
131 else if (atp->index() == Compile::AliasIdxRaw)
132 st->print(", idx=Raw;");
133 else {
134 ciField* field = atp->field();
135 if (field) {
136 st->print(", name=");
137 field->print_name_on(st);
138 }
139 st->print(", idx=%d;", atp->index());
140 }
141 }
142 }
143
144 extern void print_alias_types();
145
146 #endif
147
148 // Find the memory output corresponding to the fall-through path of a call
149 static Node* find_call_fallthrough_mem_output(CallNode* call) {
150 ResourceMark rm;
151 CallProjections* projs = call->extract_projections(false, false);
152 Node* res = projs->fallthrough_memproj;
153 assert(res != nullptr, "must have a fallthrough mem output");
154 return res;
155 }
156
157 // Try to find a better memory input for a load from a strict final field
158 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
159 intptr_t offset = 0;
160 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
161 if (base == nullptr) {
162 return nullptr;
163 }
164
165 Node* base_uncasted = base->uncast();
166 if (base_uncasted->is_Proj()) {
167 MultiNode* multi = base_uncasted->in(0)->as_Multi();
168 if (multi->is_Allocate()) {
169 base_local = base_uncasted->as_Proj();
170 return nullptr;
171 } else if (multi->is_Call()) {
172 // The oop is returned from a call, the memory can be the fallthrough output of the call
173 return find_call_fallthrough_mem_output(multi->as_Call());
174 } else if (multi->is_Start()) {
175 // The oop is a parameter
176 if (phase->C->method()->is_object_constructor() && base_uncasted->as_Proj()->_con == TypeFunc::Parms) {
177 // The receiver of a constructor is similar to the result of an AllocateNode
178 base_local = base_uncasted->as_Proj();
179 return nullptr;
180 } else {
181 // Use the start memory otherwise
182 return multi->proj_out(TypeFunc::Memory);
183 }
184 }
185 }
186
187 return nullptr;
188 }
189
190 // Whether a call can modify a strict final field, given that the object is allocated inside the
191 // current compilation unit, or is the first parameter when the compilation root is a constructor.
192 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
193 // the target method is a subclass of the class declaring 'field'.
194 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
195 if (!call->is_CallJava()) {
196 return false;
197 }
198
199 ciMethod* target = call->as_CallJava()->method();
200 if (target == nullptr || !target->is_object_constructor()) {
201 return false;
202 }
203
204 // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
205 // then the field is set inside the constructor, else the field must be set before the
206 // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
207 // while a field Sub.y must be set before Super::<init> is invoked.
208 // We can try to be more heroic and decide if the receiver of the constructor invocation is the
209 // object from which we are loading from. This, however, may be problematic as deciding if 2
210 // nodes are definitely different may not be trivial, especially if the graph is not canonical.
211 // As a result, it is made more conservative for now.
212 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
213 return target->holder()->is_subclass_of(field->holder());
214 }
215
216 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
217 assert(t_oop != nullptr, "sanity");
218 bool is_known_instance = t_oop->is_known_instance_field();
219 bool is_strict_final_load = false;
220
221 // After macro expansion, an allocation may become a call, changing the memory input to the
222 // memory output of that call would be illegal. As a result, disallow this transformation after
223 // macro expansion.
224 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
225 is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
226 #ifdef ASSERT
227 if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
228 assert(is_strict_final_load, "sanity check for basic cases");
229 }
230 #endif // ASSERT
231 }
232
233 if (!is_known_instance && !is_strict_final_load) {
234 return mchain;
235 }
236
237 Node* result = mchain;
238 ProjNode* base_local = nullptr;
239
240 ciField* field = nullptr;
241 if (is_strict_final_load) {
242 field = phase->C->alias_type(t_oop)->field();
243 assert(field != nullptr, "must point to a field");
244
245 Node* adr = load->in(MemNode::Address);
246 assert(phase->type(adr) == t_oop, "inconsistent type");
247 Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
248 if (tmp != nullptr) {
249 result = tmp;
250 }
251 }
252
253 uint instance_id = t_oop->instance_id();
254 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
255 Node* prev = nullptr;
256 while (prev != result) {
257 prev = result;
258 if (result == start_mem) {
259 // start_mem is the earliest memory possible
260 break;
261 }
262
263 // skip over a call which does not affect this memory slice
264 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
265 Node* proj_in = result->in(0);
266 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
267 // This is the allocation that creates the object from which we are loading from
268 break;
269 } else if (proj_in->is_Call()) {
270 // ArrayCopyNodes processed here as well
271 CallNode* call = proj_in->as_Call();
272 if (!call->may_modify(t_oop, phase)) {
273 result = call->in(TypeFunc::Memory);
274 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
275 result = call->in(TypeFunc::Memory);
276 }
277 } else if (proj_in->is_Initialize()) {
278 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
279 // Stop if this is the initialization for the object instance which
280 // contains this memory slice, otherwise skip over it.
281 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
282 break;
283 }
284 if (is_known_instance) {
285 result = proj_in->in(TypeFunc::Memory);
286 } else if (is_strict_final_load) {
287 Node* klass = alloc->in(AllocateNode::KlassNode);
288 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
289 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
290 // Allocation of another type, must be another object
291 result = proj_in->in(TypeFunc::Memory);
292 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
293 // Allocation of another object
294 result = proj_in->in(TypeFunc::Memory);
295 }
296 }
297 } else if (proj_in->is_MemBar()) {
298 ArrayCopyNode* ac = nullptr;
299 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
300 break;
301 }
302 result = proj_in->in(TypeFunc::Memory);
303 } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
304 if (is_strict_final_load) {
305 // LoadFlat and StoreFlat cannot happen to strict final fields
306 result = proj_in->in(TypeFunc::Memory);
307 }
308 } else if (proj_in->is_top()) {
309 break; // dead code
310 } else {
311 assert(false, "unexpected projection of %s", proj_in->Name());
312 }
313 } else if (result->is_ClearArray()) {
314 if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
315 // Can not bypass initialization of the instance
316 // we are looking for.
317 break;
318 }
319 // Otherwise skip it (the call updated 'result' value).
320 } else if (result->is_MergeMem()) {
321 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
322 }
323 }
324 return result;
325 }
326
327 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
328 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
329 if (t_oop == nullptr)
330 return mchain; // don't try to optimize non-oop types
331 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
332 bool is_instance = t_oop->is_known_instance_field();
333 PhaseIterGVN *igvn = phase->is_IterGVN();
334 if (is_instance && igvn != nullptr && result->is_Phi()) {
335 PhiNode *mphi = result->as_Phi();
336 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
337 const TypePtr *t = mphi->adr_type();
338 bool do_split = false;
339 // In the following cases, Load memory input can be further optimized based on
340 // its precise address type
341 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
342 do_split = true;
343 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
344 const TypeOopPtr* mem_t =
345 t->is_oopptr()->cast_to_exactness(true)
346 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
347 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
348 if (t_oop->isa_aryptr()) {
349 mem_t = mem_t->is_aryptr()
350 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
351 ->cast_to_size(t_oop->is_aryptr()->size())
352 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
353 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
354 ->with_offset(t_oop->is_aryptr()->offset())
355 ->is_aryptr();
356 }
357 do_split = mem_t == t_oop;
358 }
359 if (do_split) {
360 // clone the Phi with our address type
361 result = mphi->split_out_instance(t_adr, igvn);
362 } else {
363 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
364 }
365 }
366 return result;
367 }
368
369 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
370 uint alias_idx = phase->C->get_alias_index(tp);
371 Node *mem = mmem;
372 #ifdef ASSERT
373 {
374 // Check that current type is consistent with the alias index used during graph construction
375 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
376 bool consistent = adr_check == nullptr || adr_check->empty() ||
377 phase->C->must_alias(adr_check, alias_idx );
378 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
379 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
380 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
381 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
382 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
383 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
384 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
385 // don't assert if it is dead code.
386 consistent = true;
387 }
388 if( !consistent ) {
389 st->print("alias_idx==%d, adr_check==", alias_idx);
390 if( adr_check == nullptr ) {
391 st->print("null");
392 } else {
393 adr_check->dump();
394 }
395 st->cr();
396 print_alias_types();
397 assert(consistent, "adr_check must match alias idx");
398 }
399 }
400 #endif
1072 "use LoadKlassNode instead");
1073 assert(!(adr_type->isa_aryptr() &&
1074 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1075 "use LoadRangeNode instead");
1076 // Check control edge of raw loads
1077 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1078 // oop will be recorded in oop map if load crosses safepoint
1079 rt->isa_oopptr() || is_immutable_value(adr),
1080 "raw memory operations should have control edge");
1081 LoadNode* load = nullptr;
1082 switch (bt) {
1083 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1084 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1085 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1086 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1087 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1088 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1089 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1090 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1091 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1092 case T_ARRAY:
1093 case T_OBJECT:
1094 case T_NARROWOOP:
1095 #ifdef _LP64
1096 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1097 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1098 } else
1099 #endif
1100 {
1101 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1102 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1103 }
1104 break;
1105 default:
1106 assert(false, "unexpected basic type %s", type2name(bt));
1107 break;
1108 }
1109 assert(load != nullptr, "LoadNode should have been created");
1110 if (unaligned) {
1111 load->set_unaligned_access();
1112 }
1113 if (mismatched) {
1114 load->set_mismatched_access();
1115 }
1116 if (unsafe) {
1117 load->set_unsafe_access();
1118 }
1119 load->set_barrier_data(barrier_data);
1120 if (load->Opcode() == Op_LoadN) {
1121 Node* ld = gvn.transform(load);
1122 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1123 }
1124
1125 return load;
1126 }
1127
1128 //------------------------------hash-------------------------------------------
1129 uint LoadNode::hash() const {
1130 // unroll addition of interesting fields
1131 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1132 }
1133
1134 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1135 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1136 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1137 bool is_stable_ary = FoldStableValues &&
1138 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1139 tp->isa_aryptr()->is_stable();
1140
1141 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1142 }
1143
1144 return false;
1145 }
1146
1147 LoadNode* LoadNode::pin_array_access_node() const {
1148 const TypePtr* adr_type = this->adr_type();
1149 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1150 return clone_pinned();
1151 }
1152 return nullptr;
1153 }
1154
1155 // Is the value loaded previously stored by an arraycopy? If so return
1156 // a load node that reads from the source array so we may be able to
1157 // optimize out the ArrayCopy node later.
1158 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1159 Node* ld_adr = in(MemNode::Address);
1160 intptr_t ld_off = 0;
1161 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1177 if (ac->as_ArrayCopy()->is_clonebasic()) {
1178 assert(ld_alloc != nullptr, "need an alloc");
1179 assert(addp->is_AddP(), "address must be addp");
1180 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1181 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1182 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1183 addp->set_req(AddPNode::Base, src);
1184 addp->set_req(AddPNode::Address, src);
1185 } else {
1186 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1187 ac->as_ArrayCopy()->is_copyof_validated() ||
1188 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1189 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1190 addp->set_req(AddPNode::Base, src);
1191 addp->set_req(AddPNode::Address, src);
1192
1193 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1194 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1195 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1196
1197 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1198
1199 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1200 #ifdef _LP64
1201 diff = phase->transform(new ConvI2LNode(diff));
1202 #endif
1203 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1204
1205 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1206 addp->set_req(AddPNode::Offset, offset);
1207 }
1208 addp = phase->transform(addp);
1209 #ifdef ASSERT
1210 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1211 ld->_adr_type = adr_type;
1212 #endif
1213 ld->set_req(MemNode::Address, addp);
1214 ld->set_req(0, ctl);
1215 ld->set_req(MemNode::Memory, mem);
1216 return ld;
1217 }
1218 return nullptr;
1219 }
1220
1221 static Node* see_through_inline_type(PhaseValues* phase, const MemNode* load, Node* base, int offset) {
1222 if (!load->is_mismatched_access() && base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1223 InlineTypeNode* vt = base->as_InlineType();
1224 Node* value = vt->field_value_by_offset(offset, true);
1225 assert(value != nullptr, "must see some value");
1226 return value;
1227 }
1228
1229 return nullptr;
1230 }
1231
1232 //---------------------------can_see_stored_value------------------------------
1233 // This routine exists to make sure this set of tests is done the same
1234 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1235 // will change the graph shape in a way which makes memory alive twice at the
1236 // same time (uses the Oracle model of aliasing), then some
1237 // LoadXNode::Identity will fold things back to the equivalence-class model
1238 // of aliasing.
1239 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1240 Node* ld_adr = in(MemNode::Address);
1241 intptr_t ld_off = 0;
1242 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1243 // Try to see through an InlineTypeNode
1244 // LoadN is special because the input is not compressed
1245 if (Opcode() != Op_LoadN) {
1246 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1247 if (value != nullptr) {
1248 return value;
1249 }
1250 }
1251
1252 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1253 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1254 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1255 // This is more general than load from boxing objects.
1256 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1257 uint alias_idx = atp->index();
1258 Node* result = nullptr;
1259 Node* current = st;
1260 // Skip through chains of MemBarNodes checking the MergeMems for
1261 // new states for the slice of this load. Stop once any other
1262 // kind of node is encountered. Loads from final memory can skip
1263 // through any kind of MemBar but normal loads shouldn't skip
1264 // through MemBarAcquire since the could allow them to move out of
1265 // a synchronized region. It is not safe to step over MemBarCPUOrder,
1266 // because alias info above them may be inaccurate (e.g., due to
1267 // mixed/mismatched unsafe accesses).
1268 bool is_final_mem = !atp->is_rewritable();
1269 while (current->is_Proj()) {
1270 int opc = current->in(0)->Opcode();
1271 if ((is_final_mem && (opc == Op_MemBarAcquire ||
1315 // Same base, same offset.
1316 // Possible improvement for arrays: check index value instead of absolute offset.
1317
1318 // At this point we have proven something like this setup:
1319 // B = << base >>
1320 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1321 // S = StoreQ(AddP( B , #Off), V)
1322 // (Actually, we haven't yet proven the Q's are the same.)
1323 // In other words, we are loading from a casted version of
1324 // the same pointer-and-offset that we stored to.
1325 // Casted version may carry a dependency and it is respected.
1326 // Thus, we are able to replace L by V.
1327 }
1328 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1329 if (store_Opcode() != st->Opcode()) {
1330 return nullptr;
1331 }
1332 // LoadVector/StoreVector needs additional check to ensure the types match.
1333 if (st->is_StoreVector()) {
1334 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1335 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1336 if (in_vt != out_vt) {
1337 return nullptr;
1338 }
1339 }
1340 return st->in(MemNode::ValueIn);
1341 }
1342
1343 // A load from a freshly-created object always returns zero.
1344 // (This can happen after LoadNode::Ideal resets the load's memory input
1345 // to find_captured_store, which returned InitializeNode::zero_memory.)
1346 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1347 (st->in(0) == ld_alloc) &&
1348 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1349 // return a zero value for the load's basic type
1350 // (This is one of the few places where a generic PhaseTransform
1351 // can create new nodes. Think of it as lazily manifesting
1352 // virtually pre-existing constants.)
1353 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1354 if (init_value != nullptr) {
1355 // TODO 8350865 Scalar replacement does not work well for flat arrays.
1356 // Is this correct for non-all-zero init values? Don't we need field_value_by_offset?
1357 return init_value;
1358 }
1359 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1360 if (value_basic_type() != T_VOID) {
1361 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1362 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1363 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1364 // by the ArrayCopyNode.
1365 return phase->zerocon(value_basic_type());
1366 }
1367 } else {
1368 // TODO: materialize all-zero vector constant
1369 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1370 }
1371 }
1372
1373 // A load from an initialization barrier can match a captured store.
1374 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1375 InitializeNode* init = st->in(0)->as_Initialize();
1376 AllocateNode* alloc = init->allocation();
1377 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1378 // examine a captured store value
1379 st = init->find_captured_store(ld_off, memory_size(), phase);
1392 base = bs->step_over_gc_barrier(base);
1393 if (base != nullptr && base->is_Proj() &&
1394 base->as_Proj()->_con == TypeFunc::Parms &&
1395 base->in(0)->is_CallStaticJava() &&
1396 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1397 return base->in(0)->in(TypeFunc::Parms);
1398 }
1399 }
1400
1401 break;
1402 }
1403
1404 return nullptr;
1405 }
1406
1407 //----------------------is_instance_field_load_with_local_phi------------------
1408 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1409 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1410 in(Address)->is_AddP() ) {
1411 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1412 // Only known instances and immutable fields
1413 if( t_oop != nullptr &&
1414 (t_oop->is_ptr_to_strict_final_field() ||
1415 t_oop->is_known_instance_field()) &&
1416 t_oop->offset() != Type::OffsetBot &&
1417 t_oop->offset() != Type::OffsetTop) {
1418 return true;
1419 }
1420 }
1421 return false;
1422 }
1423
1424 //------------------------------Identity---------------------------------------
1425 // Loads are identity if previous store is to same address
1426 Node* LoadNode::Identity(PhaseGVN* phase) {
1427 // If the previous store-maker is the right kind of Store, and the store is
1428 // to the same address, then we are equal to the value stored.
1429 Node* mem = in(Memory);
1430 Node* value = can_see_stored_value(mem, phase);
1431 if( value ) {
1432 // byte, short & char stores truncate naturally.
1433 // A load has to load the truncated value which requires
1434 // some sort of masking operation and that requires an
1443 // usually runs first, producing the singleton type of the Con.)
1444 if (!has_pinned_control_dependency() || value->is_Con()) {
1445 return value;
1446 } else {
1447 return this;
1448 }
1449 }
1450
1451 if (has_pinned_control_dependency()) {
1452 return this;
1453 }
1454 // Search for an existing data phi which was generated before for the same
1455 // instance's field to avoid infinite generation of phis in a loop.
1456 Node *region = mem->in(0);
1457 if (is_instance_field_load_with_local_phi(region)) {
1458 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1459 int this_index = phase->C->get_alias_index(addr_t);
1460 int this_offset = addr_t->offset();
1461 int this_iid = addr_t->instance_id();
1462 if (!addr_t->is_known_instance() &&
1463 addr_t->is_ptr_to_strict_final_field()) {
1464 // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1465 intptr_t ignore = 0;
1466 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1467 if (base == nullptr) {
1468 return this;
1469 }
1470 this_iid = base->_idx;
1471 }
1472 const Type* this_type = bottom_type();
1473 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1474 Node* phi = region->fast_out(i);
1475 if (phi->is_Phi() && phi != mem &&
1476 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1477 return phi;
1478 }
1479 }
1480 }
1481
1482 return this;
1483 }
1484
2000 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2001 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2002
2003 // Skip up past a SafePoint control. Cannot do this for Stores because
2004 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2005 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2006 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2007 !addr_mark &&
2008 (depends_only_on_test() || has_unknown_control_dependency())) {
2009 ctrl = ctrl->in(0);
2010 set_req(MemNode::Control,ctrl);
2011 progress = true;
2012 }
2013
2014 intptr_t ignore = 0;
2015 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2016 if (base != nullptr
2017 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2018 // Check for useless control edge in some common special cases
2019 if (in(MemNode::Control) != nullptr
2020 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2021 && can_remove_control()
2022 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2023 && all_controls_dominate(base, phase->C->start())) {
2024 // A method-invariant, non-null address (constant or 'this' argument).
2025 set_req(MemNode::Control, nullptr);
2026 progress = true;
2027 }
2028 }
2029
2030 Node* mem = in(MemNode::Memory);
2031 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2032
2033 if (can_reshape && (addr_t != nullptr)) {
2034 // try to optimize our memory input
2035 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2036 if (opt_mem != mem) {
2037 set_req_X(MemNode::Memory, opt_mem, phase);
2038 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2039 return this;
2040 }
2097 // fold up, do so.
2098 Node* prev_mem = find_previous_store(phase);
2099 if (prev_mem != nullptr) {
2100 Node* value = can_see_arraycopy_value(prev_mem, phase);
2101 if (value != nullptr) {
2102 return value;
2103 }
2104 }
2105 // Steps (a), (b): Walk past independent stores to find an exact match.
2106 if (prev_mem != nullptr && prev_mem != in(MemNode::Memory)) {
2107 // (c) See if we can fold up on the spot, but don't fold up here.
2108 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
2109 // just return a prior value, which is done by Identity calls.
2110 if (can_see_stored_value(prev_mem, phase)) {
2111 // Make ready for step (d):
2112 set_req_X(MemNode::Memory, prev_mem, phase);
2113 return this;
2114 }
2115 }
2116
2117 if (progress) {
2118 return this;
2119 }
2120
2121 if (!can_reshape) {
2122 phase->record_for_igvn(this);
2123 }
2124 return nullptr;
2125 }
2126
2127 // Helper to recognize certain Klass fields which are invariant across
2128 // some group of array types (e.g., int[] or all T[] where T < Object).
2129 const Type*
2130 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
2131 ciKlass* klass) const {
2132 assert(!UseCompactObjectHeaders || tkls->offset() != in_bytes(Klass::prototype_header_offset()),
2133 "must not happen");
2134 if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
2135 // The field is Klass::_access_flags. Return its (constant) value.
2136 assert(Opcode() == Op_LoadUS, "must load an unsigned short from _access_flags");
2137 return TypeInt::make(klass->access_flags());
2138 }
2139 if (tkls->offset() == in_bytes(Klass::misc_flags_offset())) {
2140 // The field is Klass::_misc_flags. Return its (constant) value.
2141 assert(Opcode() == Op_LoadUB, "must load an unsigned byte from _misc_flags");
2142 return TypeInt::make(klass->misc_flags());
2143 }
2144 if (tkls->offset() == in_bytes(Klass::layout_helper_offset())) {
2204 }
2205 }
2206
2207 // Don't do this for integer types. There is only potential profit if
2208 // the element type t is lower than _type; that is, for int types, if _type is
2209 // more restrictive than t. This only happens here if one is short and the other
2210 // char (both 16 bits), and in those cases we've made an intentional decision
2211 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2212 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2213 //
2214 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2215 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2216 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2217 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2218 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2219 // In fact, that could have been the original type of p1, and p1 could have
2220 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2221 // expression (LShiftL quux 3) independently optimized to the constant 8.
2222 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2223 && (_type->isa_vect() == nullptr)
2224 && !ary->is_flat()
2225 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2226 // t might actually be lower than _type, if _type is a unique
2227 // concrete subclass of abstract class t.
2228 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2229 const Type* jt = t->join_speculative(_type);
2230 // In any case, do not allow the join, per se, to empty out the type.
2231 if (jt->empty() && !t->empty()) {
2232 // This can happen if a interface-typed array narrows to a class type.
2233 jt = _type;
2234 }
2235 #ifdef ASSERT
2236 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2237 // The pointers in the autobox arrays are always non-null
2238 Node* base = adr->in(AddPNode::Base);
2239 if ((base != nullptr) && base->is_DecodeN()) {
2240 // Get LoadN node which loads IntegerCache.cache field
2241 base = base->in(1);
2242 }
2243 if ((base != nullptr) && base->is_Con()) {
2244 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2245 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2246 // It could be narrow oop
2247 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2248 }
2249 }
2250 }
2251 #endif
2252 return jt;
2253 }
2254 }
2255 } else if (tp->base() == Type::InstPtr) {
2256 assert( off != Type::OffsetBot ||
2257 // arrays can be cast to Objects
2258 !tp->isa_instptr() ||
2259 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2260 // Default value load
2261 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2262 // unsafe field access may not have a constant offset
2263 C->has_unsafe_access(),
2264 "Field accesses must be precise" );
2265 // For oop loads, we expect the _type to be precise.
2266
2267 const TypeInstPtr* tinst = tp->is_instptr();
2268 BasicType bt = value_basic_type();
2269
2270 // Optimize loads from constant fields.
2271 ciObject* const_oop = tinst->const_oop();
2272 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2273 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2274 if (con_type != nullptr) {
2275 return con_type;
2276 }
2277 }
2278 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2279 assert(off != Type::OffsetBot ||
2280 !tp->isa_instklassptr() ||
2281 // arrays can be cast to Objects
2282 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2283 // also allow array-loading from the primary supertype
2284 // array during subtype checks
2285 Opcode() == Op_LoadKlass,
2286 "Field accesses must be precise");
2287 // For klass/static loads, we expect the _type to be precise
2288 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2289 /* With mirrors being an indirect in the Klass*
2290 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2291 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2292 *
2293 * So check the type and klass of the node before the LoadP.
2300 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2301 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2302 return TypeInstPtr::make(klass->java_mirror());
2303 }
2304 }
2305 }
2306
2307 const TypeKlassPtr *tkls = tp->isa_klassptr();
2308 if (tkls != nullptr) {
2309 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2310 ciKlass* klass = tkls->exact_klass();
2311 // We are loading a field from a Klass metaobject whose identity
2312 // is known at compile time (the type is "exact" or "precise").
2313 // Check for fields we know are maintained as constants by the VM.
2314 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2315 // The field is Klass::_super_check_offset. Return its (constant) value.
2316 // (Folds up type checking code.)
2317 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2318 return TypeInt::make(klass->super_check_offset());
2319 }
2320 if (tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset()) && klass->is_obj_array_klass()) {
2321 // Fold loads from LibraryCallKit::load_default_refined_array_klass
2322 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr();
2323 }
2324 if (klass->is_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::properties_offset())) {
2325 assert(klass->is_type_array_klass() || tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2326 return TypeInt::make(klass->as_array_klass()->properties());
2327 }
2328 if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2329 assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2330 return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2331 }
2332 if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2333 // The field is Klass::_prototype_header. Return its (constant) value.
2334 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2335 return TypeX::make(klass->prototype_header());
2336 }
2337 // Compute index into primary_supers array
2338 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2339 // Check for overflowing; use unsigned compare to handle the negative case.
2340 if( depth < ciKlass::primary_super_limit() ) {
2341 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2342 // (Folds up type checking code.)
2343 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2344 ciKlass *ss = klass->super_of_depth(depth);
2345 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2346 }
2347 const Type* aift = load_array_final_field(tkls, klass);
2348 if (aift != nullptr) return aift;
2349 }
2350
2351 // We can still check if we are loading from the primary_supers array at a
2352 // shallow enough depth. Even though the klass is not exact, entries less
2353 // than or equal to its super depth are correct.
2354 if (tkls->is_loaded()) {
2355 ciKlass* klass = nullptr;
2386 !tkls->is_instklassptr()->might_be_an_array() // not the supertype of all T[] (java.lang.Object) or has an interface that is not Serializable or Cloneable
2387 ) {
2388 assert(Opcode() == Op_LoadI, "must load an int from _layout_helper");
2389 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2390 // The key property of this type is that it folds up tests
2391 // for array-ness, since it proves that the layout_helper is positive.
2392 // Thus, a generic value like the basic object layout helper works fine.
2393 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2394 }
2395 }
2396
2397 bool is_vect = (_type->isa_vect() != nullptr);
2398 if (is_instance && !is_vect) {
2399 // If we have an instance type and our memory input is the
2400 // programs's initial memory state, there is no matching store,
2401 // so just return a zero of the appropriate type -
2402 // except if it is vectorized - then we have no zero constant.
2403 Node *mem = in(MemNode::Memory);
2404 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2405 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2406 // TODO 8350865 Scalar replacement does not work well for flat arrays.
2407 // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2408 // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2409 // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2410 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2411 intptr_t offset = 0;
2412 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2413 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2414 if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2415 return _type;
2416 }
2417 }
2418 return Type::get_zero_type(_type->basic_type());
2419 }
2420 }
2421 if (!UseCompactObjectHeaders) {
2422 Node* alloc = is_new_object_mark_load();
2423 if (alloc != nullptr) {
2424 if (EnableValhalla) {
2425 // The mark word may contain property bits (inline, flat, null-free)
2426 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2427 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2428 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2429 return TypeX::make(tkls->exact_klass()->prototype_header());
2430 }
2431 } else {
2432 return TypeX::make(markWord::prototype().value());
2433 }
2434 }
2435 }
2436
2437 return _type;
2438 }
2439
2440 //------------------------------match_edge-------------------------------------
2441 // Do we Match on this edge index or not? Match only the address.
2442 uint LoadNode::match_edge(uint idx) const {
2443 return idx == MemNode::Address;
2444 }
2445
2446 //--------------------------LoadBNode::Ideal--------------------------------------
2447 //
2448 // If the previous store is to the same address as this load,
2449 // and the value stored was larger than a byte, replace this load
2450 // with the value stored truncated to a byte. If no truncation is
2451 // needed, the replacement is done in LoadNode::Identity().
2452 //
2453 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2562 }
2563 }
2564 // Identity call will handle the case where truncation is not needed.
2565 return LoadNode::Ideal(phase, can_reshape);
2566 }
2567
2568 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2569 Node* mem = in(MemNode::Memory);
2570 Node* value = can_see_stored_value(mem,phase);
2571 if (value != nullptr && value->is_Con() &&
2572 !value->bottom_type()->higher_equal(_type)) {
2573 // If the input to the store does not fit with the load's result type,
2574 // it must be truncated. We can't delay until Ideal call since
2575 // a singleton Value is needed for split_thru_phi optimization.
2576 int con = value->get_int();
2577 return TypeInt::make((con << 16) >> 16);
2578 }
2579 return LoadNode::Value(phase);
2580 }
2581
2582 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2583 // Loading from an InlineType, find the input and make an EncodeP
2584 Node* addr = in(Address);
2585 intptr_t offset;
2586 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
2587 Node* value = see_through_inline_type(phase, this, base, offset);
2588 if (value != nullptr) {
2589 return new EncodePNode(value, type());
2590 }
2591
2592 return LoadNode::Ideal(phase, can_reshape);
2593 }
2594
2595 //=============================================================================
2596 //----------------------------LoadKlassNode::make------------------------------
2597 // Polymorphic factory method:
2598 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2599 // sanity check the alias category against the created node type
2600 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2601 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2602 #ifdef _LP64
2603 if (adr_type->is_ptr_to_narrowklass()) {
2604 assert(UseCompressedClassPointers, "no compressed klasses");
2605 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2606 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2607 }
2608 #endif
2609 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2610 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2611 }
2612
2613 //------------------------------Value------------------------------------------
2614 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2648 }
2649 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2650 }
2651 if (!t->is_klass()) {
2652 // a primitive Class (e.g., int.class) has null for a klass field
2653 return TypePtr::NULL_PTR;
2654 }
2655 // Fold up the load of the hidden field
2656 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2657 }
2658 // non-constant mirror, so we can't tell what's going on
2659 }
2660 if (!tinst->is_loaded())
2661 return _type; // Bail out if not loaded
2662 if (offset == oopDesc::klass_offset_in_bytes()) {
2663 return tinst->as_klass_type(true);
2664 }
2665 }
2666
2667 // Check for loading klass from an array
2668 const TypeAryPtr* tary = tp->isa_aryptr();
2669 if (tary != nullptr &&
2670 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2671 const TypeAryKlassPtr* res = tary->as_klass_type(true)->is_aryklassptr();
2672 // The klass of an array object must be a refined array klass
2673 return res->cast_to_refined_array_klass_ptr();
2674 }
2675
2676 // Check for loading klass from an array klass
2677 const TypeKlassPtr *tkls = tp->isa_klassptr();
2678 if (tkls != nullptr && !StressReflectiveCode) {
2679 if (!tkls->is_loaded())
2680 return _type; // Bail out if not loaded
2681 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2682 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2683 // // Always returning precise element type is incorrect,
2684 // // e.g., element type could be object and array may contain strings
2685 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2686
2687 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2688 // according to the element type's subclassing.
2689 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2690 }
2691 if (tkls->isa_aryklassptr() != nullptr && tkls->klass_is_exact() &&
2692 !tkls->exact_klass()->is_type_array_klass() &&
2693 tkls->offset() == in_bytes(Klass::super_offset())) {
2694 // We are loading the super klass of a refined array klass, return the non-refined klass pointer
2695 assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2696 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr(false);
2697 }
2698 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2699 tkls->offset() == in_bytes(Klass::super_offset())) {
2700 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2701 // The field is Klass::_super. Return its (constant) value.
2702 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2703 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2704 }
2705 }
2706
2707 if (tkls != nullptr && !UseSecondarySupersCache
2708 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2709 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2710 return TypePtr::NULL_PTR;
2711 }
2712
2713 // Bailout case
2714 return LoadNode::Value(phase);
2715 }
2716
2717 //------------------------------Identity---------------------------------------
2740 base = bs->step_over_gc_barrier(base);
2741 }
2742
2743 // We can fetch the klass directly through an AllocateNode.
2744 // This works even if the klass is not constant (clone or newArray).
2745 if (offset == oopDesc::klass_offset_in_bytes()) {
2746 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2747 if (allocated_klass != nullptr) {
2748 return allocated_klass;
2749 }
2750 }
2751
2752 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2753 // See inline_native_Class_query for occurrences of these patterns.
2754 // Java Example: x.getClass().isAssignableFrom(y)
2755 //
2756 // This improves reflective code, often making the Class
2757 // mirror go completely dead. (Current exception: Class
2758 // mirrors may appear in debug info, but we could clean them out by
2759 // introducing a new debug info operator for Klass.java_mirror).
2760 //
2761 // This optimization does not apply to arrays because if k is not a
2762 // constant, it was obtained via load_klass which returns the VM type
2763 // and '.java_mirror.as_klass' should return the Java type instead.
2764
2765 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2766 && offset == java_lang_Class::klass_offset()) {
2767 if (base->is_Load()) {
2768 Node* base2 = base->in(MemNode::Address);
2769 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2770 Node* adr2 = base2->in(MemNode::Address);
2771 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2772 if (tkls != nullptr && !tkls->empty()
2773 && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2774 && adr2->is_AddP()) {
2775 int mirror_field = in_bytes(Klass::java_mirror_offset());
2776 if (tkls->offset() == mirror_field) {
2777 return adr2->in(AddPNode::Base);
2778 }
2779 }
2780 }
2781 }
2782 }
2783
2784 return this;
2785 }
2786
2787 LoadNode* LoadNode::clone_pinned() const {
2788 LoadNode* ld = clone()->as_Load();
2789 ld->_control_dependency = UnknownControl;
2790 return ld;
2791 }
2792
2793
2794 //------------------------------Value------------------------------------------
2899 //---------------------------StoreNode::make-----------------------------------
2900 // Polymorphic factory method:
2901 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2902 assert((mo == unordered || mo == release), "unexpected");
2903 Compile* C = gvn.C;
2904 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2905 ctl != nullptr, "raw memory operations should have control edge");
2906
2907 switch (bt) {
2908 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2909 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2910 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2911 case T_CHAR:
2912 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2913 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2914 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2915 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2916 case T_METADATA:
2917 case T_ADDRESS:
2918 case T_OBJECT:
2919 case T_ARRAY:
2920 #ifdef _LP64
2921 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2922 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2923 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2924 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2925 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2926 adr->bottom_type()->isa_rawptr())) {
2927 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2928 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2929 }
2930 #endif
2931 {
2932 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2933 }
2934 default:
2935 assert(false, "unexpected basic type %s", type2name(bt));
2936 return (StoreNode*)nullptr;
2937 }
2938 }
2939
2940 //--------------------------bottom_type----------------------------------------
2941 const Type *StoreNode::bottom_type() const {
2942 return Type::MEMORY;
2943 }
2944
2945 //------------------------------hash-------------------------------------------
2946 uint StoreNode::hash() const {
2947 // unroll addition of interesting fields
2948 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2949
2950 // Since they are not commoned, do not hash them:
2951 return NO_HASH;
2952 }
2953
2954 // Link together multiple stores (B/S/C/I) into a longer one.
2955 //
3577 }
3578 ss.print_cr("[TraceMergeStores]: with");
3579 merged_input_value->dump("\n", false, &ss);
3580 merged_store->dump("\n", false, &ss);
3581 tty->print("%s", ss.as_string());
3582 }
3583 #endif
3584
3585 //------------------------------Ideal------------------------------------------
3586 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3587 // When a store immediately follows a relevant allocation/initialization,
3588 // try to capture it into the initialization, or hoist it above.
3589 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3590 Node* p = MemNode::Ideal_common(phase, can_reshape);
3591 if (p) return (p == NodeSentinel) ? nullptr : p;
3592
3593 Node* mem = in(MemNode::Memory);
3594 Node* address = in(MemNode::Address);
3595 Node* value = in(MemNode::ValueIn);
3596 // Back-to-back stores to same address? Fold em up. Generally
3597 // unsafe if I have intervening uses...
3598 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3599 Node* st = mem;
3600 // If Store 'st' has more than one use, we cannot fold 'st' away.
3601 // For example, 'st' might be the final state at a conditional
3602 // return. Or, 'st' might be used by some node which is live at
3603 // the same time 'st' is live, which might be unschedulable. So,
3604 // require exactly ONE user until such time as we clone 'mem' for
3605 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3606 // true).
3607 while (st->is_Store() && st->outcnt() == 1) {
3608 // Looking at a dead closed cycle of memory?
3609 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3610 assert(Opcode() == st->Opcode() ||
3611 st->Opcode() == Op_StoreVector ||
3612 Opcode() == Op_StoreVector ||
3613 st->Opcode() == Op_StoreVectorScatter ||
3614 Opcode() == Op_StoreVectorScatter ||
3615 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3616 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3617 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3618 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3619 (st->adr_type()->isa_aryptr() && st->adr_type()->is_aryptr()->is_flat()) || // TODO 8343835
3620 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3621 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3622
3623 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3624 st->as_Store()->memory_size() <= this->memory_size()) {
3625 Node* use = st->raw_out(0);
3626 if (phase->is_IterGVN()) {
3627 phase->is_IterGVN()->rehash_node_delayed(use);
3628 }
3629 // It's OK to do this in the parser, since DU info is always accurate,
3630 // and the parser always refers to nodes via SafePointNode maps.
3631 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3632 return this;
3633 }
3634 st = st->in(MemNode::Memory);
3635 }
3636 }
3637
3638
3639 // Capture an unaliased, unconditional, simple store into an initializer.
3737 const StoreVectorNode* store_vector = as_StoreVector();
3738 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3739 const Node* store_indices = store_vector->indices();
3740 const Node* mem_indices = mem_vector->indices();
3741 const Node* store_mask = store_vector->mask();
3742 const Node* mem_mask = mem_vector->mask();
3743 // Ensure types, indices, and masks match
3744 if (store_vector->vect_type() == mem_vector->vect_type() &&
3745 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3746 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3747 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3748 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3749 result = mem;
3750 }
3751 }
3752 }
3753
3754 // Store of zero anywhere into a freshly-allocated object?
3755 // Then the store is useless.
3756 // (It must already have been captured by the InitializeNode.)
3757 if (result == this && ReduceFieldZeroing) {
3758 // a newly allocated object is already all-zeroes everywhere
3759 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3760 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3761 result = mem;
3762 }
3763
3764 if (result == this && phase->type(val)->is_zero_type()) {
3765 // the store may also apply to zero-bits in an earlier object
3766 Node* prev_mem = find_previous_store(phase);
3767 // Steps (a), (b): Walk past independent stores to find an exact match.
3768 if (prev_mem != nullptr) {
3769 Node* prev_val = can_see_stored_value(prev_mem, phase);
3770 if (prev_val != nullptr && prev_val == val) {
3771 // prev_val and val might differ by a cast; it would be good
3772 // to keep the more informative of the two.
3773 result = mem;
3774 }
3775 }
3776 }
3777 }
3778
3779 PhaseIterGVN* igvn = phase->is_IterGVN();
3780 if (result != this && igvn != nullptr) {
3781 MemBarNode* trailing = trailing_membar();
3782 if (trailing != nullptr) {
3783 #ifdef ASSERT
3784 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
4248 // Clearing a short array is faster with stores
4249 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4250 // Already know this is a large node, do not try to ideal it
4251 if (_is_large) return nullptr;
4252
4253 const int unit = BytesPerLong;
4254 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4255 if (!t) return nullptr;
4256 if (!t->is_con()) return nullptr;
4257 intptr_t raw_count = t->get_con();
4258 intptr_t size = raw_count;
4259 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4260 // Clearing nothing uses the Identity call.
4261 // Negative clears are possible on dead ClearArrays
4262 // (see jck test stmt114.stmt11402.val).
4263 if (size <= 0 || size % unit != 0) return nullptr;
4264 intptr_t count = size / unit;
4265 // Length too long; communicate this to matchers and assemblers.
4266 // Assemblers are responsible to produce fast hardware clears for it.
4267 if (size > InitArrayShortSize) {
4268 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4269 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4270 return nullptr;
4271 }
4272 if (!IdealizeClearArrayNode) return nullptr;
4273 Node *mem = in(1);
4274 if( phase->type(mem)==Type::TOP ) return nullptr;
4275 Node *adr = in(3);
4276 const Type* at = phase->type(adr);
4277 if( at==Type::TOP ) return nullptr;
4278 const TypePtr* atp = at->isa_ptr();
4279 // adjust atp to be the correct array element address type
4280 if (atp == nullptr) atp = TypePtr::BOTTOM;
4281 else atp = atp->add_offset(Type::OffsetBot);
4282 // Get base for derived pointer purposes
4283 if( adr->Opcode() != Op_AddP ) Unimplemented();
4284 Node *base = adr->in(1);
4285
4286 Node *val = in(4);
4287 Node *off = phase->MakeConX(BytesPerLong);
4288 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4289 count--;
4290 while( count-- ) {
4291 mem = phase->transform(mem);
4292 adr = phase->transform(new AddPNode(base,adr,off));
4293 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4294 }
4295 return mem;
4296 }
4297
4298 //----------------------------step_through----------------------------------
4299 // Return allocation input memory edge if it is different instance
4300 // or itself if it is the one we are looking for.
4301 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4302 Node* n = *np;
4303 assert(n->is_ClearArray(), "sanity");
4304 intptr_t offset;
4305 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4306 // This method is called only before Allocate nodes are expanded
4307 // during macro nodes expansion. Before that ClearArray nodes are
4308 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4309 // Allocate nodes are expanded) which follows allocations.
4310 assert(alloc != nullptr, "should have allocation");
4311 if (alloc->_idx == instance_id) {
4312 // Can not bypass initialization of the instance we are looking for.
4313 return false;
4314 }
4315 // Otherwise skip it.
4316 InitializeNode* init = alloc->initialization();
4317 if (init != nullptr)
4318 *np = init->in(TypeFunc::Memory);
4319 else
4320 *np = alloc->in(TypeFunc::Memory);
4321 return true;
4322 }
4323
4324 //----------------------------clear_memory-------------------------------------
4325 // Generate code to initialize object storage to zero.
4326 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4327 Node* val,
4328 Node* raw_val,
4329 intptr_t start_offset,
4330 Node* end_offset,
4331 PhaseGVN* phase) {
4332 intptr_t offset = start_offset;
4333
4334 int unit = BytesPerLong;
4335 if ((offset % unit) != 0) {
4336 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
4337 adr = phase->transform(adr);
4338 const TypePtr* atp = TypeRawPtr::BOTTOM;
4339 if (val != nullptr) {
4340 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4341 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4342 } else {
4343 assert(raw_val == nullptr, "val may not be null");
4344 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4345 }
4346 mem = phase->transform(mem);
4347 offset += BytesPerInt;
4348 }
4349 assert((offset % unit) == 0, "");
4350
4351 // Initialize the remaining stuff, if any, with a ClearArray.
4352 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
4353 }
4354
4355 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4356 Node* raw_val,
4357 Node* start_offset,
4358 Node* end_offset,
4359 PhaseGVN* phase) {
4360 if (start_offset == end_offset) {
4361 // nothing to do
4362 return mem;
4363 }
4364
4365 int unit = BytesPerLong;
4366 Node* zbase = start_offset;
4367 Node* zend = end_offset;
4368
4369 // Scale to the unit required by the CPU:
4370 if (!Matcher::init_array_count_is_in_bytes) {
4371 Node* shift = phase->intcon(exact_log2(unit));
4372 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4373 zend = phase->transform(new URShiftXNode(zend, shift) );
4374 }
4375
4376 // Bulk clear double-words
4377 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4378 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
4379 if (raw_val == nullptr) {
4380 raw_val = phase->MakeConX(0);
4381 }
4382 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4383 return phase->transform(mem);
4384 }
4385
4386 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4387 Node* val,
4388 Node* raw_val,
4389 intptr_t start_offset,
4390 intptr_t end_offset,
4391 PhaseGVN* phase) {
4392 if (start_offset == end_offset) {
4393 // nothing to do
4394 return mem;
4395 }
4396
4397 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4398 intptr_t done_offset = end_offset;
4399 if ((done_offset % BytesPerLong) != 0) {
4400 done_offset -= BytesPerInt;
4401 }
4402 if (done_offset > start_offset) {
4403 mem = clear_memory(ctl, mem, dest, val, raw_val,
4404 start_offset, phase->MakeConX(done_offset), phase);
4405 }
4406 if (done_offset < end_offset) { // emit the final 32-bit store
4407 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
4408 adr = phase->transform(adr);
4409 const TypePtr* atp = TypeRawPtr::BOTTOM;
4410 if (val != nullptr) {
4411 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4412 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4413 } else {
4414 assert(raw_val == nullptr, "val may not be null");
4415 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4416 }
4417 mem = phase->transform(mem);
4418 done_offset += BytesPerInt;
4419 }
4420 assert(done_offset == end_offset, "");
4421 return mem;
4422 }
4423
4424 //=============================================================================
4425 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4426 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4427 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4428 #ifdef ASSERT
4429 , _pair_idx(0)
4430 #endif
4431 {
4432 init_class_id(Class_MemBar);
4433 Node* top = C->top();
4434 init_req(TypeFunc::I_O,top);
4435 init_req(TypeFunc::FramePtr,top);
4436 init_req(TypeFunc::ReturnAdr,top);
4539 PhaseIterGVN* igvn = phase->is_IterGVN();
4540 remove(igvn);
4541 // Must return either the original node (now dead) or a new node
4542 // (Do not return a top here, since that would break the uniqueness of top.)
4543 return new ConINode(TypeInt::ZERO);
4544 }
4545 }
4546 return progress ? this : nullptr;
4547 }
4548
4549 //------------------------------Value------------------------------------------
4550 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4551 if( !in(0) ) return Type::TOP;
4552 if( phase->type(in(0)) == Type::TOP )
4553 return Type::TOP;
4554 return TypeTuple::MEMBAR;
4555 }
4556
4557 //------------------------------match------------------------------------------
4558 // Construct projections for memory.
4559 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4560 switch (proj->_con) {
4561 case TypeFunc::Control:
4562 case TypeFunc::Memory:
4563 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4564 }
4565 ShouldNotReachHere();
4566 return nullptr;
4567 }
4568
4569 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4570 trailing->_kind = TrailingStore;
4571 leading->_kind = LeadingStore;
4572 #ifdef ASSERT
4573 trailing->_pair_idx = leading->_idx;
4574 leading->_pair_idx = leading->_idx;
4575 #endif
4576 }
4577
4578 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4579 trailing->_kind = TrailingLoadStore;
4826 return (req() > RawStores);
4827 }
4828
4829 void InitializeNode::set_complete(PhaseGVN* phase) {
4830 assert(!is_complete(), "caller responsibility");
4831 _is_complete = Complete;
4832
4833 // After this node is complete, it contains a bunch of
4834 // raw-memory initializations. There is no need for
4835 // it to have anything to do with non-raw memory effects.
4836 // Therefore, tell all non-raw users to re-optimize themselves,
4837 // after skipping the memory effects of this initialization.
4838 PhaseIterGVN* igvn = phase->is_IterGVN();
4839 if (igvn) igvn->add_users_to_worklist(this);
4840 }
4841
4842 // convenience function
4843 // return false if the init contains any stores already
4844 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4845 InitializeNode* init = initialization();
4846 if (init == nullptr || init->is_complete()) {
4847 return false;
4848 }
4849 init->remove_extra_zeroes();
4850 // for now, if this allocation has already collected any inits, bail:
4851 if (init->is_non_zero()) return false;
4852 init->set_complete(phase);
4853 return true;
4854 }
4855
4856 void InitializeNode::remove_extra_zeroes() {
4857 if (req() == RawStores) return;
4858 Node* zmem = zero_memory();
4859 uint fill = RawStores;
4860 for (uint i = fill; i < req(); i++) {
4861 Node* n = in(i);
4862 if (n->is_top() || n == zmem) continue; // skip
4863 if (fill < i) set_req(fill, n); // compact
4864 ++fill;
4865 }
4866 // delete any empty spaces created:
4867 while (fill < req()) {
4868 del_req(fill);
5012 // store node that we'd like to capture. We need to check
5013 // the uses of the MergeMemNode.
5014 mems.push(n);
5015 }
5016 } else if (n->is_Mem()) {
5017 Node* other_adr = n->in(MemNode::Address);
5018 if (other_adr == adr) {
5019 failed = true;
5020 break;
5021 } else {
5022 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5023 if (other_t_adr != nullptr) {
5024 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5025 if (other_alias_idx == alias_idx) {
5026 // A load from the same memory slice as the store right
5027 // after the InitializeNode. We check the control of the
5028 // object/array that is loaded from. If it's the same as
5029 // the store control then we cannot capture the store.
5030 assert(!n->is_Store(), "2 stores to same slice on same control?");
5031 Node* base = other_adr;
5032 if (base->is_Phi()) {
5033 // In rare case, base may be a PhiNode and it may read
5034 // the same memory slice between InitializeNode and store.
5035 failed = true;
5036 break;
5037 }
5038 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5039 base = base->in(AddPNode::Base);
5040 if (base != nullptr) {
5041 base = base->uncast();
5042 if (base->is_Proj() && base->in(0) == alloc) {
5043 failed = true;
5044 break;
5045 }
5046 }
5047 }
5048 }
5049 }
5050 } else {
5051 failed = true;
5052 break;
5053 }
5054 }
5055 }
5056 }
5057 if (failed) {
5604 // z's_done 12 16 16 16 12 16 12
5605 // z's_needed 12 16 16 16 16 16 16
5606 // zsize 0 0 0 0 4 0 4
5607 if (next_full_store < 0) {
5608 // Conservative tack: Zero to end of current word.
5609 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5610 } else {
5611 // Zero to beginning of next fully initialized word.
5612 // Or, don't zero at all, if we are already in that word.
5613 assert(next_full_store >= zeroes_needed, "must go forward");
5614 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5615 zeroes_needed = next_full_store;
5616 }
5617 }
5618
5619 if (zeroes_needed > zeroes_done) {
5620 intptr_t zsize = zeroes_needed - zeroes_done;
5621 // Do some incremental zeroing on rawmem, in parallel with inits.
5622 zeroes_done = align_down(zeroes_done, BytesPerInt);
5623 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5624 allocation()->in(AllocateNode::InitValue),
5625 allocation()->in(AllocateNode::RawInitValue),
5626 zeroes_done, zeroes_needed,
5627 phase);
5628 zeroes_done = zeroes_needed;
5629 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5630 do_zeroing = false; // leave the hole, next time
5631 }
5632 }
5633
5634 // Collect the store and move on:
5635 phase->replace_input_of(st, MemNode::Memory, inits);
5636 inits = st; // put it on the linearized chain
5637 set_req(i, zmem); // unhook from previous position
5638
5639 if (zeroes_done == st_off)
5640 zeroes_done = next_init_off;
5641
5642 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5643
5644 #ifdef ASSERT
5645 // Various order invariants. Weaker than stores_are_sane because
5665 remove_extra_zeroes(); // clear out all the zmems left over
5666 add_req(inits);
5667
5668 if (!(UseTLAB && ZeroTLAB)) {
5669 // If anything remains to be zeroed, zero it all now.
5670 zeroes_done = align_down(zeroes_done, BytesPerInt);
5671 // if it is the last unused 4 bytes of an instance, forget about it
5672 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5673 if (zeroes_done + BytesPerLong >= size_limit) {
5674 AllocateNode* alloc = allocation();
5675 assert(alloc != nullptr, "must be present");
5676 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5677 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5678 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5679 if (zeroes_done == k->layout_helper())
5680 zeroes_done = size_limit;
5681 }
5682 }
5683 if (zeroes_done < size_limit) {
5684 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5685 allocation()->in(AllocateNode::InitValue),
5686 allocation()->in(AllocateNode::RawInitValue),
5687 zeroes_done, size_in_bytes, phase);
5688 }
5689 }
5690
5691 set_complete(phase);
5692 return rawmem;
5693 }
5694
5695
5696 #ifdef ASSERT
5697 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5698 if (is_complete())
5699 return true; // stores could be anything at this point
5700 assert(allocation() != nullptr, "must be present");
5701 intptr_t last_off = allocation()->minimum_header_size();
5702 for (uint i = InitializeNode::RawStores; i < req(); i++) {
5703 Node* st = in(i);
5704 intptr_t st_off = get_store_offset(st, phase);
5705 if (st_off < 0) continue; // ignore dead garbage
5706 if (last_off > st_off) {
|