6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/compile.hpp"
38 #include "opto/connode.hpp"
39 #include "opto/convertnode.hpp"
40 #include "opto/loopnode.hpp"
41 #include "opto/machnode.hpp"
42 #include "opto/matcher.hpp"
43 #include "opto/memnode.hpp"
44 #include "opto/mempointer.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/opcodes.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regalloc.hpp"
50 #include "opto/regmask.hpp"
51 #include "opto/rootnode.hpp"
52 #include "opto/traceMergeStoresTag.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/globalDefinitions.hpp"
57 #include "utilities/macros.hpp"
58 #include "utilities/powerOfTwo.hpp"
59 #include "utilities/vmError.hpp"
60
61 // Portions of code courtesy of Clifford Click
62
63 // Optimization - Graph Style
64
65 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
66
67 //=============================================================================
68 uint MemNode::size_of() const { return sizeof(*this); }
69
70 const TypePtr *MemNode::adr_type() const {
71 Node* adr = in(Address);
72 if (adr == nullptr) return nullptr; // node is dead
73 const TypePtr* cross_check = nullptr;
127 st->print(", idx=Bot;");
128 else if (atp->index() == Compile::AliasIdxTop)
129 st->print(", idx=Top;");
130 else if (atp->index() == Compile::AliasIdxRaw)
131 st->print(", idx=Raw;");
132 else {
133 ciField* field = atp->field();
134 if (field) {
135 st->print(", name=");
136 field->print_name_on(st);
137 }
138 st->print(", idx=%d;", atp->index());
139 }
140 }
141 }
142
143 extern void print_alias_types();
144
145 #endif
146
147 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
148 assert((t_oop != nullptr), "sanity");
149 bool is_instance = t_oop->is_known_instance_field();
150 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
151 (load != nullptr) && load->is_Load() &&
152 (phase->is_IterGVN() != nullptr);
153 if (!(is_instance || is_boxed_value_load))
154 return mchain; // don't try to optimize non-instance types
155 uint instance_id = t_oop->instance_id();
156 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
157 Node *prev = nullptr;
158 Node *result = mchain;
159 while (prev != result) {
160 prev = result;
161 if (result == start_mem)
162 break; // hit one of our sentinels
163 // skip over a call which does not affect this memory slice
164 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
165 Node *proj_in = result->in(0);
166 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
167 break; // hit one of our sentinels
168 } else if (proj_in->is_Call()) {
169 // ArrayCopyNodes processed here as well
170 CallNode *call = proj_in->as_Call();
171 if (!call->may_modify(t_oop, phase)) { // returns false for instances
172 result = call->in(TypeFunc::Memory);
173 }
174 } else if (proj_in->is_Initialize()) {
175 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
176 // Stop if this is the initialization for the object instance which
177 // contains this memory slice, otherwise skip over it.
178 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
179 break;
180 }
181 if (is_instance) {
182 result = proj_in->in(TypeFunc::Memory);
183 } else if (is_boxed_value_load) {
184 Node* klass = alloc->in(AllocateNode::KlassNode);
185 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
186 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
187 result = proj_in->in(TypeFunc::Memory); // not related allocation
188 }
189 }
190 } else if (proj_in->is_MemBar()) {
191 ArrayCopyNode* ac = nullptr;
192 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
193 break;
194 }
195 result = proj_in->in(TypeFunc::Memory);
196 } else if (proj_in->is_top()) {
197 break; // dead code
198 } else {
199 assert(false, "unexpected projection");
200 }
201 } else if (result->is_ClearArray()) {
202 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
203 // Can not bypass initialization of the instance
204 // we are looking for.
205 break;
206 }
207 // Otherwise skip it (the call updated 'result' value).
208 } else if (result->is_MergeMem()) {
209 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
210 }
211 }
212 return result;
213 }
214
215 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
216 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
217 if (t_oop == nullptr)
218 return mchain; // don't try to optimize non-oop types
219 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
220 bool is_instance = t_oop->is_known_instance_field();
221 PhaseIterGVN *igvn = phase->is_IterGVN();
222 if (is_instance && igvn != nullptr && result->is_Phi()) {
223 PhiNode *mphi = result->as_Phi();
224 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
225 const TypePtr *t = mphi->adr_type();
226 bool do_split = false;
227 // In the following cases, Load memory input can be further optimized based on
228 // its precise address type
229 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
230 do_split = true;
231 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
232 const TypeOopPtr* mem_t =
233 t->is_oopptr()->cast_to_exactness(true)
234 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
235 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
236 if (t_oop->isa_aryptr()) {
237 mem_t = mem_t->is_aryptr()
238 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
239 ->cast_to_size(t_oop->is_aryptr()->size())
240 ->with_offset(t_oop->is_aryptr()->offset())
241 ->is_aryptr();
242 }
243 do_split = mem_t == t_oop;
244 }
245 if (do_split) {
246 // clone the Phi with our address type
247 result = mphi->split_out_instance(t_adr, igvn);
248 } else {
249 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
250 }
251 }
252 return result;
253 }
254
255 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
256 uint alias_idx = phase->C->get_alias_index(tp);
257 Node *mem = mmem;
258 #ifdef ASSERT
259 {
260 // Check that current type is consistent with the alias index used during graph construction
261 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
262 bool consistent = adr_check == nullptr || adr_check->empty() ||
263 phase->C->must_alias(adr_check, alias_idx );
264 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
265 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
266 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
267 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
268 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
269 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
270 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
271 // don't assert if it is dead code.
272 consistent = true;
273 }
274 if( !consistent ) {
275 st->print("alias_idx==%d, adr_check==", alias_idx);
276 if( adr_check == nullptr ) {
277 st->print("null");
278 } else {
279 adr_check->dump();
280 }
281 st->cr();
282 print_alias_types();
283 assert(consistent, "adr_check must match alias idx");
284 }
285 }
286 #endif
589 }
590
591 // Find an arraycopy ac that produces the memory state represented by parameter mem.
592 // Return ac if
593 // (a) can_see_stored_value=true and ac must have set the value for this load or if
594 // (b) can_see_stored_value=false and ac could have set the value for this load or if
595 // (c) can_see_stored_value=false and ac cannot have set the value for this load.
596 // In case (c) change the parameter mem to the memory input of ac to skip it
597 // when searching stored value.
598 // Otherwise return null.
599 Node* LoadNode::find_previous_arraycopy(PhaseValues* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const {
600 ArrayCopyNode* ac = find_array_copy_clone(ld_alloc, mem);
601 if (ac != nullptr) {
602 Node* ld_addp = in(MemNode::Address);
603 Node* src = ac->in(ArrayCopyNode::Src);
604 const TypeAryPtr* ary_t = phase->type(src)->isa_aryptr();
605
606 // This is a load from a cloned array. The corresponding arraycopy ac must
607 // have set the value for the load and we can return ac but only if the load
608 // is known to be within bounds. This is checked below.
609 if (ary_t != nullptr && ld_addp->is_AddP()) {
610 Node* ld_offs = ld_addp->in(AddPNode::Offset);
611 BasicType ary_elem = ary_t->elem()->array_element_basic_type();
612 jlong header = arrayOopDesc::base_offset_in_bytes(ary_elem);
613 jlong elemsize = type2aelembytes(ary_elem);
614
615 const TypeX* ld_offs_t = phase->type(ld_offs)->isa_intptr_t();
616 const TypeInt* sizetype = ary_t->size();
617
618 if (ld_offs_t->_lo >= header && ld_offs_t->_hi < (sizetype->_lo * elemsize + header)) {
619 // The load is known to be within bounds. It receives its value from ac.
620 return ac;
621 }
622 // The load is known to be out-of-bounds.
623 }
624 // The load could be out-of-bounds. It must not be hoisted but must remain
625 // dependent on the runtime range check. This is achieved by returning null.
626 } else if (mem->is_Proj() && mem->in(0) != nullptr && mem->in(0)->is_ArrayCopy()) {
627 ArrayCopyNode* ac = mem->in(0)->as_ArrayCopy();
628
629 if (ac->is_arraycopy_validated() ||
999 in_bytes(JavaThread::vthread_offset()),
1000 in_bytes(JavaThread::scopedValueCache_offset()),
1001 };
1002
1003 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1004 if (offset == offsets[i]) {
1005 return true;
1006 }
1007 }
1008 }
1009
1010 return false;
1011 }
1012 #endif
1013
1014 //----------------------------LoadNode::make-----------------------------------
1015 // Polymorphic factory method:
1016 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1017 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1018 Compile* C = gvn.C;
1019 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
1020
1021 // sanity check the alias category against the created node type
1022 assert(!(adr_type->isa_oopptr() &&
1023 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1024 "use LoadKlassNode instead");
1025 assert(!(adr_type->isa_aryptr() &&
1026 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1027 "use LoadRangeNode instead");
1028 // Check control edge of raw loads
1029 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1030 // oop will be recorded in oop map if load crosses safepoint
1031 rt->isa_oopptr() || is_immutable_value(adr),
1032 "raw memory operations should have control edge");
1033 LoadNode* load = nullptr;
1034 switch (bt) {
1035 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1036 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1037 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1038 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1039 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1040 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1041 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1042 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1043 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1044 case T_OBJECT:
1045 case T_NARROWOOP:
1046 #ifdef _LP64
1047 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1048 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1049 } else
1050 #endif
1051 {
1052 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1053 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1054 }
1055 break;
1056 default:
1057 ShouldNotReachHere();
1058 break;
1059 }
1060 assert(load != nullptr, "LoadNode should have been created");
1061 if (unaligned) {
1062 load->set_unaligned_access();
1063 }
1064 if (mismatched) {
1065 load->set_mismatched_access();
1066 }
1067 if (unsafe) {
1068 load->set_unsafe_access();
1069 }
1070 load->set_barrier_data(barrier_data);
1071 if (load->Opcode() == Op_LoadN) {
1072 Node* ld = gvn.transform(load);
1073 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1074 }
1075
1076 return load;
1077 }
1078
1079 //------------------------------hash-------------------------------------------
1080 uint LoadNode::hash() const {
1081 // unroll addition of interesting fields
1082 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1083 }
1084
1085 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1086 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1087 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1088 bool is_stable_ary = FoldStableValues &&
1089 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1090 tp->isa_aryptr()->is_stable();
1091
1092 return (eliminate_boxing && non_volatile) || is_stable_ary;
1093 }
1094
1095 return false;
1096 }
1097
1098 // Is the value loaded previously stored by an arraycopy? If so return
1099 // a load node that reads from the source array so we may be able to
1100 // optimize out the ArrayCopy node later.
1101 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1102 Node* ld_adr = in(MemNode::Address);
1103 intptr_t ld_off = 0;
1104 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1105 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1106 if (ac != nullptr) {
1107 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1108
1109 Node* mem = ac->in(TypeFunc::Memory);
1110 Node* ctl = ac->in(0);
1111 Node* src = ac->in(ArrayCopyNode::Src);
1112
1120 if (ac->as_ArrayCopy()->is_clonebasic()) {
1121 assert(ld_alloc != nullptr, "need an alloc");
1122 assert(addp->is_AddP(), "address must be addp");
1123 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1124 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1125 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1126 addp->set_req(AddPNode::Base, src);
1127 addp->set_req(AddPNode::Address, src);
1128 } else {
1129 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1130 ac->as_ArrayCopy()->is_copyof_validated() ||
1131 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1132 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1133 addp->set_req(AddPNode::Base, src);
1134 addp->set_req(AddPNode::Address, src);
1135
1136 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1137 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1138 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1139
1140 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1141 uint shift = exact_log2(type2aelembytes(ary_elem));
1142
1143 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1144 #ifdef _LP64
1145 diff = phase->transform(new ConvI2LNode(diff));
1146 #endif
1147 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1148
1149 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1150 addp->set_req(AddPNode::Offset, offset);
1151 }
1152 addp = phase->transform(addp);
1153 #ifdef ASSERT
1154 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1155 ld->_adr_type = adr_type;
1156 #endif
1157 ld->set_req(MemNode::Address, addp);
1158 ld->set_req(0, ctl);
1159 ld->set_req(MemNode::Memory, mem);
1160 return ld;
1161 }
1162 return nullptr;
1163 }
1164
1165 // This routine exists to make sure this set of tests is done the same
1166 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1167 // will change the graph shape in a way which makes memory alive twice at the
1168 // same time (uses the Oracle model of aliasing), then some
1169 // LoadXNode::Identity will fold things back to the equivalence-class model
1170 // of aliasing.
1171 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1172 Node* ld_adr = in(MemNode::Address);
1173 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1174 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1175
1176 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1177 uint alias_idx = atp->index();
1178 Node* result = nullptr;
1179 Node* current = st;
1180 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1181 // this load. Stop once any other kind of node is encountered.
1182 //
1183 // In principle, folding a load is moving it up until it meets a matching store.
1184 //
1185 // store(ptr, v); store(ptr, v); store(ptr, v);
1186 // membar1; -> membar1; -> load(ptr);
1187 // membar2; load(ptr); membar1;
1188 // load(ptr); membar2; membar2;
1189 //
1190 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1191 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1192 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1204 MergeMemNode* merge = mem->as_MergeMem();
1205 Node* new_st = merge->memory_at(alias_idx);
1206 if (new_st == merge->base_memory()) {
1207 // Keep searching
1208 current = new_st;
1209 continue;
1210 }
1211 // Save the new memory state for the slice and fall through
1212 // to exit.
1213 result = new_st;
1214 }
1215 }
1216 break;
1217 }
1218 if (result != nullptr) {
1219 st = result;
1220 }
1221 }
1222
1223 Node* res = can_see_stored_value(st, phase);
1224 assert(res == nullptr || is_java_primitive(value_basic_type()) || res->bottom_type()->higher_equal(type()), "the fold is unsafe");
1225 return res;
1226 }
1227
1228 // If st is a store to the same location as this, return the stored value
1229 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1230 Node* ld_adr = in(MemNode::Address);
1231 intptr_t ld_off = 0;
1232 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1233 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1234 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1235
1236 // Loop around twice in the case Load -> Initialize -> Store.
1237 // (See PhaseIterGVN::add_users_to_worklist, which knows about this case.)
1238 for (int trip = 0; trip <= 1; trip++) {
1239
1240 if (st->is_Store()) {
1241 Node* st_adr = st->in(MemNode::Address);
1242 if (st_adr != ld_adr) {
1243 // Try harder before giving up. Unify base pointers with casts (e.g., raw/non-raw pointers).
1244 intptr_t st_off = 0;
1292
1293 // There are some cases in which the Type of the load is narrower than the Type of the value
1294 // that is stored into that location. The most common case is array polymorphism, when the
1295 // type of an array element depends on the type of the array. In addition, there are some
1296 // corner cases, the first one is concurrent class loading, when CHA can result in a narrower
1297 // Type than what is declared only after the child class is loaded, and the second case is
1298 // unsafe accesses when we do not check for type safety. See JDK-8388184.
1299 return nullptr;
1300 }
1301
1302 // A load from a freshly-created object always returns zero.
1303 // (This can happen after LoadNode::Ideal resets the load's memory input
1304 // to find_captured_store, which returned InitializeNode::zero_memory.)
1305 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1306 (st->in(0) == ld_alloc) &&
1307 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1308 // return a zero value for the load's basic type
1309 // (This is one of the few places where a generic PhaseTransform
1310 // can create new nodes. Think of it as lazily manifesting
1311 // virtually pre-existing constants.)
1312 if (value_basic_type() != T_VOID) {
1313 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1314 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1315 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1316 // by the ArrayCopyNode.
1317 return phase->zerocon(value_basic_type());
1318 }
1319 } else {
1320 // TODO: materialize all-zero vector constant
1321 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1322 }
1323 }
1324
1325 // A load from an initialization barrier can match a captured store.
1326 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1327 InitializeNode* init = st->in(0)->as_Initialize();
1328 AllocateNode* alloc = init->allocation();
1329 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1330 // examine a captured store value
1331 st = init->find_captured_store(ld_off, memory_size(), phase);
1344 base = bs->step_over_gc_barrier(base);
1345 if (base != nullptr && base->is_Proj() &&
1346 base->as_Proj()->_con == TypeFunc::Parms &&
1347 base->in(0)->is_CallStaticJava() &&
1348 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1349 return base->in(0)->in(TypeFunc::Parms);
1350 }
1351 }
1352
1353 break;
1354 }
1355
1356 return nullptr;
1357 }
1358
1359 //----------------------is_instance_field_load_with_local_phi------------------
1360 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1361 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1362 in(Address)->is_AddP() ) {
1363 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1364 // Only instances and boxed values.
1365 if( t_oop != nullptr &&
1366 (t_oop->is_ptr_to_boxed_value() ||
1367 t_oop->is_known_instance_field()) &&
1368 t_oop->offset() != Type::OffsetBot &&
1369 t_oop->offset() != Type::OffsetTop) {
1370 return true;
1371 }
1372 }
1373 return false;
1374 }
1375
1376 //------------------------------Identity---------------------------------------
1377 // Loads are identity if previous store is to same address
1378 Node* LoadNode::Identity(PhaseGVN* phase) {
1379 // If the previous store-maker is the right kind of Store, and the store is
1380 // to the same address, then we are equal to the value stored.
1381 Node* mem = in(Memory);
1382 Node* value = can_see_stored_value_through_membars(mem, phase);
1383 if( value ) {
1384 // byte, short & char stores truncate naturally.
1385 // A load has to load the truncated value which requires
1386 // some sort of masking operation and that requires an
1387 // Ideal call instead of an Identity call.
1388 if (memory_size() < BytesPerInt) {
1389 // If the input to the store does not fit with the load's result type,
1390 // it must be truncated via an Ideal call.
1391 if (!phase->type(value)->higher_equal(phase->type(this)))
1392 return this;
1393 }
1394 // (This works even when value is a Con, but LoadNode::Value
1395 // usually runs first, producing the singleton type of the Con.)
1396 if (!has_pinned_control_dependency() || value->is_Con()) {
1397 return value;
1398 } else {
1399 return this;
1400 }
1401 }
1402
1403 if (has_pinned_control_dependency()) {
1404 return this;
1405 }
1406 // Search for an existing data phi which was generated before for the same
1407 // instance's field to avoid infinite generation of phis in a loop.
1408 Node *region = mem->in(0);
1409 if (is_instance_field_load_with_local_phi(region)) {
1410 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1411 int this_index = phase->C->get_alias_index(addr_t);
1412 int this_offset = addr_t->offset();
1413 int this_iid = addr_t->instance_id();
1414 if (!addr_t->is_known_instance() &&
1415 addr_t->is_ptr_to_boxed_value()) {
1416 // Use _idx of address base (could be Phi node) for boxed values.
1417 intptr_t ignore = 0;
1418 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1419 if (base == nullptr) {
1420 return this;
1421 }
1422 this_iid = base->_idx;
1423 }
1424 const Type* this_type = bottom_type();
1425 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1426 Node* phi = region->fast_out(i);
1427 if (phi->is_Phi() && phi != mem &&
1428 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1429 return phi;
1430 }
1431 }
1432 }
1433
1434 return this;
1435 }
1436
1971 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1972 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1973
1974 // Skip up past a SafePoint control. Cannot do this for Stores because
1975 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1976 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1977 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1978 !addr_mark &&
1979 (depends_only_on_test() || has_unknown_control_dependency())) {
1980 ctrl = ctrl->in(0);
1981 set_req(MemNode::Control,ctrl);
1982 return this;
1983 }
1984
1985 intptr_t ignore = 0;
1986 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1987 if (base != nullptr
1988 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1989 // Check for useless control edge in some common special cases
1990 if (in(MemNode::Control) != nullptr
1991 && can_remove_control()
1992 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1993 && all_controls_dominate(base, phase->C->start(), phase)) {
1994 // A method-invariant, non-null address (constant or 'this' argument).
1995 set_req(MemNode::Control, nullptr);
1996 return this;
1997 }
1998 }
1999
2000 Node* mem = in(MemNode::Memory);
2001 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2002
2003 if (can_reshape && (addr_t != nullptr)) {
2004 // try to optimize our memory input
2005 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2006 if (opt_mem != mem) {
2007 set_req_X(MemNode::Memory, opt_mem, phase);
2008 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2009 return this;
2010 }
2123 // No match.
2124 return nullptr;
2125 }
2126
2127 //------------------------------Value-----------------------------------------
2128 const Type* LoadNode::Value(PhaseGVN* phase) const {
2129 // Either input is TOP ==> the result is TOP
2130 Node* mem = in(MemNode::Memory);
2131 const Type *t1 = phase->type(mem);
2132 if (t1 == Type::TOP) return Type::TOP;
2133 Node* adr = in(MemNode::Address);
2134 const TypePtr* tp = phase->type(adr)->isa_ptr();
2135 if (tp == nullptr || tp->empty()) return Type::TOP;
2136 int off = tp->offset();
2137 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2138 Compile* C = phase->C;
2139
2140 // If load can see a previous constant store, use that.
2141 Node* value = can_see_stored_value_through_membars(mem, phase);
2142 if (value != nullptr && value->is_Con()) {
2143 assert(value->bottom_type()->higher_equal(_type), "sanity");
2144 return value->bottom_type();
2145 }
2146
2147 // Try to guess loaded type from pointer type
2148 if (tp->isa_aryptr()) {
2149 const TypeAryPtr* ary = tp->is_aryptr();
2150 const Type* t = ary->elem();
2151
2152 // Determine whether the reference is beyond the header or not, by comparing
2153 // the offset against the offset of the start of the array's data.
2154 // Different array types begin at slightly different offsets (12 vs. 16).
2155 // We choose T_BYTE as an example base type that is least restrictive
2156 // as to alignment, which will therefore produce the smallest
2157 // possible base offset.
2158 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2159 const bool off_beyond_header = (off >= min_base_off);
2160
2161 // Try to constant-fold a stable array element.
2162 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2163 // Make sure the reference is not into the header and the offset is constant
2164 ciObject* aobj = ary->const_oop();
2165 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2166 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2167 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off,
2168 stable_dimension,
2169 value_basic_type(), is_unsigned());
2170 if (con_type != nullptr) {
2171 return con_type;
2172 }
2173 }
2174 }
2175
2176 // Don't do this for integer types. There is only potential profit if
2177 // the element type t is lower than _type; that is, for int types, if _type is
2178 // more restrictive than t. This only happens here if one is short and the other
2179 // char (both 16 bits), and in those cases we've made an intentional decision
2180 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2181 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2182 //
2183 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2184 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2185 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2186 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2187 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2188 // In fact, that could have been the original type of p1, and p1 could have
2189 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2190 // expression (LShiftL quux 3) independently optimized to the constant 8.
2191 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2192 && (_type->isa_vect() == nullptr)
2193 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2194 // t might actually be lower than _type, if _type is a unique
2195 // concrete subclass of abstract class t.
2196 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2197 const Type* jt = t->join_speculative(_type);
2198 // In any case, do not allow the join, per se, to empty out the type.
2199 if (jt->empty() && !t->empty()) {
2200 // This can happen if a interface-typed array narrows to a class type.
2201 jt = _type;
2202 }
2203 #ifdef ASSERT
2204 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2205 // The pointers in the autobox arrays are always non-null
2206 Node* base = adr->in(AddPNode::Base);
2207 if ((base != nullptr) && base->is_DecodeN()) {
2208 // Get LoadN node which loads IntegerCache.cache field
2209 base = base->in(1);
2210 }
2211 if ((base != nullptr) && base->is_Con()) {
2212 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2213 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2214 // It could be narrow oop
2215 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2216 }
2217 }
2218 }
2219 #endif
2220 return jt;
2221 }
2222 }
2223 } else if (tp->base() == Type::InstPtr) {
2224 assert( off != Type::OffsetBot ||
2225 // arrays can be cast to Objects
2226 !tp->isa_instptr() ||
2227 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2228 // unsafe field access may not have a constant offset
2229 C->has_unsafe_access(),
2230 "Field accesses must be precise" );
2231 // For oop loads, we expect the _type to be precise.
2232
2233 // Optimize loads from constant fields.
2234 const TypeInstPtr* tinst = tp->is_instptr();
2235 ciObject* const_oop = tinst->const_oop();
2236 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2237 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2238 if (con_type != nullptr) {
2239 return con_type;
2240 }
2241 }
2242 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2243 assert(off != Type::OffsetBot ||
2244 !tp->isa_instklassptr() ||
2245 // arrays can be cast to Objects
2246 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2247 // also allow array-loading from the primary supertype
2248 // array during subtype checks
2249 Opcode() == Op_LoadKlass,
2250 "Field accesses must be precise");
2251 // For klass/static loads, we expect the _type to be precise
2252 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2253 /* With mirrors being an indirect in the Klass*
2254 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2255 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2256 *
2257 * So check the type and klass of the node before the LoadP.
2264 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2265 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2266 return TypeInstPtr::make(klass->java_mirror());
2267 }
2268 }
2269 }
2270
2271 const TypeKlassPtr *tkls = tp->isa_klassptr();
2272 if (tkls != nullptr) {
2273 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2274 ciKlass* klass = tkls->exact_klass();
2275 // We are loading a field from a Klass metaobject whose identity
2276 // is known at compile time (the type is "exact" or "precise").
2277 // Check for fields we know are maintained as constants by the VM.
2278 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2279 // The field is Klass::_super_check_offset. Return its (constant) value.
2280 // (Folds up type checking code.)
2281 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2282 return TypeInt::make(klass->super_check_offset());
2283 }
2284 if (UseCompactObjectHeaders) {
2285 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2286 // The field is Klass::_prototype_header. Return its (constant) value.
2287 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2288 return TypeX::make(klass->prototype_header());
2289 }
2290 }
2291 // Compute index into primary_supers array
2292 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2293 // Check for overflowing; use unsigned compare to handle the negative case.
2294 if( depth < ciKlass::primary_super_limit() ) {
2295 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2296 // (Folds up type checking code.)
2297 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2298 ciKlass *ss = klass->super_of_depth(depth);
2299 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2300 }
2301 const Type* aift = load_array_final_field(tkls, klass);
2302 if (aift != nullptr) return aift;
2303 }
2304
2305 // We can still check if we are loading from the primary_supers array at a
2306 // shallow enough depth. Even though the klass is not exact, entries less
2307 // than or equal to its super depth are correct.
2308 if (tkls->is_loaded()) {
2309 ciKlass* klass = nullptr;
2343 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2344 // The key property of this type is that it folds up tests
2345 // for array-ness, since it proves that the layout_helper is positive.
2346 // Thus, a generic value like the basic object layout helper works fine.
2347 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2348 }
2349 }
2350
2351 // If we are loading from a freshly-allocated object/array, produce a zero.
2352 // Things to check:
2353 // 1. Load is beyond the header: headers are not guaranteed to be zero
2354 // 2. Load is not vectorized: vectors have no zero constant
2355 // 3. Load has no matching store, i.e. the input is the initial memory state
2356 const TypeOopPtr* tinst = tp->isa_oopptr();
2357 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2358 bool is_not_vect = (_type->isa_vect() == nullptr);
2359 if (is_not_header && is_not_vect) {
2360 Node* mem = in(MemNode::Memory);
2361 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2362 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2363 return Type::get_zero_type(_type->basic_type());
2364 }
2365 }
2366
2367 if (!UseCompactObjectHeaders) {
2368 Node* alloc = is_new_object_mark_load();
2369 if (alloc != nullptr) {
2370 return TypeX::make(markWord::prototype().value());
2371 }
2372 }
2373
2374 return _type;
2375 }
2376
2377 //------------------------------match_edge-------------------------------------
2378 // Do we Match on this edge index or not? Match only the address.
2379 uint LoadNode::match_edge(uint idx) const {
2380 return idx == MemNode::Address;
2381 }
2382
2383 //--------------------------LoadBNode::Ideal--------------------------------------
2384 //
2385 // If the previous store is to the same address as this load,
2386 // and the value stored was larger than a byte, replace this load
2387 // with the value stored truncated to a byte. If no truncation is
2388 // needed, the replacement is done in LoadNode::Identity().
2389 //
2390 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2499 }
2500 }
2501 // Identity call will handle the case where truncation is not needed.
2502 return LoadNode::Ideal(phase, can_reshape);
2503 }
2504
2505 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2506 Node* mem = in(MemNode::Memory);
2507 Node* value = can_see_stored_value_through_membars(mem, phase);
2508 if (value != nullptr && value->is_Con() &&
2509 !value->bottom_type()->higher_equal(_type)) {
2510 // If the input to the store does not fit with the load's result type,
2511 // it must be truncated. We can't delay until Ideal call since
2512 // a singleton Value is needed for split_thru_phi optimization.
2513 int con = value->get_int();
2514 return TypeInt::make((con << 16) >> 16);
2515 }
2516 return LoadNode::Value(phase);
2517 }
2518
2519 //=============================================================================
2520 //----------------------------LoadKlassNode::make------------------------------
2521 // Polymorphic factory method:
2522 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2523 // sanity check the alias category against the created node type
2524 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2525 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2526 #ifdef _LP64
2527 if (adr_type->is_ptr_to_narrowklass()) {
2528 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2529 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2530 }
2531 #endif
2532 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2533 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2534 }
2535
2536 //------------------------------Value------------------------------------------
2537 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2538 return klass_value_common(phase);
2571 }
2572 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2573 }
2574 if (!t->is_klass()) {
2575 // a primitive Class (e.g., int.class) has null for a klass field
2576 return TypePtr::NULL_PTR;
2577 }
2578 // Fold up the load of the hidden field
2579 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2580 }
2581 // non-constant mirror, so we can't tell what's going on
2582 }
2583 if (!tinst->is_loaded())
2584 return _type; // Bail out if not loaded
2585 if (offset == oopDesc::klass_offset_in_bytes()) {
2586 return tinst->as_klass_type(true);
2587 }
2588 }
2589
2590 // Check for loading klass from an array
2591 const TypeAryPtr *tary = tp->isa_aryptr();
2592 if (tary != nullptr &&
2593 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2594 return tary->as_klass_type(true);
2595 }
2596
2597 // Check for loading klass from an array klass
2598 const TypeKlassPtr *tkls = tp->isa_klassptr();
2599 if (tkls != nullptr && !StressReflectiveCode) {
2600 if (!tkls->is_loaded())
2601 return _type; // Bail out if not loaded
2602 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2603 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2604 // // Always returning precise element type is incorrect,
2605 // // e.g., element type could be object and array may contain strings
2606 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2607
2608 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2609 // according to the element type's subclassing.
2610 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2611 }
2612 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2613 tkls->offset() == in_bytes(Klass::super_offset())) {
2614 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2615 // The field is Klass::_super. Return its (constant) value.
2616 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2617 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2618 }
2619 }
2620
2621 if (tkls != nullptr && !UseSecondarySupersCache
2622 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2623 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2624 return TypePtr::NULL_PTR;
2625 }
2626
2627 // Bailout case
2628 return LoadNode::Value(phase);
2629 }
2630
2631 //------------------------------Identity---------------------------------------
2654 base = bs->step_over_gc_barrier(base);
2655 }
2656
2657 // We can fetch the klass directly through an AllocateNode.
2658 // This works even if the klass is not constant (clone or newArray).
2659 if (offset == oopDesc::klass_offset_in_bytes()) {
2660 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2661 if (allocated_klass != nullptr) {
2662 return allocated_klass;
2663 }
2664 }
2665
2666 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2667 // See inline_native_Class_query for occurrences of these patterns.
2668 // Java Example: x.getClass().isAssignableFrom(y)
2669 //
2670 // This improves reflective code, often making the Class
2671 // mirror go completely dead. (Current exception: Class
2672 // mirrors may appear in debug info, but we could clean them out by
2673 // introducing a new debug info operator for Klass.java_mirror).
2674
2675 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2676 && offset == java_lang_Class::klass_offset()) {
2677 if (base->is_Load()) {
2678 Node* base2 = base->in(MemNode::Address);
2679 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2680 Node* adr2 = base2->in(MemNode::Address);
2681 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2682 if (tkls != nullptr && !tkls->empty()
2683 && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2684 && adr2->is_AddP()
2685 ) {
2686 int mirror_field = in_bytes(Klass::java_mirror_offset());
2687 if (tkls->offset() == mirror_field) {
2688 #ifdef ASSERT
2689 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2690 assert(tkls2->offset() == 0, "not a load of java_mirror");
2691 #endif
2692 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2693 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2694 return adr2->in(AddPNode::Address);
2695 }
2696 }
2697 }
2698 }
2699 }
2700
2701 return this;
2702 }
2703
2704 LoadNode* LoadNode::clone_pinned() const {
2705 LoadNode* ld = clone()->as_Load();
2832 // Polymorphic factory method:
2833 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2834 assert((mo == unordered || mo == release), "unexpected");
2835 Compile* C = gvn.C;
2836 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
2837 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2838 ctl != nullptr, "raw memory operations should have control edge");
2839
2840 switch (bt) {
2841 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2842 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2843 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2844 case T_CHAR:
2845 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2846 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2847 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2848 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2849 case T_METADATA:
2850 case T_ADDRESS:
2851 case T_OBJECT:
2852 #ifdef _LP64
2853 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2854 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2855 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2856 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2857 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
2858 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2859 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2860 }
2861 #endif
2862 {
2863 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2864 }
2865 default:
2866 ShouldNotReachHere();
2867 return (StoreNode*)nullptr;
2868 }
2869 }
2870
2871 //--------------------------bottom_type----------------------------------------
2872 const Type *StoreNode::bottom_type() const {
2873 return Type::MEMORY;
2874 }
2875
2876 //------------------------------hash-------------------------------------------
2877 uint StoreNode::hash() const {
2878 // unroll addition of interesting fields
2879 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2880
2881 // Since they are not commoned, do not hash them:
2882 return NO_HASH;
2883 }
2884
2885 // Link together multiple stores (B/S/C/I) into a longer one.
2886 //
3508 }
3509 ss.print_cr("[TraceMergeStores]: with");
3510 merged_input_value->dump("\n", false, &ss);
3511 merged_store->dump("\n", false, &ss);
3512 tty->print("%s", ss.as_string());
3513 }
3514 #endif
3515
3516 //------------------------------Ideal------------------------------------------
3517 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3518 // When a store immediately follows a relevant allocation/initialization,
3519 // try to capture it into the initialization, or hoist it above.
3520 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3521 Node* p = MemNode::Ideal_common(phase, can_reshape);
3522 if (p) return (p == NodeSentinel) ? nullptr : p;
3523
3524 Node* mem = in(MemNode::Memory);
3525 Node* address = in(MemNode::Address);
3526 Node* value = in(MemNode::ValueIn);
3527 // Back-to-back stores to same address? Fold em up. Generally
3528 // unsafe if I have intervening uses. Also unsafe for masked or
3529 // scatter vector stores as the wider store.
3530 if (!this->is_StoreVector() || this->Opcode() == Op_StoreVector) {
3531 Node* st = mem;
3532 // If Store 'st' has more than one use, we cannot fold 'st' away.
3533 // For example, 'st' might be the final state at a conditional
3534 // return. Or, 'st' might be used by some node which is live at
3535 // the same time 'st' is live, which might be unschedulable. So,
3536 // require exactly ONE user until such time as we clone 'mem' for
3537 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3538 // true). Further, 'st' must be a contiguous store, otherwise
3539 // memory_size does not make sense for measuring overlap.
3540 while (st->is_Store() && st->outcnt() == 1 && (!st->is_StoreVector() || st->Opcode() == Op_StoreVector)) {
3541 // Looking at a dead closed cycle of memory?
3542 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3543 assert(Opcode() == st->Opcode() ||
3544 st->Opcode() == Op_StoreVector ||
3545 Opcode() == Op_StoreVector ||
3546 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3547 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3548 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3549 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3550 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3551
3552 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3553 st->as_Store()->memory_size() <= this->memory_size()) {
3554 assert(!is_predicated_vector() && !is_StoreVectorMasked() &&
3555 !is_StoreVectorScatter() && !is_StoreVectorScatterMasked() &&
3556 !st->is_predicated_vector() && !st->is_StoreVectorMasked() &&
3557 !st->is_StoreVectorScatter() && !st->is_StoreVectorScatterMasked(),
3558 "optimization only correct for full-width stores without holes");
3559 Node* use = st->raw_out(0);
3560 if (phase->is_IterGVN()) {
3561 phase->is_IterGVN()->rehash_node_delayed(use);
3562 }
3563 // It's OK to do this in the parser, since DU info is always accurate,
3564 // and the parser always refers to nodes via SafePointNode maps.
3565 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3566 return this;
3567 }
3568 st = st->in(MemNode::Memory);
3674 const StoreVectorNode* store_vector = as_StoreVector();
3675 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3676 const Node* store_indices = store_vector->indices();
3677 const Node* mem_indices = mem_vector->indices();
3678 const Node* store_mask = store_vector->mask();
3679 const Node* mem_mask = mem_vector->mask();
3680 // Ensure types, indices, and masks match
3681 if (store_vector->vect_type() == mem_vector->vect_type() &&
3682 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3683 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3684 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3685 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3686 result = mem;
3687 }
3688 }
3689 }
3690
3691 // Store of zero anywhere into a freshly-allocated object?
3692 // Then the store is useless.
3693 // (It must already have been captured by the InitializeNode.)
3694 if (result == this &&
3695 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3696 // a newly allocated object is already all-zeroes everywhere
3697 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3698 result = mem;
3699 }
3700
3701 if (result == this) {
3702 // the store may also apply to zero-bits in an earlier object
3703 Node* prev_mem = find_previous_store(phase);
3704 // Steps (a), (b): Walk past independent stores to find an exact match.
3705 if (prev_mem != nullptr) {
3706 if (prev_mem->is_top()) {
3707 // find_previous_store returns top when the access is dead
3708 return prev_mem;
3709 }
3710 Node* prev_val = can_see_stored_value(prev_mem, phase);
3711 if (prev_val != nullptr && prev_val == val) {
3712 // prev_val and val might differ by a cast; it would be good
3713 // to keep the more informative of the two.
3714 result = mem;
3715 }
3716 }
3717 }
3718 }
3719
3720 PhaseIterGVN* igvn = phase->is_IterGVN();
3721 if (result != this && igvn != nullptr) {
4214 // Clearing a short array is faster with stores
4215 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4216 // Already know this is a large node, do not try to ideal it
4217 if (_is_large) return nullptr;
4218
4219 const int unit = BytesPerLong;
4220 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4221 if (!t) return nullptr;
4222 if (!t->is_con()) return nullptr;
4223 intptr_t raw_count = t->get_con();
4224 intptr_t size = raw_count;
4225 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4226 // Clearing nothing uses the Identity call.
4227 // Negative clears are possible on dead ClearArrays
4228 // (see jck test stmt114.stmt11402.val).
4229 if (size <= 0 || size % unit != 0) return nullptr;
4230 intptr_t count = size / unit;
4231 // Length too long; communicate this to matchers and assemblers.
4232 // Assemblers are responsible to produce fast hardware clears for it.
4233 if (size > InitArrayShortSize) {
4234 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4235 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4236 return nullptr;
4237 }
4238 if (!IdealizeClearArrayNode) return nullptr;
4239 Node *mem = in(1);
4240 if( phase->type(mem)==Type::TOP ) return nullptr;
4241 Node *adr = in(3);
4242 const Type* at = phase->type(adr);
4243 if( at==Type::TOP ) return nullptr;
4244 const TypePtr* atp = at->isa_ptr();
4245 // adjust atp to be the correct array element address type
4246 if (atp == nullptr) atp = TypePtr::BOTTOM;
4247 else atp = atp->add_offset(Type::OffsetBot);
4248 // Get base for derived pointer purposes
4249 if( adr->Opcode() != Op_AddP ) Unimplemented();
4250 Node *base = adr->in(1);
4251
4252 Node *zero = phase->makecon(TypeLong::ZERO);
4253 Node *off = phase->MakeConX(BytesPerLong);
4254 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4255 count--;
4256 while (count--) {
4257 mem = phase->transform(mem);
4258 adr = phase->transform(AddPNode::make_with_base(base, adr, off));
4259 mem = new StoreLNode(in(0), mem, adr, atp, zero, MemNode::unordered, false);
4260 }
4261 return mem;
4262 }
4263
4264 //----------------------------step_through----------------------------------
4265 // Return allocation input memory edge if it is different instance
4266 // or itself if it is the one we are looking for.
4267 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4268 Node* n = *np;
4269 assert(n->is_ClearArray(), "sanity");
4270 intptr_t offset;
4271 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4272 // This method is called only before Allocate nodes are expanded
4273 // during macro nodes expansion. Before that ClearArray nodes are
4274 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4275 // Allocate nodes are expanded) which follows allocations.
4276 assert(alloc != nullptr, "should have allocation");
4277 if (alloc->_idx == instance_id) {
4278 // Can not bypass initialization of the instance we are looking for.
4279 return false;
4282 InitializeNode* init = alloc->initialization();
4283 if (init != nullptr)
4284 *np = init->in(TypeFunc::Memory);
4285 else
4286 *np = alloc->in(TypeFunc::Memory);
4287 return true;
4288 }
4289
4290 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4291 Node* base = dest;
4292 if (raw_base) {
4293 // May be called as part of the initialization of a just allocated object
4294 base = phase->C->top();
4295 }
4296 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4297 }
4298
4299 //----------------------------clear_memory-------------------------------------
4300 // Generate code to initialize object storage to zero.
4301 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4302 intptr_t start_offset,
4303 Node* end_offset,
4304 bool raw_base,
4305 PhaseGVN* phase) {
4306 intptr_t offset = start_offset;
4307
4308 int unit = BytesPerLong;
4309 if ((offset % unit) != 0) {
4310 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4311 const TypePtr* atp = TypeRawPtr::BOTTOM;
4312 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4313 mem = phase->transform(mem);
4314 offset += BytesPerInt;
4315 }
4316 assert((offset % unit) == 0, "");
4317
4318 // Initialize the remaining stuff, if any, with a ClearArray.
4319 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, raw_base, phase);
4320 }
4321
4322 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4323 Node* start_offset,
4324 Node* end_offset,
4325 bool raw_base,
4326 PhaseGVN* phase) {
4327 if (start_offset == end_offset) {
4328 // nothing to do
4329 return mem;
4330 }
4331
4332 int unit = BytesPerLong;
4333 Node* zbase = start_offset;
4334 Node* zend = end_offset;
4335
4336 // Scale to the unit required by the CPU:
4337 if (!Matcher::init_array_count_is_in_bytes) {
4338 Node* shift = phase->intcon(exact_log2(unit));
4339 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4340 zend = phase->transform(new URShiftXNode(zend, shift) );
4341 }
4342
4343 // Bulk clear double-words
4344 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4345 Node* adr = make_address(dest, start_offset, raw_base, phase);
4346 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4347 return phase->transform(mem);
4348 }
4349
4350 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4351 intptr_t start_offset,
4352 intptr_t end_offset,
4353 bool raw_base,
4354 PhaseGVN* phase) {
4355 if (start_offset == end_offset) {
4356 // nothing to do
4357 return mem;
4358 }
4359
4360 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4361 intptr_t done_offset = end_offset;
4362 if ((done_offset % BytesPerLong) != 0) {
4363 done_offset -= BytesPerInt;
4364 }
4365 if (done_offset > start_offset) {
4366 mem = clear_memory(ctl, mem, dest,
4367 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4368 }
4369 if (done_offset < end_offset) { // emit the final 32-bit store
4370 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4371 const TypePtr* atp = TypeRawPtr::BOTTOM;
4372 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4373 mem = phase->transform(mem);
4374 done_offset += BytesPerInt;
4375 }
4376 assert(done_offset == end_offset, "");
4377 return mem;
4378 }
4379
4380 //=============================================================================
4381 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4382 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4383 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4384 #ifdef ASSERT
4385 , _pair_idx(0)
4386 #endif
4387 {
4388 init_class_id(Class_MemBar);
4389 Node* top = C->top();
4390 init_req(TypeFunc::I_O,top);
4391 init_req(TypeFunc::FramePtr,top);
4392 init_req(TypeFunc::ReturnAdr,top);
4501 PhaseIterGVN* igvn = phase->is_IterGVN();
4502 remove(igvn);
4503 // Must return either the original node (now dead) or a new node
4504 // (Do not return a top here, since that would break the uniqueness of top.)
4505 return new ConINode(TypeInt::ZERO);
4506 }
4507 }
4508 return progress ? this : nullptr;
4509 }
4510
4511 //------------------------------Value------------------------------------------
4512 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4513 if( !in(0) ) return Type::TOP;
4514 if( phase->type(in(0)) == Type::TOP )
4515 return Type::TOP;
4516 return TypeTuple::MEMBAR;
4517 }
4518
4519 //------------------------------match------------------------------------------
4520 // Construct projections for memory.
4521 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4522 switch (proj->_con) {
4523 case TypeFunc::Control:
4524 case TypeFunc::Memory:
4525 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4526 }
4527 ShouldNotReachHere();
4528 return nullptr;
4529 }
4530
4531 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4532 trailing->_kind = TrailingStore;
4533 leading->_kind = LeadingStore;
4534 #ifdef ASSERT
4535 trailing->_pair_idx = leading->_idx;
4536 leading->_pair_idx = leading->_idx;
4537 #endif
4538 }
4539
4540 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4541 trailing->_kind = TrailingLoadStore;
4788 return (req() > RawStores);
4789 }
4790
4791 void InitializeNode::set_complete(PhaseGVN* phase) {
4792 assert(!is_complete(), "caller responsibility");
4793 _is_complete = Complete;
4794
4795 // After this node is complete, it contains a bunch of
4796 // raw-memory initializations. There is no need for
4797 // it to have anything to do with non-raw memory effects.
4798 // Therefore, tell all non-raw users to re-optimize themselves,
4799 // after skipping the memory effects of this initialization.
4800 PhaseIterGVN* igvn = phase->is_IterGVN();
4801 if (igvn) igvn->add_users_to_worklist(this);
4802 }
4803
4804 // convenience function
4805 // return false if the init contains any stores already
4806 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4807 InitializeNode* init = initialization();
4808 if (init == nullptr || init->is_complete()) return false;
4809 init->remove_extra_zeroes();
4810 // for now, if this allocation has already collected any inits, bail:
4811 if (init->is_non_zero()) return false;
4812 init->set_complete(phase);
4813 return true;
4814 }
4815
4816 void InitializeNode::remove_extra_zeroes() {
4817 if (req() == RawStores) return;
4818 Node* zmem = zero_memory();
4819 uint fill = RawStores;
4820 for (uint i = fill; i < req(); i++) {
4821 Node* n = in(i);
4822 if (n->is_top() || n == zmem) continue; // skip
4823 if (fill < i) set_req(fill, n); // compact
4824 ++fill;
4825 }
4826 // delete any empty spaces created:
4827 while (fill < req()) {
4828 del_req(fill);
4972 // store node that we'd like to capture. We need to check
4973 // the uses of the MergeMemNode.
4974 mems.push(n);
4975 }
4976 } else if (n->is_Mem()) {
4977 Node* other_adr = n->in(MemNode::Address);
4978 if (other_adr == adr) {
4979 failed = true;
4980 break;
4981 } else {
4982 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4983 if (other_t_adr != nullptr) {
4984 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4985 if (other_alias_idx == alias_idx) {
4986 // A load from the same memory slice as the store right
4987 // after the InitializeNode. We check the control of the
4988 // object/array that is loaded from. If it's the same as
4989 // the store control then we cannot capture the store.
4990 assert(!n->is_Store(), "2 stores to same slice on same control?");
4991 Node* base = other_adr;
4992 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4993 base = base->in(AddPNode::Base);
4994 if (base != nullptr) {
4995 base = base->uncast();
4996 if (base->is_Proj() && base->in(0) == alloc) {
4997 failed = true;
4998 break;
4999 }
5000 }
5001 }
5002 }
5003 }
5004 } else {
5005 failed = true;
5006 break;
5007 }
5008 }
5009 }
5010 }
5011 if (failed) {
5557 // z's_done 12 16 16 16 12 16 12
5558 // z's_needed 12 16 16 16 16 16 16
5559 // zsize 0 0 0 0 4 0 4
5560 if (next_full_store < 0) {
5561 // Conservative tack: Zero to end of current word.
5562 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5563 } else {
5564 // Zero to beginning of next fully initialized word.
5565 // Or, don't zero at all, if we are already in that word.
5566 assert(next_full_store >= zeroes_needed, "must go forward");
5567 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5568 zeroes_needed = next_full_store;
5569 }
5570 }
5571
5572 if (zeroes_needed > zeroes_done) {
5573 intptr_t zsize = zeroes_needed - zeroes_done;
5574 // Do some incremental zeroing on rawmem, in parallel with inits.
5575 zeroes_done = align_down(zeroes_done, BytesPerInt);
5576 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5577 zeroes_done, zeroes_needed,
5578 true,
5579 phase);
5580 zeroes_done = zeroes_needed;
5581 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5582 do_zeroing = false; // leave the hole, next time
5583 }
5584 }
5585
5586 // Collect the store and move on:
5587 phase->replace_input_of(st, MemNode::Memory, inits);
5588 inits = st; // put it on the linearized chain
5589 set_req(i, zmem); // unhook from previous position
5590
5591 if (zeroes_done == st_off)
5592 zeroes_done = next_init_off;
5593
5594 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5595
5596 #ifdef ASSERT
5617 remove_extra_zeroes(); // clear out all the zmems left over
5618 add_req(inits);
5619
5620 if (!(UseTLAB && ZeroTLAB)) {
5621 // If anything remains to be zeroed, zero it all now.
5622 zeroes_done = align_down(zeroes_done, BytesPerInt);
5623 // if it is the last unused 4 bytes of an instance, forget about it
5624 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5625 if (zeroes_done + BytesPerLong >= size_limit) {
5626 AllocateNode* alloc = allocation();
5627 assert(alloc != nullptr, "must be present");
5628 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5629 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5630 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5631 if (zeroes_done == k->layout_helper())
5632 zeroes_done = size_limit;
5633 }
5634 }
5635 if (zeroes_done < size_limit) {
5636 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5637 zeroes_done, size_in_bytes, true, phase);
5638 }
5639 }
5640
5641 set_complete(phase);
5642 return rawmem;
5643 }
5644
5645 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5646 auto replace_proj = [&](ProjNode* proj) {
5647 C->gvn_replace_by(proj, mem);
5648 return CONTINUE;
5649 };
5650 apply_to_projs(replace_proj, TypeFunc::Memory);
5651 }
5652
5653 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5654 DUIterator_Fast imax, i = fast_outs(imax);
5655 auto replace_proj = [&](ProjNode* proj) {
5656 igvn->replace_node(proj, mem);
5854 //------------------------------Identity---------------------------------------
5855 Node* MergeMemNode::Identity(PhaseGVN* phase) {
5856 // Identity if this merge point does not record any interesting memory
5857 // disambiguations.
5858 Node* base_mem = base_memory();
5859 Node* empty_mem = empty_memory();
5860 if (base_mem != empty_mem) { // Memory path is not dead?
5861 for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
5862 Node* mem = in(i);
5863 if (mem != empty_mem && mem != base_mem) {
5864 return this; // Many memory splits; no change
5865 }
5866 }
5867 }
5868 return base_mem; // No memory splits; ID on the one true input
5869 }
5870
5871 //------------------------------Ideal------------------------------------------
5872 // This method is invoked recursively on chains of MergeMem nodes
5873 Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) {
5874 // Remove chain'd MergeMems
5875 //
5876 // This is delicate, because the each "in(i)" (i >= Raw) is interpreted
5877 // relative to the "in(Bot)". Since we are patching both at the same time,
5878 // we have to be careful to read each "in(i)" relative to the old "in(Bot)",
5879 // but rewrite each "in(i)" relative to the new "in(Bot)".
5880 Node *progress = nullptr;
5881
5882
5883 Node* old_base = base_memory();
5884 Node* empty_mem = empty_memory();
5885 if (old_base == empty_mem)
5886 return nullptr; // Dead memory path.
5887
5888 MergeMemNode* old_mbase;
5889 if (old_base != nullptr && old_base->is_MergeMem())
5890 old_mbase = old_base->as_MergeMem();
5891 else
5892 old_mbase = nullptr;
5893 Node* new_base = old_base;
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "ci/ciInlineKlass.hpp"
28 #include "ci/ciInstanceKlass.hpp"
29 #include "classfile/javaClasses.hpp"
30 #include "classfile/systemDictionary.hpp"
31 #include "classfile/vmIntrinsics.hpp"
32 #include "compiler/compileLog.hpp"
33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/c2/barrierSetC2.hpp"
35 #include "gc/shared/tlab_globals.hpp"
36 #include "memory/allocation.inline.hpp"
37 #include "memory/resourceArea.hpp"
38 #include "oops/flatArrayKlass.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "opto/addnode.hpp"
41 #include "opto/arraycopynode.hpp"
42 #include "opto/callnode.hpp"
43 #include "opto/cfgnode.hpp"
44 #include "opto/compile.hpp"
45 #include "opto/connode.hpp"
46 #include "opto/convertnode.hpp"
47 #include "opto/inlinetypenode.hpp"
48 #include "opto/loopnode.hpp"
49 #include "opto/machnode.hpp"
50 #include "opto/matcher.hpp"
51 #include "opto/memnode.hpp"
52 #include "opto/mempointer.hpp"
53 #include "opto/mulnode.hpp"
54 #include "opto/narrowptrnode.hpp"
55 #include "opto/opcodes.hpp"
56 #include "opto/phaseX.hpp"
57 #include "opto/regalloc.hpp"
58 #include "opto/regmask.hpp"
59 #include "opto/rootnode.hpp"
60 #include "opto/traceMergeStoresTag.hpp"
61 #include "opto/vectornode.hpp"
62 #include "runtime/arguments.hpp"
63 #include "utilities/align.hpp"
64 #include "utilities/copy.hpp"
65 #include "utilities/globalDefinitions.hpp"
66 #include "utilities/macros.hpp"
67 #include "utilities/powerOfTwo.hpp"
68 #include "utilities/vmError.hpp"
69
70 // Portions of code courtesy of Clifford Click
71
72 // Optimization - Graph Style
73
74 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
75
76 //=============================================================================
77 uint MemNode::size_of() const { return sizeof(*this); }
78
79 const TypePtr *MemNode::adr_type() const {
80 Node* adr = in(Address);
81 if (adr == nullptr) return nullptr; // node is dead
82 const TypePtr* cross_check = nullptr;
136 st->print(", idx=Bot;");
137 else if (atp->index() == Compile::AliasIdxTop)
138 st->print(", idx=Top;");
139 else if (atp->index() == Compile::AliasIdxRaw)
140 st->print(", idx=Raw;");
141 else {
142 ciField* field = atp->field();
143 if (field) {
144 st->print(", name=");
145 field->print_name_on(st);
146 }
147 st->print(", idx=%d;", atp->index());
148 }
149 }
150 }
151
152 extern void print_alias_types();
153
154 #endif
155
156 // Find the memory output corresponding to the fall-through path of a call
157 static Node* find_call_fallthrough_mem_output(CallNode* call) {
158 ResourceMark rm;
159 CallProjections* projs = call->extract_projections(false, false);
160 Node* res = projs->fallthrough_memproj;
161 assert(res != nullptr, "must have a fallthrough mem output");
162 return res;
163 }
164
165 // Try to find a better memory input for a load from a strict final field
166 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
167 intptr_t offset = 0;
168 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
169 if (base == nullptr) {
170 return nullptr;
171 }
172
173 Node* base_uncasted = base->uncast();
174 if (base_uncasted->is_Proj()) {
175 Node* multi = base_uncasted->in(0);
176 if (multi->is_top()) {
177 // The pointer dies, make the memory die, too
178 return multi;
179 } else if (multi->is_Allocate()) {
180 base_local = base_uncasted->as_Proj();
181 return nullptr;
182 } else if (multi->is_Call()) {
183 if (!multi->is_CallJava() || multi->as_CallJava()->method() == nullptr || !multi->as_CallJava()->method()->return_value_is_larval()) {
184 // The oop is returned from a call, the memory can be the fallthrough output of the call
185 return find_call_fallthrough_mem_output(multi->as_Call());
186 }
187 } else if (multi->is_Start()) {
188 // The oop is a parameter
189 if (base_uncasted->as_Proj()->_con == TypeFunc::Parms && phase->C->method()->receiver_maybe_larval()) {
190 // The receiver of a constructor is similar to the result of an AllocateNode
191 base_local = base_uncasted->as_Proj();
192 return nullptr;
193 } else {
194 // Use the start memory otherwise
195 return multi->as_Start()->proj_out(TypeFunc::Memory);
196 }
197 }
198 }
199
200 return nullptr;
201 }
202
203 // Whether a call can modify a strict final field, given that the object is allocated inside the
204 // current compilation unit, or is the first parameter when the compilation root is a constructor.
205 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
206 // the target method is a subclass of the class declaring 'field'.
207 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
208 if (!call->is_CallJava()) {
209 return false;
210 }
211
212 ciMethod* target = call->as_CallJava()->method();
213 if (target == nullptr) {
214 return false;
215 } else if (target->intrinsic_id() == vmIntrinsicID::_linkToSpecial) {
216 // linkToSpecial can be used to call a constructor, used in the construction of objects in the
217 // reflection API
218 return true;
219 } else if (!target->is_object_constructor()) {
220 return false;
221 }
222
223 // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
224 // then the field is set inside the constructor, else the field must be set before the
225 // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
226 // while a field Sub.y must be set before Super::<init> is invoked.
227 // We can try to be more heroic and decide if the receiver of the constructor invocation is the
228 // object from which we are loading from. This, however, may be problematic as deciding if 2
229 // nodes are definitely different may not be trivial, especially if the graph is not canonical.
230 // As a result, it is made more conservative for now.
231 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
232 return target->holder()->is_subclass_of(field->holder());
233 }
234
235 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
236 assert(t_oop != nullptr, "sanity");
237 bool is_known_instance = t_oop->is_known_instance_field();
238 bool is_strict_final_load = false;
239
240 // After macro expansion, an allocation may become a call, changing the memory input to the
241 // memory output of that call would be illegal. As a result, disallow this transformation after
242 // macro expansion.
243 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
244 is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
245 #ifdef ASSERT
246 if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
247 assert(is_strict_final_load, "sanity check for basic cases");
248 }
249 #endif // ASSERT
250 }
251
252 if (!is_known_instance && !is_strict_final_load) {
253 return mchain;
254 }
255
256 Node* result = mchain;
257 ProjNode* base_local = nullptr;
258
259 ciField* field = nullptr;
260 if (is_strict_final_load) {
261 field = phase->C->alias_type(t_oop)->field();
262 assert(field != nullptr, "must point to a field");
263
264 Node* adr = load->in(MemNode::Address);
265 assert(phase->type(adr) == t_oop, "inconsistent type");
266 Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
267 if (tmp != nullptr) {
268 result = tmp;
269 }
270 }
271
272 uint instance_id = t_oop->instance_id();
273 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
274 Node* prev = nullptr;
275 while (prev != result) {
276 prev = result;
277 if (result == start_mem) {
278 // start_mem is the earliest memory possible
279 break;
280 }
281
282 // skip over a call which does not affect this memory slice
283 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
284 Node* proj_in = result->in(0);
285 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
286 // This is the allocation that creates the object from which we are loading from
287 break;
288 } else if (proj_in->is_Call()) {
289 // ArrayCopyNodes processed here as well
290 CallNode* call = proj_in->as_Call();
291 if (!call->may_modify(t_oop, phase)) {
292 result = call->in(TypeFunc::Memory);
293 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
294 result = call->in(TypeFunc::Memory);
295 }
296 } else if (proj_in->Opcode() == Op_Tuple) {
297 // The call will be folded, skip over it.
298 break;
299 } else if (proj_in->is_Initialize()) {
300 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
301 // Stop if this is the initialization for the object instance which
302 // contains this memory slice, otherwise skip over it.
303 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
304 break;
305 }
306 if (is_known_instance) {
307 result = proj_in->in(TypeFunc::Memory);
308 } else if (is_strict_final_load) {
309 Node* klass = alloc->in(AllocateNode::KlassNode);
310 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
311 if (tklass->klass_is_exact() && !tklass->exact_klass()->is_subclass_of(t_oop->is_instptr()->instance_klass())) {
312 // Allocation of an unrelated type, must be another object
313 result = proj_in->in(TypeFunc::Memory);
314 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
315 // Allocation of another object
316 result = proj_in->in(TypeFunc::Memory);
317 }
318 }
319 } else if (proj_in->is_MemBar()) {
320 ArrayCopyNode* ac = nullptr;
321 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
322 break;
323 }
324 result = proj_in->in(TypeFunc::Memory);
325 } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
326 bool mismatched = proj_in->is_LoadFlat() ? proj_in->as_LoadFlat()->is_mismatched() : proj_in->as_StoreFlat()->is_mismatched();
327 if (is_strict_final_load || (is_known_instance && !mismatched)) {
328 // LoadFlat and StoreFlat cannot happen to strict final fields
329 // LoadFlat and StoreFlat to known instances are removed at the end of EA unless mismatched: this one is unrelated
330 result = proj_in->in(TypeFunc::Memory);
331 }
332 } else if (proj_in->is_top()) {
333 break; // dead code
334 } else {
335 assert(false, "unexpected projection of %s", proj_in->Name());
336 }
337 } else if (result->is_ClearArray()) {
338 if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
339 // Can not bypass initialization of the instance
340 // we are looking for.
341 break;
342 }
343 // Otherwise skip it (the call updated 'result' value).
344 } else if (result->is_MergeMem()) {
345 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
346 }
347 }
348 return result;
349 }
350
351 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
352 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
353 if (t_oop == nullptr)
354 return mchain; // don't try to optimize non-oop types
355 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
356 bool is_instance = t_oop->is_known_instance_field();
357 PhaseIterGVN *igvn = phase->is_IterGVN();
358 if (is_instance && igvn != nullptr && result->is_Phi()) {
359 PhiNode *mphi = result->as_Phi();
360 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
361 const TypePtr *t = mphi->adr_type();
362 bool do_split = false;
363 // In the following cases, Load memory input can be further optimized based on
364 // its precise address type
365 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
366 do_split = true;
367 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
368 const TypeOopPtr* mem_t =
369 t->is_oopptr()->cast_to_exactness(true)
370 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
371 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
372 if (t_oop->isa_aryptr()) {
373 mem_t = mem_t->is_aryptr()
374 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
375 ->cast_to_size(t_oop->is_aryptr()->size())
376 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
377 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
378 ->with_offset(t_oop->is_aryptr()->offset())
379 ->is_aryptr();
380 }
381 do_split = mem_t == t_oop;
382 }
383 if (do_split) {
384 // clone the Phi with our address type
385 result = mphi->split_out_instance(t_adr, igvn);
386 } else {
387 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
388 }
389 }
390 return result;
391 }
392
393 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
394 uint alias_idx = phase->C->get_alias_index(tp);
395 Node *mem = mmem;
396 #ifdef ASSERT
397 {
398 // Check that current type is consistent with the alias index used during graph construction
399 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
400 bool consistent = adr_check == nullptr || adr_check->empty() ||
401 phase->C->must_alias(adr_check, alias_idx );
402 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
403 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
404 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
405 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
406 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
407 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
408 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
409 // don't assert if it is dead code.
410 consistent = true;
411 }
412 if( !consistent ) {
413 st->print("alias_idx==%d, adr_check==", alias_idx);
414 if( adr_check == nullptr ) {
415 st->print("null");
416 } else {
417 adr_check->dump();
418 }
419 st->cr();
420 print_alias_types();
421 assert(consistent, "adr_check must match alias idx");
422 }
423 }
424 #endif
727 }
728
729 // Find an arraycopy ac that produces the memory state represented by parameter mem.
730 // Return ac if
731 // (a) can_see_stored_value=true and ac must have set the value for this load or if
732 // (b) can_see_stored_value=false and ac could have set the value for this load or if
733 // (c) can_see_stored_value=false and ac cannot have set the value for this load.
734 // In case (c) change the parameter mem to the memory input of ac to skip it
735 // when searching stored value.
736 // Otherwise return null.
737 Node* LoadNode::find_previous_arraycopy(PhaseValues* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const {
738 ArrayCopyNode* ac = find_array_copy_clone(ld_alloc, mem);
739 if (ac != nullptr) {
740 Node* ld_addp = in(MemNode::Address);
741 Node* src = ac->in(ArrayCopyNode::Src);
742 const TypeAryPtr* ary_t = phase->type(src)->isa_aryptr();
743
744 // This is a load from a cloned array. The corresponding arraycopy ac must
745 // have set the value for the load and we can return ac but only if the load
746 // is known to be within bounds. This is checked below.
747 // TODO 8350865: Support flat arrays in LoadNode::find_previous_arraycopy
748 if (ary_t != nullptr && ary_t->is_not_flat() && ld_addp->is_AddP()) {
749 Node* ld_offs = ld_addp->in(AddPNode::Offset);
750 BasicType ary_elem = ary_t->elem()->array_element_basic_type();
751 jlong header = arrayOopDesc::base_offset_in_bytes(ary_elem);
752 jlong elemsize = type2aelembytes(ary_elem);
753
754 const TypeX* ld_offs_t = phase->type(ld_offs)->isa_intptr_t();
755 const TypeInt* sizetype = ary_t->size();
756
757 if (ld_offs_t->_lo >= header && ld_offs_t->_hi < (sizetype->_lo * elemsize + header)) {
758 // The load is known to be within bounds. It receives its value from ac.
759 return ac;
760 }
761 // The load is known to be out-of-bounds.
762 }
763 // The load could be out-of-bounds. It must not be hoisted but must remain
764 // dependent on the runtime range check. This is achieved by returning null.
765 } else if (mem->is_Proj() && mem->in(0) != nullptr && mem->in(0)->is_ArrayCopy()) {
766 ArrayCopyNode* ac = mem->in(0)->as_ArrayCopy();
767
768 if (ac->is_arraycopy_validated() ||
1138 in_bytes(JavaThread::vthread_offset()),
1139 in_bytes(JavaThread::scopedValueCache_offset()),
1140 };
1141
1142 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1143 if (offset == offsets[i]) {
1144 return true;
1145 }
1146 }
1147 }
1148
1149 return false;
1150 }
1151 #endif
1152
1153 //----------------------------LoadNode::make-----------------------------------
1154 // Polymorphic factory method:
1155 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1156 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1157 Compile* C = gvn.C;
1158 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr(), true) == C->get_alias_index(adr_type, true), "adr and adr_type must agree");
1159
1160 // sanity check the alias category against the created node type
1161 assert(!(adr_type->isa_oopptr() &&
1162 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1163 "use LoadKlassNode instead");
1164 assert(!(adr_type->isa_aryptr() &&
1165 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1166 "use LoadRangeNode instead");
1167 // Check control edge of raw loads
1168 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1169 // oop will be recorded in oop map if load crosses safepoint
1170 rt->isa_oopptr() || is_immutable_value(adr),
1171 "raw memory operations should have control edge");
1172 LoadNode* load = nullptr;
1173 switch (bt) {
1174 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1175 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1176 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1177 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1178 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1179 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1180 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1181 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1182 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1183 case T_ARRAY:
1184 case T_OBJECT:
1185 case T_NARROWOOP:
1186 #ifdef _LP64
1187 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1188 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1189 } else
1190 #endif
1191 {
1192 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1193 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1194 }
1195 break;
1196 default:
1197 guarantee(false, "unexpected basic type %s", type2name(bt));
1198 break;
1199 }
1200 assert(load != nullptr, "LoadNode should have been created");
1201 if (unaligned) {
1202 load->set_unaligned_access();
1203 }
1204 if (mismatched) {
1205 load->set_mismatched_access();
1206 }
1207 if (unsafe) {
1208 load->set_unsafe_access();
1209 }
1210 load->set_barrier_data(barrier_data);
1211 if (load->Opcode() == Op_LoadN) {
1212 Node* ld = gvn.transform(load);
1213 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1214 }
1215
1216 return load;
1217 }
1218
1219 //------------------------------hash-------------------------------------------
1220 uint LoadNode::hash() const {
1221 // unroll addition of interesting fields
1222 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1223 }
1224
1225 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1226 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1227 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1228 bool is_stable_ary = FoldStableValues &&
1229 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1230 tp->isa_aryptr()->is_stable();
1231
1232 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1233 }
1234
1235 return false;
1236 }
1237
1238 // Is the value loaded previously stored by an arraycopy? If so return
1239 // a load node that reads from the source array so we may be able to
1240 // optimize out the ArrayCopy node later.
1241 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1242 Node* ld_adr = in(MemNode::Address);
1243 intptr_t ld_off = 0;
1244 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1245 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1246 if (ac != nullptr) {
1247 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1248
1249 Node* mem = ac->in(TypeFunc::Memory);
1250 Node* ctl = ac->in(0);
1251 Node* src = ac->in(ArrayCopyNode::Src);
1252
1260 if (ac->as_ArrayCopy()->is_clonebasic()) {
1261 assert(ld_alloc != nullptr, "need an alloc");
1262 assert(addp->is_AddP(), "address must be addp");
1263 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1264 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1265 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1266 addp->set_req(AddPNode::Base, src);
1267 addp->set_req(AddPNode::Address, src);
1268 } else {
1269 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1270 ac->as_ArrayCopy()->is_copyof_validated() ||
1271 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1272 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1273 addp->set_req(AddPNode::Base, src);
1274 addp->set_req(AddPNode::Address, src);
1275
1276 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1277 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1278 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1279
1280 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1281
1282 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1283 #ifdef _LP64
1284 diff = phase->transform(new ConvI2LNode(diff));
1285 #endif
1286 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1287
1288 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1289 addp->set_req(AddPNode::Offset, offset);
1290 }
1291 addp = phase->transform(addp);
1292 #ifdef ASSERT
1293 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1294 ld->_adr_type = adr_type;
1295 #endif
1296 ld->set_req(MemNode::Address, addp);
1297 ld->set_req(0, ctl);
1298 ld->set_req(MemNode::Memory, mem);
1299 return ld;
1300 }
1301 return nullptr;
1302 }
1303
1304 static Node* see_through_inline_type(PhaseValues* phase, const LoadNode* load, Node* base, int offset) {
1305 if (load->is_mismatched_access() || base == nullptr) {
1306 return nullptr;
1307 }
1308
1309 InlineTypeNode* vt = base->isa_InlineType();
1310 if (vt == nullptr || offset < vt->type()->inline_klass()->payload_offset()) {
1311 return nullptr;
1312 }
1313
1314 Node* value = vt->field_value_by_offset(offset, true);
1315 assert(value != nullptr, "must see some value");
1316 return value;
1317 }
1318
1319 // This routine exists to make sure this set of tests is done the same
1320 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1321 // will change the graph shape in a way which makes memory alive twice at the
1322 // same time (uses the Oracle model of aliasing), then some
1323 // LoadXNode::Identity will fold things back to the equivalence-class model
1324 // of aliasing.
1325 // This method may find an unencoded node instead of the corresponding encoded one.
1326 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1327 Node* ld_adr = in(MemNode::Address);
1328 intptr_t ld_off = 0;
1329 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1330 // Try to see through an InlineTypeNode
1331 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1332 if (value != nullptr) {
1333 return value;
1334 }
1335
1336 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1337 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1338
1339 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1340 uint alias_idx = atp->index();
1341 Node* result = nullptr;
1342 Node* current = st;
1343 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1344 // this load. Stop once any other kind of node is encountered.
1345 //
1346 // In principle, folding a load is moving it up until it meets a matching store.
1347 //
1348 // store(ptr, v); store(ptr, v); store(ptr, v);
1349 // membar1; -> membar1; -> load(ptr);
1350 // membar2; load(ptr); membar1;
1351 // load(ptr); membar2; membar2;
1352 //
1353 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1354 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1355 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1367 MergeMemNode* merge = mem->as_MergeMem();
1368 Node* new_st = merge->memory_at(alias_idx);
1369 if (new_st == merge->base_memory()) {
1370 // Keep searching
1371 current = new_st;
1372 continue;
1373 }
1374 // Save the new memory state for the slice and fall through
1375 // to exit.
1376 result = new_st;
1377 }
1378 }
1379 break;
1380 }
1381 if (result != nullptr) {
1382 st = result;
1383 }
1384 }
1385
1386 Node* res = can_see_stored_value(st, phase);
1387 // TODO: reimplement assert, see: JDK-8386157
1388 //assert(res == nullptr || is_java_primitive(value_basic_type()) || res->bottom_type()->higher_equal(type()), "the fold is unsafe");
1389 return res;
1390 }
1391
1392 // If st is a store to the same location as this, return the stored value
1393 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1394 Node* ld_adr = in(MemNode::Address);
1395 intptr_t ld_off = 0;
1396 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1397 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1398 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1399
1400 // Loop around twice in the case Load -> Initialize -> Store.
1401 // (See PhaseIterGVN::add_users_to_worklist, which knows about this case.)
1402 for (int trip = 0; trip <= 1; trip++) {
1403
1404 if (st->is_Store()) {
1405 Node* st_adr = st->in(MemNode::Address);
1406 if (st_adr != ld_adr) {
1407 // Try harder before giving up. Unify base pointers with casts (e.g., raw/non-raw pointers).
1408 intptr_t st_off = 0;
1456
1457 // There are some cases in which the Type of the load is narrower than the Type of the value
1458 // that is stored into that location. The most common case is array polymorphism, when the
1459 // type of an array element depends on the type of the array. In addition, there are some
1460 // corner cases, the first one is concurrent class loading, when CHA can result in a narrower
1461 // Type than what is declared only after the child class is loaded, and the second case is
1462 // unsafe accesses when we do not check for type safety. See JDK-8388184.
1463 return nullptr;
1464 }
1465
1466 // A load from a freshly-created object always returns zero.
1467 // (This can happen after LoadNode::Ideal resets the load's memory input
1468 // to find_captured_store, which returned InitializeNode::zero_memory.)
1469 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1470 (st->in(0) == ld_alloc) &&
1471 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1472 // return a zero value for the load's basic type
1473 // (This is one of the few places where a generic PhaseTransform
1474 // can create new nodes. Think of it as lazily manifesting
1475 // virtually pre-existing constants.)
1476 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1477 if (init_value != nullptr) {
1478 const TypeAryPtr* ld_adr_type = phase->type(ld_adr)->isa_aryptr();
1479 if (ld_adr_type == nullptr) {
1480 return nullptr;
1481 }
1482
1483 // We know that this is not a flat array, the load should return the whole oop
1484 if (ld_adr_type->is_not_flat()) {
1485 return init_value;
1486 }
1487
1488 // If this is a flat array, try to see through init_value
1489 if (init_value->is_EncodeP()) {
1490 init_value = init_value->in(1);
1491 }
1492 if (!init_value->is_InlineType() || ld_adr_type->field_offset() == Type::Offset::bottom) {
1493 return nullptr;
1494 }
1495
1496 ciInlineKlass* vk = phase->type(init_value)->inline_klass();
1497 int field_offset_in_payload = ld_adr_type->field_offset().get();
1498 if (field_offset_in_payload == vk->null_marker_offset_in_payload()) {
1499 return init_value->as_InlineType()->get_null_marker();
1500 } else {
1501 return init_value->as_InlineType()->field_value_by_offset(field_offset_in_payload + vk->payload_offset(), true);
1502 }
1503 }
1504 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1505 if (value_basic_type() != T_VOID) {
1506 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1507 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1508 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1509 // by the ArrayCopyNode.
1510 return phase->zerocon(value_basic_type());
1511 }
1512 } else {
1513 // TODO: materialize all-zero vector constant
1514 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1515 }
1516 }
1517
1518 // A load from an initialization barrier can match a captured store.
1519 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1520 InitializeNode* init = st->in(0)->as_Initialize();
1521 AllocateNode* alloc = init->allocation();
1522 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1523 // examine a captured store value
1524 st = init->find_captured_store(ld_off, memory_size(), phase);
1537 base = bs->step_over_gc_barrier(base);
1538 if (base != nullptr && base->is_Proj() &&
1539 base->as_Proj()->_con == TypeFunc::Parms &&
1540 base->in(0)->is_CallStaticJava() &&
1541 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1542 return base->in(0)->in(TypeFunc::Parms);
1543 }
1544 }
1545
1546 break;
1547 }
1548
1549 return nullptr;
1550 }
1551
1552 //----------------------is_instance_field_load_with_local_phi------------------
1553 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1554 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1555 in(Address)->is_AddP() ) {
1556 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1557 // Only known instances and immutable fields
1558 if( t_oop != nullptr &&
1559 (t_oop->is_ptr_to_strict_final_field() ||
1560 t_oop->is_known_instance_field()) &&
1561 t_oop->offset() != Type::OffsetBot &&
1562 t_oop->offset() != Type::OffsetTop) {
1563 return true;
1564 }
1565 }
1566 return false;
1567 }
1568
1569 //------------------------------Identity---------------------------------------
1570 // Loads are identity if previous store is to same address
1571 Node* LoadNode::Identity(PhaseGVN* phase) {
1572 // If the previous store-maker is the right kind of Store, and the store is
1573 // to the same address, then we are equal to the value stored.
1574 Node* mem = in(Memory);
1575 Node* value = can_see_stored_value_through_membars(mem, phase);
1576 if( value ) {
1577 // byte, short & char stores truncate naturally.
1578 // A load has to load the truncated value which requires
1579 // some sort of masking operation and that requires an
1580 // Ideal call instead of an Identity call.
1581 if (memory_size() < BytesPerInt) {
1582 // If the input to the store does not fit with the load's result type,
1583 // it must be truncated via an Ideal call.
1584 if (!phase->type(value)->higher_equal(phase->type(this)))
1585 return this;
1586 }
1587
1588 if (phase->type(value)->isa_ptr() && phase->type(this)->isa_narrowoop()) {
1589 return this;
1590 }
1591 // (This works even when value is a Con, but LoadNode::Value
1592 // usually runs first, producing the singleton type of the Con.)
1593 if (!has_pinned_control_dependency() || value->is_Con()) {
1594 return value;
1595 } else {
1596 return this;
1597 }
1598 }
1599
1600 if (has_pinned_control_dependency()) {
1601 return this;
1602 }
1603 // Search for an existing data phi which was generated before for the same
1604 // instance's field to avoid infinite generation of phis in a loop.
1605 Node *region = mem->in(0);
1606 if (is_instance_field_load_with_local_phi(region)) {
1607 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1608 int this_index = phase->C->get_alias_index(addr_t);
1609 int this_offset = addr_t->offset();
1610 int this_iid = addr_t->instance_id();
1611 if (!addr_t->is_known_instance() &&
1612 addr_t->is_ptr_to_strict_final_field()) {
1613 // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1614 intptr_t ignore = 0;
1615 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1616 if (base == nullptr) {
1617 return this;
1618 }
1619 this_iid = base->_idx;
1620 }
1621 const Type* this_type = bottom_type();
1622 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1623 Node* phi = region->fast_out(i);
1624 if (phi->is_Phi() && phi != mem &&
1625 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1626 return phi;
1627 }
1628 }
1629 }
1630
1631 return this;
1632 }
1633
2168 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2169 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2170
2171 // Skip up past a SafePoint control. Cannot do this for Stores because
2172 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2173 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2174 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2175 !addr_mark &&
2176 (depends_only_on_test() || has_unknown_control_dependency())) {
2177 ctrl = ctrl->in(0);
2178 set_req(MemNode::Control,ctrl);
2179 return this;
2180 }
2181
2182 intptr_t ignore = 0;
2183 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2184 if (base != nullptr
2185 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2186 // Check for useless control edge in some common special cases
2187 if (in(MemNode::Control) != nullptr
2188 // TODO 8350865 Can we re-enable this?
2189 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2190 && can_remove_control()
2191 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2192 && all_controls_dominate(base, phase->C->start(), phase)) {
2193 // A method-invariant, non-null address (constant or 'this' argument).
2194 set_req(MemNode::Control, nullptr);
2195 return this;
2196 }
2197 }
2198
2199 Node* mem = in(MemNode::Memory);
2200 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2201
2202 if (can_reshape && (addr_t != nullptr)) {
2203 // try to optimize our memory input
2204 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2205 if (opt_mem != mem) {
2206 set_req_X(MemNode::Memory, opt_mem, phase);
2207 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2208 return this;
2209 }
2322 // No match.
2323 return nullptr;
2324 }
2325
2326 //------------------------------Value-----------------------------------------
2327 const Type* LoadNode::Value(PhaseGVN* phase) const {
2328 // Either input is TOP ==> the result is TOP
2329 Node* mem = in(MemNode::Memory);
2330 const Type *t1 = phase->type(mem);
2331 if (t1 == Type::TOP) return Type::TOP;
2332 Node* adr = in(MemNode::Address);
2333 const TypePtr* tp = phase->type(adr)->isa_ptr();
2334 if (tp == nullptr || tp->empty()) return Type::TOP;
2335 int off = tp->offset();
2336 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2337 Compile* C = phase->C;
2338
2339 // If load can see a previous constant store, use that.
2340 Node* value = can_see_stored_value_through_membars(mem, phase);
2341 if (value != nullptr && value->is_Con()) {
2342 if (phase->type(value)->isa_ptr() && _type->isa_narrowoop()) {
2343 return phase->type(value)->make_narrowoop();
2344 } else {
2345 assert(value->bottom_type()->higher_equal(_type), "sanity");
2346 return phase->type(value);
2347 }
2348 }
2349 // Try to guess loaded type from pointer type
2350 if (tp->isa_aryptr()) {
2351 const TypeAryPtr* ary = tp->is_aryptr();
2352 const Type* t = ary->elem();
2353
2354 // Determine whether the reference is beyond the header or not, by comparing
2355 // the offset against the offset of the start of the array's data.
2356 // Different array types begin at slightly different offsets (12 vs. 16).
2357 // We choose T_BYTE as an example base type that is least restrictive
2358 // as to alignment, which will therefore produce the smallest
2359 // possible base offset.
2360 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2361 const bool off_beyond_header = (off >= min_base_off);
2362
2363 // Try to constant-fold a stable array element.
2364 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2365 // Make sure the reference is not into the header and the offset is constant
2366 ciObject* aobj = ary->const_oop();
2367 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2368 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2369 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off, ary->field_offset().get(),
2370 stable_dimension,
2371 value_basic_type(), is_unsigned());
2372 if (con_type != nullptr) {
2373 return con_type;
2374 }
2375 }
2376 }
2377
2378 // Don't do this for integer types. There is only potential profit if
2379 // the element type t is lower than _type; that is, for int types, if _type is
2380 // more restrictive than t. This only happens here if one is short and the other
2381 // char (both 16 bits), and in those cases we've made an intentional decision
2382 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2383 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2384 //
2385 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2386 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2387 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2388 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2389 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2390 // In fact, that could have been the original type of p1, and p1 could have
2391 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2392 // expression (LShiftL quux 3) independently optimized to the constant 8.
2393 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2394 && (_type->isa_vect() == nullptr)
2395 && !ary->is_flat()
2396 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2397 // t might actually be lower than _type, if _type is a unique
2398 // concrete subclass of abstract class t.
2399 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2400 const Type* jt = t->join_speculative(_type);
2401 // In any case, do not allow the join, per se, to empty out the type.
2402 if (jt->empty() && !t->empty()) {
2403 // This can happen if a interface-typed array narrows to a class type.
2404 jt = _type;
2405 }
2406 #ifdef ASSERT
2407 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2408 // The pointers in the autobox arrays are always non-null
2409 Node* base = adr->in(AddPNode::Base);
2410 if ((base != nullptr) && base->is_DecodeN()) {
2411 // Get LoadN node which loads IntegerCache.cache field
2412 base = base->in(1);
2413 }
2414 if ((base != nullptr) && base->is_Con()) {
2415 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2416 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2417 // It could be narrow oop
2418 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2419 }
2420 }
2421 }
2422 #endif
2423 return jt;
2424 }
2425 }
2426 } else if (tp->base() == Type::InstPtr) {
2427 assert( off != Type::OffsetBot ||
2428 // arrays can be cast to Objects
2429 !tp->isa_instptr() ||
2430 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2431 // Default value load
2432 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2433 // unsafe field access may not have a constant offset
2434 is_unsafe_access(),
2435 "Field accesses must be precise" );
2436 // For oop loads, we expect the _type to be precise.
2437
2438 const TypeInstPtr* tinst = tp->is_instptr();
2439 BasicType bt = value_basic_type();
2440
2441 // Fold loads of the field map
2442 if (tinst != nullptr) {
2443 ciInstanceKlass* ik = tinst->instance_klass();
2444 int offset = tinst->offset();
2445 if (ik == phase->C->env()->Class_klass()) {
2446 ciType* t = tinst->java_mirror_type();
2447 if (t != nullptr && t->is_inlinetype() && offset == t->as_inline_klass()->field_map_offset()) {
2448 ciConstant map = t->as_inline_klass()->get_field_map();
2449 bool is_narrow_oop = (bt == T_NARROWOOP);
2450 return Type::make_from_constant(map, true, 1, is_narrow_oop);
2451 }
2452 }
2453 }
2454
2455 // Optimize loads from constant fields.
2456 ciObject* const_oop = tinst->const_oop();
2457 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2458 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2459 if (con_type != nullptr) {
2460 return con_type;
2461 }
2462 }
2463 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2464 assert(off != Type::OffsetBot ||
2465 !tp->isa_instklassptr() ||
2466 // arrays can be cast to Objects
2467 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2468 // also allow array-loading from the primary supertype
2469 // array during subtype checks
2470 Opcode() == Op_LoadKlass,
2471 "Field accesses must be precise");
2472 // For klass/static loads, we expect the _type to be precise
2473 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2474 /* With mirrors being an indirect in the Klass*
2475 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2476 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2477 *
2478 * So check the type and klass of the node before the LoadP.
2485 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2486 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2487 return TypeInstPtr::make(klass->java_mirror());
2488 }
2489 }
2490 }
2491
2492 const TypeKlassPtr *tkls = tp->isa_klassptr();
2493 if (tkls != nullptr) {
2494 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2495 ciKlass* klass = tkls->exact_klass();
2496 // We are loading a field from a Klass metaobject whose identity
2497 // is known at compile time (the type is "exact" or "precise").
2498 // Check for fields we know are maintained as constants by the VM.
2499 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2500 // The field is Klass::_super_check_offset. Return its (constant) value.
2501 // (Folds up type checking code.)
2502 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2503 return TypeInt::make(klass->super_check_offset());
2504 }
2505 if (klass->is_inlinetype() && tkls->offset() == in_bytes(InstanceKlass::acmp_maps_offset_offset())) {
2506 return TypeInt::make(klass->as_inline_klass()->field_map_offset());
2507 }
2508 if (klass->is_obj_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset())) {
2509 // Fold loads from LibraryCallKit::load_default_refined_array_klass
2510 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr();
2511 }
2512 if (klass->is_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::properties_offset())) {
2513 assert(klass->is_type_array_klass() || tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2514 return TypeInt::make((jint)klass->as_array_klass()->properties().value());
2515 }
2516 if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2517 assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2518 return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2519 }
2520 if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2521 // The field is Klass::_prototype_header. Return its (constant) value.
2522 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2523 return TypeX::make(klass->prototype_header());
2524 }
2525 // Compute index into primary_supers array
2526 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2527 // Check for overflowing; use unsigned compare to handle the negative case.
2528 if( depth < ciKlass::primary_super_limit() ) {
2529 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2530 // (Folds up type checking code.)
2531 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2532 ciKlass *ss = klass->super_of_depth(depth);
2533 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2534 }
2535 const Type* aift = load_array_final_field(tkls, klass);
2536 if (aift != nullptr) return aift;
2537 }
2538
2539 // We can still check if we are loading from the primary_supers array at a
2540 // shallow enough depth. Even though the klass is not exact, entries less
2541 // than or equal to its super depth are correct.
2542 if (tkls->is_loaded()) {
2543 ciKlass* klass = nullptr;
2577 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2578 // The key property of this type is that it folds up tests
2579 // for array-ness, since it proves that the layout_helper is positive.
2580 // Thus, a generic value like the basic object layout helper works fine.
2581 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2582 }
2583 }
2584
2585 // If we are loading from a freshly-allocated object/array, produce a zero.
2586 // Things to check:
2587 // 1. Load is beyond the header: headers are not guaranteed to be zero
2588 // 2. Load is not vectorized: vectors have no zero constant
2589 // 3. Load has no matching store, i.e. the input is the initial memory state
2590 const TypeOopPtr* tinst = tp->isa_oopptr();
2591 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2592 bool is_not_vect = (_type->isa_vect() == nullptr);
2593 if (is_not_header && is_not_vect) {
2594 Node* mem = in(MemNode::Memory);
2595 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2596 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2597 // TODO 8350865 Scalar replacement does not work well for flat arrays.
2598 // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2599 // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2600 // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2601 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2602 intptr_t offset = 0;
2603 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2604 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2605 if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2606 return _type;
2607 }
2608 }
2609 return Type::get_zero_type(_type->basic_type());
2610 }
2611 }
2612 if (!UseCompactObjectHeaders) {
2613 Node* alloc = is_new_object_mark_load();
2614 if (alloc != nullptr) {
2615 if (Arguments::is_valhalla_enabled()) {
2616 // The mark word may contain property bits (inline, flat, null-free)
2617 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2618 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2619 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2620 return TypeX::make(tkls->exact_klass()->prototype_header());
2621 }
2622 } else {
2623 return TypeX::make(markWord::prototype().value());
2624 }
2625 }
2626 }
2627
2628 return _type;
2629 }
2630
2631 //------------------------------match_edge-------------------------------------
2632 // Do we Match on this edge index or not? Match only the address.
2633 uint LoadNode::match_edge(uint idx) const {
2634 return idx == MemNode::Address;
2635 }
2636
2637 //--------------------------LoadBNode::Ideal--------------------------------------
2638 //
2639 // If the previous store is to the same address as this load,
2640 // and the value stored was larger than a byte, replace this load
2641 // with the value stored truncated to a byte. If no truncation is
2642 // needed, the replacement is done in LoadNode::Identity().
2643 //
2644 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2753 }
2754 }
2755 // Identity call will handle the case where truncation is not needed.
2756 return LoadNode::Ideal(phase, can_reshape);
2757 }
2758
2759 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2760 Node* mem = in(MemNode::Memory);
2761 Node* value = can_see_stored_value_through_membars(mem, phase);
2762 if (value != nullptr && value->is_Con() &&
2763 !value->bottom_type()->higher_equal(_type)) {
2764 // If the input to the store does not fit with the load's result type,
2765 // it must be truncated. We can't delay until Ideal call since
2766 // a singleton Value is needed for split_thru_phi optimization.
2767 int con = value->get_int();
2768 return TypeInt::make((con << 16) >> 16);
2769 }
2770 return LoadNode::Value(phase);
2771 }
2772
2773 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2774 // Can see the corresponding value, may need to add an EncodeP
2775 Node* value = can_see_stored_value_through_membars(in(Memory), phase);
2776 if (value != nullptr && phase->type(value)->isa_ptr() && type()->isa_narrowoop()) {
2777 return new EncodePNode(value, type());
2778 }
2779
2780 // Identity call will handle the case where EncodeP is unnecessary
2781 return LoadNode::Ideal(phase, can_reshape);
2782 }
2783
2784 //=============================================================================
2785 //----------------------------LoadKlassNode::make------------------------------
2786 // Polymorphic factory method:
2787 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2788 // sanity check the alias category against the created node type
2789 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2790 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2791 #ifdef _LP64
2792 if (adr_type->is_ptr_to_narrowklass()) {
2793 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2794 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2795 }
2796 #endif
2797 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2798 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2799 }
2800
2801 //------------------------------Value------------------------------------------
2802 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2803 return klass_value_common(phase);
2836 }
2837 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2838 }
2839 if (!t->is_klass()) {
2840 // a primitive Class (e.g., int.class) has null for a klass field
2841 return TypePtr::NULL_PTR;
2842 }
2843 // Fold up the load of the hidden field
2844 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2845 }
2846 // non-constant mirror, so we can't tell what's going on
2847 }
2848 if (!tinst->is_loaded())
2849 return _type; // Bail out if not loaded
2850 if (offset == oopDesc::klass_offset_in_bytes()) {
2851 return tinst->as_klass_type(true);
2852 }
2853 }
2854
2855 // Check for loading klass from an array
2856 const TypeAryPtr* tary = tp->isa_aryptr();
2857 if (tary != nullptr &&
2858 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2859 return tary->as_klass_type(true)->is_aryklassptr();
2860 }
2861
2862 // Check for loading klass from an array klass
2863 const TypeKlassPtr *tkls = tp->isa_klassptr();
2864 if (tkls != nullptr && !StressReflectiveCode) {
2865 if (!tkls->is_loaded())
2866 return _type; // Bail out if not loaded
2867 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2868 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2869 // // Always returning precise element type is incorrect,
2870 // // e.g., element type could be object and array may contain strings
2871 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2872
2873 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2874 // according to the element type's subclassing.
2875 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2876 }
2877 if (tkls->isa_aryklassptr() != nullptr && tkls->klass_is_exact() &&
2878 !tkls->exact_klass()->is_type_array_klass() &&
2879 tkls->offset() == in_bytes(Klass::super_offset())) {
2880 // We are loading the super klass of a refined array klass, return the non-refined klass pointer
2881 assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2882 return tkls->is_aryklassptr()->with_offset(0)->cast_to_non_refined();
2883 }
2884 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2885 tkls->offset() == in_bytes(Klass::super_offset())) {
2886 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2887 // The field is Klass::_super. Return its (constant) value.
2888 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2889 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2890 }
2891 }
2892
2893 if (tkls != nullptr && !UseSecondarySupersCache
2894 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2895 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2896 return TypePtr::NULL_PTR;
2897 }
2898
2899 // Bailout case
2900 return LoadNode::Value(phase);
2901 }
2902
2903 //------------------------------Identity---------------------------------------
2926 base = bs->step_over_gc_barrier(base);
2927 }
2928
2929 // We can fetch the klass directly through an AllocateNode.
2930 // This works even if the klass is not constant (clone or newArray).
2931 if (offset == oopDesc::klass_offset_in_bytes()) {
2932 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2933 if (allocated_klass != nullptr) {
2934 return allocated_klass;
2935 }
2936 }
2937
2938 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2939 // See inline_native_Class_query for occurrences of these patterns.
2940 // Java Example: x.getClass().isAssignableFrom(y)
2941 //
2942 // This improves reflective code, often making the Class
2943 // mirror go completely dead. (Current exception: Class
2944 // mirrors may appear in debug info, but we could clean them out by
2945 // introducing a new debug info operator for Klass.java_mirror).
2946 //
2947 // This optimization does not apply to arrays because if k is not a
2948 // constant, it was obtained via load_klass which returns the refined type
2949 // and '.java_mirror.as_klass' should return the Java type instead.
2950
2951 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2952 && offset == java_lang_Class::klass_offset()) {
2953 if (base->is_Load()) {
2954 Node* base2 = base->in(MemNode::Address);
2955 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2956 Node* adr2 = base2->in(MemNode::Address);
2957 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2958 if (tkls != nullptr && !tkls->empty()
2959 && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2960 && adr2->is_AddP()) {
2961 int mirror_field = in_bytes(Klass::java_mirror_offset());
2962 if (tkls->offset() == mirror_field) {
2963 #ifdef ASSERT
2964 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2965 assert(tkls2->offset() == 0, "not a load of java_mirror");
2966 #endif
2967 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2968 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2969 return adr2->in(AddPNode::Address);
2970 }
2971 }
2972 }
2973 }
2974 }
2975
2976 return this;
2977 }
2978
2979 LoadNode* LoadNode::clone_pinned() const {
2980 LoadNode* ld = clone()->as_Load();
3107 // Polymorphic factory method:
3108 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
3109 assert((mo == unordered || mo == release), "unexpected");
3110 Compile* C = gvn.C;
3111 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
3112 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
3113 ctl != nullptr, "raw memory operations should have control edge");
3114
3115 switch (bt) {
3116 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
3117 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
3118 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
3119 case T_CHAR:
3120 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
3121 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3122 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
3123 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3124 case T_METADATA:
3125 case T_ADDRESS:
3126 case T_OBJECT:
3127 case T_ARRAY:
3128 #ifdef _LP64
3129 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3130 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
3131 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
3132 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
3133 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
3134 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
3135 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
3136 }
3137 #endif
3138 {
3139 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
3140 }
3141 default:
3142 guarantee(false, "unexpected basic type %s", type2name(bt));
3143 return (StoreNode*)nullptr;
3144 }
3145 }
3146
3147 //--------------------------bottom_type----------------------------------------
3148 const Type *StoreNode::bottom_type() const {
3149 return Type::MEMORY;
3150 }
3151
3152 //------------------------------hash-------------------------------------------
3153 uint StoreNode::hash() const {
3154 // unroll addition of interesting fields
3155 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
3156
3157 // Since they are not commoned, do not hash them:
3158 return NO_HASH;
3159 }
3160
3161 // Link together multiple stores (B/S/C/I) into a longer one.
3162 //
3784 }
3785 ss.print_cr("[TraceMergeStores]: with");
3786 merged_input_value->dump("\n", false, &ss);
3787 merged_store->dump("\n", false, &ss);
3788 tty->print("%s", ss.as_string());
3789 }
3790 #endif
3791
3792 //------------------------------Ideal------------------------------------------
3793 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3794 // When a store immediately follows a relevant allocation/initialization,
3795 // try to capture it into the initialization, or hoist it above.
3796 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3797 Node* p = MemNode::Ideal_common(phase, can_reshape);
3798 if (p) return (p == NodeSentinel) ? nullptr : p;
3799
3800 Node* mem = in(MemNode::Memory);
3801 Node* address = in(MemNode::Address);
3802 Node* value = in(MemNode::ValueIn);
3803 // Back-to-back stores to same address? Fold em up. Generally
3804 // unsafe if I have intervening uses...
3805 if ((!this->is_StoreVector() || this->Opcode() == Op_StoreVector) &&
3806 phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3807 Node* st = mem;
3808 // If Store 'st' has more than one use, we cannot fold 'st' away.
3809 // For example, 'st' might be the final state at a conditional
3810 // return. Or, 'st' might be used by some node which is live at
3811 // the same time 'st' is live, which might be unschedulable. So,
3812 // require exactly ONE user until such time as we clone 'mem' for
3813 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3814 // true). Further, 'st' must be a contiguous store, otherwise
3815 // memory_size does not make sense for measuring overlap.
3816 while (st->is_Store() && st->outcnt() == 1 && (!st->is_StoreVector() || st->Opcode() == Op_StoreVector)) {
3817 // Looking at a dead closed cycle of memory?
3818 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3819 assert(Opcode() == st->Opcode() ||
3820 st->Opcode() == Op_StoreVector ||
3821 Opcode() == Op_StoreVector ||
3822 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3823 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3824 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3825 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3826 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3827 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3828
3829 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3830 st->as_Store()->memory_size() <= this->memory_size()) {
3831 assert(!is_predicated_vector() && !is_StoreVectorMasked() &&
3832 !is_StoreVectorScatter() && !is_StoreVectorScatterMasked() &&
3833 !st->is_predicated_vector() && !st->is_StoreVectorMasked() &&
3834 !st->is_StoreVectorScatter() && !st->is_StoreVectorScatterMasked(),
3835 "optimization only correct for full-width stores without holes");
3836 Node* use = st->raw_out(0);
3837 if (phase->is_IterGVN()) {
3838 phase->is_IterGVN()->rehash_node_delayed(use);
3839 }
3840 // It's OK to do this in the parser, since DU info is always accurate,
3841 // and the parser always refers to nodes via SafePointNode maps.
3842 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3843 return this;
3844 }
3845 st = st->in(MemNode::Memory);
3951 const StoreVectorNode* store_vector = as_StoreVector();
3952 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3953 const Node* store_indices = store_vector->indices();
3954 const Node* mem_indices = mem_vector->indices();
3955 const Node* store_mask = store_vector->mask();
3956 const Node* mem_mask = mem_vector->mask();
3957 // Ensure types, indices, and masks match
3958 if (store_vector->vect_type() == mem_vector->vect_type() &&
3959 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3960 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3961 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3962 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3963 result = mem;
3964 }
3965 }
3966 }
3967
3968 // Store of zero anywhere into a freshly-allocated object?
3969 // Then the store is useless.
3970 // (It must already have been captured by the InitializeNode.)
3971 if (result == this && ReduceFieldZeroing) {
3972 // a newly allocated object is already all-zeroes everywhere
3973 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3974 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3975 result = mem;
3976 }
3977
3978 if (result == this && phase->type(val)->is_zero_type()) {
3979 // the store may also apply to zero-bits in an earlier object
3980 Node* prev_mem = find_previous_store(phase);
3981 // Steps (a), (b): Walk past independent stores to find an exact match.
3982 if (prev_mem != nullptr) {
3983 if (prev_mem->is_top()) {
3984 // find_previous_store returns top when the access is dead
3985 return prev_mem;
3986 }
3987 Node* prev_val = can_see_stored_value(prev_mem, phase);
3988 if (prev_val != nullptr && prev_val == val) {
3989 // prev_val and val might differ by a cast; it would be good
3990 // to keep the more informative of the two.
3991 result = mem;
3992 }
3993 }
3994 }
3995 }
3996
3997 PhaseIterGVN* igvn = phase->is_IterGVN();
3998 if (result != this && igvn != nullptr) {
4491 // Clearing a short array is faster with stores
4492 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4493 // Already know this is a large node, do not try to ideal it
4494 if (_is_large) return nullptr;
4495
4496 const int unit = BytesPerLong;
4497 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4498 if (!t) return nullptr;
4499 if (!t->is_con()) return nullptr;
4500 intptr_t raw_count = t->get_con();
4501 intptr_t size = raw_count;
4502 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4503 // Clearing nothing uses the Identity call.
4504 // Negative clears are possible on dead ClearArrays
4505 // (see jck test stmt114.stmt11402.val).
4506 if (size <= 0 || size % unit != 0) return nullptr;
4507 intptr_t count = size / unit;
4508 // Length too long; communicate this to matchers and assemblers.
4509 // Assemblers are responsible to produce fast hardware clears for it.
4510 if (size > InitArrayShortSize) {
4511 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4512 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4513 return nullptr;
4514 }
4515 if (!IdealizeClearArrayNode) return nullptr;
4516 Node *mem = in(1);
4517 if( phase->type(mem)==Type::TOP ) return nullptr;
4518 Node *adr = in(3);
4519 const Type* at = phase->type(adr);
4520 if( at==Type::TOP ) return nullptr;
4521 const TypePtr* atp = at->isa_ptr();
4522 // adjust atp to be the correct array element address type
4523 if (atp == nullptr) atp = TypePtr::BOTTOM;
4524 else atp = atp->add_offset(Type::OffsetBot);
4525 // Get base for derived pointer purposes
4526 if( adr->Opcode() != Op_AddP ) Unimplemented();
4527 Node *base = adr->in(1);
4528
4529 Node *val = in(4);
4530 Node *off = phase->MakeConX(BytesPerLong);
4531 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4532 count--;
4533 while (count--) {
4534 mem = phase->transform(mem);
4535 adr = phase->transform(AddPNode::make_with_base(base,adr,off));
4536 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4537 }
4538 return mem;
4539 }
4540
4541 //----------------------------step_through----------------------------------
4542 // Return allocation input memory edge if it is different instance
4543 // or itself if it is the one we are looking for.
4544 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4545 Node* n = *np;
4546 assert(n->is_ClearArray(), "sanity");
4547 intptr_t offset;
4548 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4549 // This method is called only before Allocate nodes are expanded
4550 // during macro nodes expansion. Before that ClearArray nodes are
4551 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4552 // Allocate nodes are expanded) which follows allocations.
4553 assert(alloc != nullptr, "should have allocation");
4554 if (alloc->_idx == instance_id) {
4555 // Can not bypass initialization of the instance we are looking for.
4556 return false;
4559 InitializeNode* init = alloc->initialization();
4560 if (init != nullptr)
4561 *np = init->in(TypeFunc::Memory);
4562 else
4563 *np = alloc->in(TypeFunc::Memory);
4564 return true;
4565 }
4566
4567 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4568 Node* base = dest;
4569 if (raw_base) {
4570 // May be called as part of the initialization of a just allocated object
4571 base = phase->C->top();
4572 }
4573 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4574 }
4575
4576 //----------------------------clear_memory-------------------------------------
4577 // Generate code to initialize object storage to zero.
4578 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4579 Node* val,
4580 Node* raw_val,
4581 intptr_t start_offset,
4582 Node* end_offset,
4583 bool raw_base,
4584 PhaseGVN* phase) {
4585 intptr_t offset = start_offset;
4586
4587 int unit = BytesPerLong;
4588 if ((offset % unit) != 0) {
4589 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4590 const TypePtr* atp = TypeRawPtr::BOTTOM;
4591 if (val != nullptr) {
4592 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4593 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4594 } else {
4595 assert(raw_val == nullptr, "val may not be null");
4596 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4597 }
4598 mem = phase->transform(mem);
4599 offset += BytesPerInt;
4600 }
4601 assert((offset % unit) == 0, "");
4602
4603 // Initialize the remaining stuff, if any, with a ClearArray.
4604 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, raw_base, phase);
4605 }
4606
4607 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4608 Node* raw_val,
4609 Node* start_offset,
4610 Node* end_offset,
4611 bool raw_base,
4612 PhaseGVN* phase) {
4613 if (start_offset == end_offset) {
4614 // nothing to do
4615 return mem;
4616 }
4617
4618 int unit = BytesPerLong;
4619 Node* zbase = start_offset;
4620 Node* zend = end_offset;
4621
4622 // Scale to the unit required by the CPU:
4623 if (!Matcher::init_array_count_is_in_bytes) {
4624 Node* shift = phase->intcon(exact_log2(unit));
4625 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4626 zend = phase->transform(new URShiftXNode(zend, shift) );
4627 }
4628
4629 // Bulk clear double-words
4630 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4631 Node* adr = make_address(dest, start_offset, raw_base, phase);
4632 if (raw_val == nullptr) {
4633 raw_val = phase->MakeConX(0);
4634 }
4635 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4636 return phase->transform(mem);
4637 }
4638
4639 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4640 Node* val,
4641 Node* raw_val,
4642 intptr_t start_offset,
4643 intptr_t end_offset,
4644 bool raw_base,
4645 PhaseGVN* phase) {
4646 if (start_offset == end_offset) {
4647 // nothing to do
4648 return mem;
4649 }
4650
4651 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4652 intptr_t done_offset = end_offset;
4653 if ((done_offset % BytesPerLong) != 0) {
4654 done_offset -= BytesPerInt;
4655 }
4656 if (done_offset > start_offset) {
4657 mem = clear_memory(ctl, mem, dest, val, raw_val,
4658 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4659 }
4660 if (done_offset < end_offset) { // emit the final 32-bit store
4661 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4662 const TypePtr* atp = TypeRawPtr::BOTTOM;
4663 if (val != nullptr) {
4664 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4665 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4666 } else {
4667 assert(raw_val == nullptr, "val may not be null");
4668 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4669 }
4670 mem = phase->transform(mem);
4671 done_offset += BytesPerInt;
4672 }
4673 assert(done_offset == end_offset, "");
4674 return mem;
4675 }
4676
4677 //=============================================================================
4678 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4679 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4680 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4681 #ifdef ASSERT
4682 , _pair_idx(0)
4683 #endif
4684 {
4685 init_class_id(Class_MemBar);
4686 Node* top = C->top();
4687 init_req(TypeFunc::I_O,top);
4688 init_req(TypeFunc::FramePtr,top);
4689 init_req(TypeFunc::ReturnAdr,top);
4798 PhaseIterGVN* igvn = phase->is_IterGVN();
4799 remove(igvn);
4800 // Must return either the original node (now dead) or a new node
4801 // (Do not return a top here, since that would break the uniqueness of top.)
4802 return new ConINode(TypeInt::ZERO);
4803 }
4804 }
4805 return progress ? this : nullptr;
4806 }
4807
4808 //------------------------------Value------------------------------------------
4809 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4810 if( !in(0) ) return Type::TOP;
4811 if( phase->type(in(0)) == Type::TOP )
4812 return Type::TOP;
4813 return TypeTuple::MEMBAR;
4814 }
4815
4816 //------------------------------match------------------------------------------
4817 // Construct projections for memory.
4818 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4819 switch (proj->_con) {
4820 case TypeFunc::Control:
4821 case TypeFunc::Memory:
4822 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4823 }
4824 ShouldNotReachHere();
4825 return nullptr;
4826 }
4827
4828 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4829 trailing->_kind = TrailingStore;
4830 leading->_kind = LeadingStore;
4831 #ifdef ASSERT
4832 trailing->_pair_idx = leading->_idx;
4833 leading->_pair_idx = leading->_idx;
4834 #endif
4835 }
4836
4837 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4838 trailing->_kind = TrailingLoadStore;
5085 return (req() > RawStores);
5086 }
5087
5088 void InitializeNode::set_complete(PhaseGVN* phase) {
5089 assert(!is_complete(), "caller responsibility");
5090 _is_complete = Complete;
5091
5092 // After this node is complete, it contains a bunch of
5093 // raw-memory initializations. There is no need for
5094 // it to have anything to do with non-raw memory effects.
5095 // Therefore, tell all non-raw users to re-optimize themselves,
5096 // after skipping the memory effects of this initialization.
5097 PhaseIterGVN* igvn = phase->is_IterGVN();
5098 if (igvn) igvn->add_users_to_worklist(this);
5099 }
5100
5101 // convenience function
5102 // return false if the init contains any stores already
5103 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
5104 InitializeNode* init = initialization();
5105 if (init == nullptr || init->is_complete()) {
5106 return false;
5107 }
5108 init->remove_extra_zeroes();
5109 // for now, if this allocation has already collected any inits, bail:
5110 if (init->is_non_zero()) return false;
5111 init->set_complete(phase);
5112 return true;
5113 }
5114
5115 void InitializeNode::remove_extra_zeroes() {
5116 if (req() == RawStores) return;
5117 Node* zmem = zero_memory();
5118 uint fill = RawStores;
5119 for (uint i = fill; i < req(); i++) {
5120 Node* n = in(i);
5121 if (n->is_top() || n == zmem) continue; // skip
5122 if (fill < i) set_req(fill, n); // compact
5123 ++fill;
5124 }
5125 // delete any empty spaces created:
5126 while (fill < req()) {
5127 del_req(fill);
5271 // store node that we'd like to capture. We need to check
5272 // the uses of the MergeMemNode.
5273 mems.push(n);
5274 }
5275 } else if (n->is_Mem()) {
5276 Node* other_adr = n->in(MemNode::Address);
5277 if (other_adr == adr) {
5278 failed = true;
5279 break;
5280 } else {
5281 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5282 if (other_t_adr != nullptr) {
5283 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5284 if (other_alias_idx == alias_idx) {
5285 // A load from the same memory slice as the store right
5286 // after the InitializeNode. We check the control of the
5287 // object/array that is loaded from. If it's the same as
5288 // the store control then we cannot capture the store.
5289 assert(!n->is_Store(), "2 stores to same slice on same control?");
5290 Node* base = other_adr;
5291 if (base->is_Phi()) {
5292 // In rare case, base may be a PhiNode and it may read
5293 // the same memory slice between InitializeNode and store.
5294 failed = true;
5295 break;
5296 }
5297 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5298 base = base->in(AddPNode::Base);
5299 if (base != nullptr) {
5300 base = base->uncast();
5301 if (base->is_Proj() && base->in(0) == alloc) {
5302 failed = true;
5303 break;
5304 }
5305 }
5306 }
5307 }
5308 }
5309 } else {
5310 failed = true;
5311 break;
5312 }
5313 }
5314 }
5315 }
5316 if (failed) {
5862 // z's_done 12 16 16 16 12 16 12
5863 // z's_needed 12 16 16 16 16 16 16
5864 // zsize 0 0 0 0 4 0 4
5865 if (next_full_store < 0) {
5866 // Conservative tack: Zero to end of current word.
5867 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5868 } else {
5869 // Zero to beginning of next fully initialized word.
5870 // Or, don't zero at all, if we are already in that word.
5871 assert(next_full_store >= zeroes_needed, "must go forward");
5872 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5873 zeroes_needed = next_full_store;
5874 }
5875 }
5876
5877 if (zeroes_needed > zeroes_done) {
5878 intptr_t zsize = zeroes_needed - zeroes_done;
5879 // Do some incremental zeroing on rawmem, in parallel with inits.
5880 zeroes_done = align_down(zeroes_done, BytesPerInt);
5881 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5882 allocation()->in(AllocateNode::InitValue),
5883 allocation()->in(AllocateNode::RawInitValue),
5884 zeroes_done, zeroes_needed,
5885 true,
5886 phase);
5887 zeroes_done = zeroes_needed;
5888 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5889 do_zeroing = false; // leave the hole, next time
5890 }
5891 }
5892
5893 // Collect the store and move on:
5894 phase->replace_input_of(st, MemNode::Memory, inits);
5895 inits = st; // put it on the linearized chain
5896 set_req(i, zmem); // unhook from previous position
5897
5898 if (zeroes_done == st_off)
5899 zeroes_done = next_init_off;
5900
5901 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5902
5903 #ifdef ASSERT
5924 remove_extra_zeroes(); // clear out all the zmems left over
5925 add_req(inits);
5926
5927 if (!(UseTLAB && ZeroTLAB)) {
5928 // If anything remains to be zeroed, zero it all now.
5929 zeroes_done = align_down(zeroes_done, BytesPerInt);
5930 // if it is the last unused 4 bytes of an instance, forget about it
5931 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5932 if (zeroes_done + BytesPerLong >= size_limit) {
5933 AllocateNode* alloc = allocation();
5934 assert(alloc != nullptr, "must be present");
5935 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5936 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5937 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5938 if (zeroes_done == k->layout_helper())
5939 zeroes_done = size_limit;
5940 }
5941 }
5942 if (zeroes_done < size_limit) {
5943 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5944 allocation()->in(AllocateNode::InitValue),
5945 allocation()->in(AllocateNode::RawInitValue),
5946 zeroes_done, size_in_bytes, true, phase);
5947 }
5948 }
5949
5950 set_complete(phase);
5951 return rawmem;
5952 }
5953
5954 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5955 auto replace_proj = [&](ProjNode* proj) {
5956 C->gvn_replace_by(proj, mem);
5957 return CONTINUE;
5958 };
5959 apply_to_projs(replace_proj, TypeFunc::Memory);
5960 }
5961
5962 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5963 DUIterator_Fast imax, i = fast_outs(imax);
5964 auto replace_proj = [&](ProjNode* proj) {
5965 igvn->replace_node(proj, mem);
6163 //------------------------------Identity---------------------------------------
6164 Node* MergeMemNode::Identity(PhaseGVN* phase) {
6165 // Identity if this merge point does not record any interesting memory
6166 // disambiguations.
6167 Node* base_mem = base_memory();
6168 Node* empty_mem = empty_memory();
6169 if (base_mem != empty_mem) { // Memory path is not dead?
6170 for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
6171 Node* mem = in(i);
6172 if (mem != empty_mem && mem != base_mem) {
6173 return this; // Many memory splits; no change
6174 }
6175 }
6176 }
6177 return base_mem; // No memory splits; ID on the one true input
6178 }
6179
6180 //------------------------------Ideal------------------------------------------
6181 // This method is invoked recursively on chains of MergeMem nodes
6182 Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) {
6183 if (Identity(phase) != this) {
6184 // Let Identity handle this case
6185 return nullptr;
6186 }
6187
6188 // Remove chain'd MergeMems
6189 //
6190 // This is delicate, because the each "in(i)" (i >= Raw) is interpreted
6191 // relative to the "in(Bot)". Since we are patching both at the same time,
6192 // we have to be careful to read each "in(i)" relative to the old "in(Bot)",
6193 // but rewrite each "in(i)" relative to the new "in(Bot)".
6194 Node *progress = nullptr;
6195
6196
6197 Node* old_base = base_memory();
6198 Node* empty_mem = empty_memory();
6199 if (old_base == empty_mem)
6200 return nullptr; // Dead memory path.
6201
6202 MergeMemNode* old_mbase;
6203 if (old_base != nullptr && old_base->is_MergeMem())
6204 old_mbase = old_base->as_MergeMem();
6205 else
6206 old_mbase = nullptr;
6207 Node* new_base = old_base;
|