6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/compile.hpp"
38 #include "opto/connode.hpp"
39 #include "opto/convertnode.hpp"
40 #include "opto/loopnode.hpp"
41 #include "opto/machnode.hpp"
42 #include "opto/matcher.hpp"
43 #include "opto/memnode.hpp"
44 #include "opto/mempointer.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/opcodes.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regalloc.hpp"
50 #include "opto/regmask.hpp"
51 #include "opto/rootnode.hpp"
52 #include "opto/traceMergeStoresTag.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/globalDefinitions.hpp"
57 #include "utilities/macros.hpp"
58 #include "utilities/powerOfTwo.hpp"
59 #include "utilities/vmError.hpp"
60
61 // Portions of code courtesy of Clifford Click
62
63 // Optimization - Graph Style
64
65 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
66
67 //=============================================================================
68 uint MemNode::size_of() const { return sizeof(*this); }
69
70 const TypePtr *MemNode::adr_type() const {
71 Node* adr = in(Address);
72 if (adr == nullptr) return nullptr; // node is dead
73 const TypePtr* cross_check = nullptr;
125 st->print(", idx=Bot;");
126 else if (atp->index() == Compile::AliasIdxTop)
127 st->print(", idx=Top;");
128 else if (atp->index() == Compile::AliasIdxRaw)
129 st->print(", idx=Raw;");
130 else {
131 ciField* field = atp->field();
132 if (field) {
133 st->print(", name=");
134 field->print_name_on(st);
135 }
136 st->print(", idx=%d;", atp->index());
137 }
138 }
139 }
140
141 extern void print_alias_types();
142
143 #endif
144
145 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
146 assert((t_oop != nullptr), "sanity");
147 bool is_instance = t_oop->is_known_instance_field();
148 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
149 (load != nullptr) && load->is_Load() &&
150 (phase->is_IterGVN() != nullptr);
151 if (!(is_instance || is_boxed_value_load))
152 return mchain; // don't try to optimize non-instance types
153 uint instance_id = t_oop->instance_id();
154 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
155 Node *prev = nullptr;
156 Node *result = mchain;
157 while (prev != result) {
158 prev = result;
159 if (result == start_mem)
160 break; // hit one of our sentinels
161 // skip over a call which does not affect this memory slice
162 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
163 Node *proj_in = result->in(0);
164 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
165 break; // hit one of our sentinels
166 } else if (proj_in->is_Call()) {
167 // ArrayCopyNodes processed here as well
168 CallNode *call = proj_in->as_Call();
169 if (!call->may_modify(t_oop, phase)) { // returns false for instances
170 result = call->in(TypeFunc::Memory);
171 }
172 } else if (proj_in->is_Initialize()) {
173 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
174 // Stop if this is the initialization for the object instance which
175 // contains this memory slice, otherwise skip over it.
176 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
177 break;
178 }
179 if (is_instance) {
180 result = proj_in->in(TypeFunc::Memory);
181 } else if (is_boxed_value_load) {
182 Node* klass = alloc->in(AllocateNode::KlassNode);
183 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
184 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
185 result = proj_in->in(TypeFunc::Memory); // not related allocation
186 }
187 }
188 } else if (proj_in->is_MemBar()) {
189 ArrayCopyNode* ac = nullptr;
190 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
191 break;
192 }
193 result = proj_in->in(TypeFunc::Memory);
194 } else if (proj_in->is_top()) {
195 break; // dead code
196 } else {
197 assert(false, "unexpected projection");
198 }
199 } else if (result->is_ClearArray()) {
200 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
201 // Can not bypass initialization of the instance
202 // we are looking for.
203 break;
204 }
205 // Otherwise skip it (the call updated 'result' value).
206 } else if (result->is_MergeMem()) {
207 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
208 }
209 }
210 return result;
211 }
212
213 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
214 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
215 if (t_oop == nullptr)
216 return mchain; // don't try to optimize non-oop types
217 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
218 bool is_instance = t_oop->is_known_instance_field();
219 PhaseIterGVN *igvn = phase->is_IterGVN();
220 if (is_instance && igvn != nullptr && result->is_Phi()) {
221 PhiNode *mphi = result->as_Phi();
222 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
223 const TypePtr *t = mphi->adr_type();
224 bool do_split = false;
225 // In the following cases, Load memory input can be further optimized based on
226 // its precise address type
227 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
228 do_split = true;
229 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
230 const TypeOopPtr* mem_t =
231 t->is_oopptr()->cast_to_exactness(true)
232 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
233 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
234 if (t_oop->isa_aryptr()) {
235 mem_t = mem_t->is_aryptr()
236 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
237 ->cast_to_size(t_oop->is_aryptr()->size())
238 ->with_offset(t_oop->is_aryptr()->offset())
239 ->is_aryptr();
240 }
241 do_split = mem_t == t_oop;
242 }
243 if (do_split) {
244 // clone the Phi with our address type
245 result = mphi->split_out_instance(t_adr, igvn);
246 } else {
247 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
248 }
249 }
250 return result;
251 }
252
253 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
254 uint alias_idx = phase->C->get_alias_index(tp);
255 Node *mem = mmem;
256 #ifdef ASSERT
257 {
258 // Check that current type is consistent with the alias index used during graph construction
259 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
260 bool consistent = adr_check == nullptr || adr_check->empty() ||
261 phase->C->must_alias(adr_check, alias_idx );
262 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
263 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
264 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
265 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
266 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
267 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
268 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
269 // don't assert if it is dead code.
270 consistent = true;
271 }
272 if( !consistent ) {
273 st->print("alias_idx==%d, adr_check==", alias_idx);
274 if( adr_check == nullptr ) {
275 st->print("null");
276 } else {
277 adr_check->dump();
278 }
279 st->cr();
280 print_alias_types();
281 assert(consistent, "adr_check must match alias idx");
282 }
283 }
284 #endif
587 }
588
589 // Find an arraycopy ac that produces the memory state represented by parameter mem.
590 // Return ac if
591 // (a) can_see_stored_value=true and ac must have set the value for this load or if
592 // (b) can_see_stored_value=false and ac could have set the value for this load or if
593 // (c) can_see_stored_value=false and ac cannot have set the value for this load.
594 // In case (c) change the parameter mem to the memory input of ac to skip it
595 // when searching stored value.
596 // Otherwise return null.
597 Node* LoadNode::find_previous_arraycopy(PhaseValues* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const {
598 ArrayCopyNode* ac = find_array_copy_clone(ld_alloc, mem);
599 if (ac != nullptr) {
600 Node* ld_addp = in(MemNode::Address);
601 Node* src = ac->in(ArrayCopyNode::Src);
602 const TypeAryPtr* ary_t = phase->type(src)->isa_aryptr();
603
604 // This is a load from a cloned array. The corresponding arraycopy ac must
605 // have set the value for the load and we can return ac but only if the load
606 // is known to be within bounds. This is checked below.
607 if (ary_t != nullptr && ld_addp->is_AddP()) {
608 Node* ld_offs = ld_addp->in(AddPNode::Offset);
609 BasicType ary_elem = ary_t->elem()->array_element_basic_type();
610 jlong header = arrayOopDesc::base_offset_in_bytes(ary_elem);
611 jlong elemsize = type2aelembytes(ary_elem);
612
613 const TypeX* ld_offs_t = phase->type(ld_offs)->isa_intptr_t();
614 const TypeInt* sizetype = ary_t->size();
615
616 if (ld_offs_t->_lo >= header && ld_offs_t->_hi < (sizetype->_lo * elemsize + header)) {
617 // The load is known to be within bounds. It receives its value from ac.
618 return ac;
619 }
620 // The load is known to be out-of-bounds.
621 }
622 // The load could be out-of-bounds. It must not be hoisted but must remain
623 // dependent on the runtime range check. This is achieved by returning null.
624 } else if (mem->is_Proj() && mem->in(0) != nullptr && mem->in(0)->is_ArrayCopy()) {
625 ArrayCopyNode* ac = mem->in(0)->as_ArrayCopy();
626
627 if (ac->is_arraycopy_validated() ||
997 in_bytes(JavaThread::vthread_offset()),
998 in_bytes(JavaThread::scopedValueCache_offset()),
999 };
1000
1001 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1002 if (offset == offsets[i]) {
1003 return true;
1004 }
1005 }
1006 }
1007
1008 return false;
1009 }
1010 #endif
1011
1012 //----------------------------LoadNode::make-----------------------------------
1013 // Polymorphic factory method:
1014 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1015 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1016 Compile* C = gvn.C;
1017 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
1018
1019 // sanity check the alias category against the created node type
1020 assert(!(adr_type->isa_oopptr() &&
1021 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1022 "use LoadKlassNode instead");
1023 assert(!(adr_type->isa_aryptr() &&
1024 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1025 "use LoadRangeNode instead");
1026 // Check control edge of raw loads
1027 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1028 // oop will be recorded in oop map if load crosses safepoint
1029 rt->isa_oopptr() || is_immutable_value(adr),
1030 "raw memory operations should have control edge");
1031 LoadNode* load = nullptr;
1032 switch (bt) {
1033 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1034 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1035 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1036 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1037 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1038 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1039 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1040 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1041 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1042 case T_OBJECT:
1043 case T_NARROWOOP:
1044 #ifdef _LP64
1045 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1046 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1047 } else
1048 #endif
1049 {
1050 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1051 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1052 }
1053 break;
1054 default:
1055 ShouldNotReachHere();
1056 break;
1057 }
1058 assert(load != nullptr, "LoadNode should have been created");
1059 if (unaligned) {
1060 load->set_unaligned_access();
1061 }
1062 if (mismatched) {
1063 load->set_mismatched_access();
1064 }
1065 if (unsafe) {
1066 load->set_unsafe_access();
1067 }
1068 load->set_barrier_data(barrier_data);
1069 if (load->Opcode() == Op_LoadN) {
1070 Node* ld = gvn.transform(load);
1071 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1072 }
1073
1074 return load;
1075 }
1076
1077 //------------------------------hash-------------------------------------------
1078 uint LoadNode::hash() const {
1079 // unroll addition of interesting fields
1080 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1081 }
1082
1083 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1084 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1085 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1086 bool is_stable_ary = FoldStableValues &&
1087 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1088 tp->isa_aryptr()->is_stable();
1089
1090 return (eliminate_boxing && non_volatile) || is_stable_ary;
1091 }
1092
1093 return false;
1094 }
1095
1096 // Is the value loaded previously stored by an arraycopy? If so return
1097 // a load node that reads from the source array so we may be able to
1098 // optimize out the ArrayCopy node later.
1099 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1100 Node* ld_adr = in(MemNode::Address);
1101 intptr_t ld_off = 0;
1102 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1103 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1104 if (ac != nullptr) {
1105 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1106
1107 Node* mem = ac->in(TypeFunc::Memory);
1108 Node* ctl = ac->in(0);
1109 Node* src = ac->in(ArrayCopyNode::Src);
1110
1118 if (ac->as_ArrayCopy()->is_clonebasic()) {
1119 assert(ld_alloc != nullptr, "need an alloc");
1120 assert(addp->is_AddP(), "address must be addp");
1121 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1122 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1123 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1124 addp->set_req(AddPNode::Base, src);
1125 addp->set_req(AddPNode::Address, src);
1126 } else {
1127 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1128 ac->as_ArrayCopy()->is_copyof_validated() ||
1129 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1130 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1131 addp->set_req(AddPNode::Base, src);
1132 addp->set_req(AddPNode::Address, src);
1133
1134 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1135 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1136 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1137
1138 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1139 uint shift = exact_log2(type2aelembytes(ary_elem));
1140
1141 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1142 #ifdef _LP64
1143 diff = phase->transform(new ConvI2LNode(diff));
1144 #endif
1145 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1146
1147 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1148 addp->set_req(AddPNode::Offset, offset);
1149 }
1150 addp = phase->transform(addp);
1151 #ifdef ASSERT
1152 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1153 ld->_adr_type = adr_type;
1154 #endif
1155 ld->set_req(MemNode::Address, addp);
1156 ld->set_req(0, ctl);
1157 ld->set_req(MemNode::Memory, mem);
1158 return ld;
1159 }
1160 return nullptr;
1161 }
1162
1163 // This routine exists to make sure this set of tests is done the same
1164 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1165 // will change the graph shape in a way which makes memory alive twice at the
1166 // same time (uses the Oracle model of aliasing), then some
1167 // LoadXNode::Identity will fold things back to the equivalence-class model
1168 // of aliasing.
1169 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1170 Node* ld_adr = in(MemNode::Address);
1171 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1172 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1173
1174 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1175 uint alias_idx = atp->index();
1176 Node* result = nullptr;
1177 Node* current = st;
1178 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1179 // this load. Stop once any other kind of node is encountered.
1180 //
1181 // In principle, folding a load is moving it up until it meets a matching store.
1182 //
1183 // store(ptr, v); store(ptr, v); store(ptr, v);
1184 // membar1; -> membar1; -> load(ptr);
1185 // membar2; load(ptr); membar1;
1186 // load(ptr); membar2; membar2;
1187 //
1188 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1189 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1190 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1202 MergeMemNode* merge = mem->as_MergeMem();
1203 Node* new_st = merge->memory_at(alias_idx);
1204 if (new_st == merge->base_memory()) {
1205 // Keep searching
1206 current = new_st;
1207 continue;
1208 }
1209 // Save the new memory state for the slice and fall through
1210 // to exit.
1211 result = new_st;
1212 }
1213 }
1214 break;
1215 }
1216 if (result != nullptr) {
1217 st = result;
1218 }
1219 }
1220
1221 Node* res = can_see_stored_value(st, phase);
1222 assert(res == nullptr || is_java_primitive(value_basic_type()) || res->bottom_type()->higher_equal(type()), "the fold is unsafe");
1223 return res;
1224 }
1225
1226 // If st is a store to the same location as this, return the stored value
1227 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1228 Node* ld_adr = in(MemNode::Address);
1229 intptr_t ld_off = 0;
1230 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1231 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1232 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1233
1234 // Loop around twice in the case Load -> Initialize -> Store.
1235 // (See PhaseIterGVN::add_users_to_worklist, which knows about this case.)
1236 for (int trip = 0; trip <= 1; trip++) {
1237
1238 if (st->is_Store()) {
1239 Node* st_adr = st->in(MemNode::Address);
1240 if (st_adr != ld_adr) {
1241 // Try harder before giving up. Unify base pointers with casts (e.g., raw/non-raw pointers).
1242 intptr_t st_off = 0;
1287 if (is_Store() || is_java_primitive(value_basic_type()) || res->bottom_type()->higher_equal(bottom_type())) {
1288 return res;
1289 }
1290
1291 // Type-unsafe stores must be due to array polymorphism
1292 const TypePtr* adr_type = this->adr_type();
1293 assert(adr_type == nullptr || adr_type->isa_aryptr() != nullptr, "unexpected type-unsafe store");
1294 return nullptr;
1295 }
1296
1297 // A load from a freshly-created object always returns zero.
1298 // (This can happen after LoadNode::Ideal resets the load's memory input
1299 // to find_captured_store, which returned InitializeNode::zero_memory.)
1300 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1301 (st->in(0) == ld_alloc) &&
1302 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1303 // return a zero value for the load's basic type
1304 // (This is one of the few places where a generic PhaseTransform
1305 // can create new nodes. Think of it as lazily manifesting
1306 // virtually pre-existing constants.)
1307 if (value_basic_type() != T_VOID) {
1308 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1309 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1310 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1311 // by the ArrayCopyNode.
1312 return phase->zerocon(value_basic_type());
1313 }
1314 } else {
1315 // TODO: materialize all-zero vector constant
1316 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1317 }
1318 }
1319
1320 // A load from an initialization barrier can match a captured store.
1321 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1322 InitializeNode* init = st->in(0)->as_Initialize();
1323 AllocateNode* alloc = init->allocation();
1324 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1325 // examine a captured store value
1326 st = init->find_captured_store(ld_off, memory_size(), phase);
1339 base = bs->step_over_gc_barrier(base);
1340 if (base != nullptr && base->is_Proj() &&
1341 base->as_Proj()->_con == TypeFunc::Parms &&
1342 base->in(0)->is_CallStaticJava() &&
1343 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1344 return base->in(0)->in(TypeFunc::Parms);
1345 }
1346 }
1347
1348 break;
1349 }
1350
1351 return nullptr;
1352 }
1353
1354 //----------------------is_instance_field_load_with_local_phi------------------
1355 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1356 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1357 in(Address)->is_AddP() ) {
1358 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1359 // Only instances and boxed values.
1360 if( t_oop != nullptr &&
1361 (t_oop->is_ptr_to_boxed_value() ||
1362 t_oop->is_known_instance_field()) &&
1363 t_oop->offset() != Type::OffsetBot &&
1364 t_oop->offset() != Type::OffsetTop) {
1365 return true;
1366 }
1367 }
1368 return false;
1369 }
1370
1371 //------------------------------Identity---------------------------------------
1372 // Loads are identity if previous store is to same address
1373 Node* LoadNode::Identity(PhaseGVN* phase) {
1374 // If the previous store-maker is the right kind of Store, and the store is
1375 // to the same address, then we are equal to the value stored.
1376 Node* mem = in(Memory);
1377 Node* value = can_see_stored_value_through_membars(mem, phase);
1378 if( value ) {
1379 // byte, short & char stores truncate naturally.
1380 // A load has to load the truncated value which requires
1381 // some sort of masking operation and that requires an
1382 // Ideal call instead of an Identity call.
1383 if (memory_size() < BytesPerInt) {
1384 // If the input to the store does not fit with the load's result type,
1385 // it must be truncated via an Ideal call.
1386 if (!phase->type(value)->higher_equal(phase->type(this)))
1387 return this;
1388 }
1389 // (This works even when value is a Con, but LoadNode::Value
1390 // usually runs first, producing the singleton type of the Con.)
1391 if (!has_pinned_control_dependency() || value->is_Con()) {
1392 return value;
1393 } else {
1394 return this;
1395 }
1396 }
1397
1398 if (has_pinned_control_dependency()) {
1399 return this;
1400 }
1401 // Search for an existing data phi which was generated before for the same
1402 // instance's field to avoid infinite generation of phis in a loop.
1403 Node *region = mem->in(0);
1404 if (is_instance_field_load_with_local_phi(region)) {
1405 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1406 int this_index = phase->C->get_alias_index(addr_t);
1407 int this_offset = addr_t->offset();
1408 int this_iid = addr_t->instance_id();
1409 if (!addr_t->is_known_instance() &&
1410 addr_t->is_ptr_to_boxed_value()) {
1411 // Use _idx of address base (could be Phi node) for boxed values.
1412 intptr_t ignore = 0;
1413 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1414 if (base == nullptr) {
1415 return this;
1416 }
1417 this_iid = base->_idx;
1418 }
1419 const Type* this_type = bottom_type();
1420 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1421 Node* phi = region->fast_out(i);
1422 if (phi->is_Phi() && phi != mem &&
1423 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1424 return phi;
1425 }
1426 }
1427 }
1428
1429 return this;
1430 }
1431
1966 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1967 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1968
1969 // Skip up past a SafePoint control. Cannot do this for Stores because
1970 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1971 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1972 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1973 !addr_mark &&
1974 (depends_only_on_test() || has_unknown_control_dependency())) {
1975 ctrl = ctrl->in(0);
1976 set_req(MemNode::Control,ctrl);
1977 return this;
1978 }
1979
1980 intptr_t ignore = 0;
1981 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1982 if (base != nullptr
1983 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1984 // Check for useless control edge in some common special cases
1985 if (in(MemNode::Control) != nullptr
1986 && can_remove_control()
1987 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1988 && all_controls_dominate(base, phase->C->start(), phase)) {
1989 // A method-invariant, non-null address (constant or 'this' argument).
1990 set_req(MemNode::Control, nullptr);
1991 return this;
1992 }
1993 }
1994
1995 Node* mem = in(MemNode::Memory);
1996 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1997
1998 if (can_reshape && (addr_t != nullptr)) {
1999 // try to optimize our memory input
2000 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2001 if (opt_mem != mem) {
2002 set_req_X(MemNode::Memory, opt_mem, phase);
2003 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2004 return this;
2005 }
2118 // No match.
2119 return nullptr;
2120 }
2121
2122 //------------------------------Value-----------------------------------------
2123 const Type* LoadNode::Value(PhaseGVN* phase) const {
2124 // Either input is TOP ==> the result is TOP
2125 Node* mem = in(MemNode::Memory);
2126 const Type *t1 = phase->type(mem);
2127 if (t1 == Type::TOP) return Type::TOP;
2128 Node* adr = in(MemNode::Address);
2129 const TypePtr* tp = phase->type(adr)->isa_ptr();
2130 if (tp == nullptr || tp->empty()) return Type::TOP;
2131 int off = tp->offset();
2132 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2133 Compile* C = phase->C;
2134
2135 // If load can see a previous constant store, use that.
2136 Node* value = can_see_stored_value_through_membars(mem, phase);
2137 if (value != nullptr && value->is_Con()) {
2138 assert(value->bottom_type()->higher_equal(_type), "sanity");
2139 return value->bottom_type();
2140 }
2141
2142 // Try to guess loaded type from pointer type
2143 if (tp->isa_aryptr()) {
2144 const TypeAryPtr* ary = tp->is_aryptr();
2145 const Type* t = ary->elem();
2146
2147 // Determine whether the reference is beyond the header or not, by comparing
2148 // the offset against the offset of the start of the array's data.
2149 // Different array types begin at slightly different offsets (12 vs. 16).
2150 // We choose T_BYTE as an example base type that is least restrictive
2151 // as to alignment, which will therefore produce the smallest
2152 // possible base offset.
2153 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2154 const bool off_beyond_header = (off >= min_base_off);
2155
2156 // Try to constant-fold a stable array element.
2157 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2158 // Make sure the reference is not into the header and the offset is constant
2159 ciObject* aobj = ary->const_oop();
2160 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2161 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2162 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off,
2163 stable_dimension,
2164 value_basic_type(), is_unsigned());
2165 if (con_type != nullptr) {
2166 return con_type;
2167 }
2168 }
2169 }
2170
2171 // Don't do this for integer types. There is only potential profit if
2172 // the element type t is lower than _type; that is, for int types, if _type is
2173 // more restrictive than t. This only happens here if one is short and the other
2174 // char (both 16 bits), and in those cases we've made an intentional decision
2175 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2176 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2177 //
2178 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2179 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2180 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2181 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2182 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2183 // In fact, that could have been the original type of p1, and p1 could have
2184 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2185 // expression (LShiftL quux 3) independently optimized to the constant 8.
2186 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2187 && (_type->isa_vect() == nullptr)
2188 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2189 // t might actually be lower than _type, if _type is a unique
2190 // concrete subclass of abstract class t.
2191 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2192 const Type* jt = t->join_speculative(_type);
2193 // In any case, do not allow the join, per se, to empty out the type.
2194 if (jt->empty() && !t->empty()) {
2195 // This can happen if a interface-typed array narrows to a class type.
2196 jt = _type;
2197 }
2198 #ifdef ASSERT
2199 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2200 // The pointers in the autobox arrays are always non-null
2201 Node* base = adr->in(AddPNode::Base);
2202 if ((base != nullptr) && base->is_DecodeN()) {
2203 // Get LoadN node which loads IntegerCache.cache field
2204 base = base->in(1);
2205 }
2206 if ((base != nullptr) && base->is_Con()) {
2207 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2208 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2209 // It could be narrow oop
2210 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2211 }
2212 }
2213 }
2214 #endif
2215 return jt;
2216 }
2217 }
2218 } else if (tp->base() == Type::InstPtr) {
2219 assert( off != Type::OffsetBot ||
2220 // arrays can be cast to Objects
2221 !tp->isa_instptr() ||
2222 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2223 // unsafe field access may not have a constant offset
2224 C->has_unsafe_access(),
2225 "Field accesses must be precise" );
2226 // For oop loads, we expect the _type to be precise.
2227
2228 // Optimize loads from constant fields.
2229 const TypeInstPtr* tinst = tp->is_instptr();
2230 ciObject* const_oop = tinst->const_oop();
2231 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2232 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2233 if (con_type != nullptr) {
2234 return con_type;
2235 }
2236 }
2237 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2238 assert(off != Type::OffsetBot ||
2239 !tp->isa_instklassptr() ||
2240 // arrays can be cast to Objects
2241 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2242 // also allow array-loading from the primary supertype
2243 // array during subtype checks
2244 Opcode() == Op_LoadKlass,
2245 "Field accesses must be precise");
2246 // For klass/static loads, we expect the _type to be precise
2247 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2248 /* With mirrors being an indirect in the Klass*
2249 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2250 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2251 *
2252 * So check the type and klass of the node before the LoadP.
2259 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2260 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2261 return TypeInstPtr::make(klass->java_mirror());
2262 }
2263 }
2264 }
2265
2266 const TypeKlassPtr *tkls = tp->isa_klassptr();
2267 if (tkls != nullptr) {
2268 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2269 ciKlass* klass = tkls->exact_klass();
2270 // We are loading a field from a Klass metaobject whose identity
2271 // is known at compile time (the type is "exact" or "precise").
2272 // Check for fields we know are maintained as constants by the VM.
2273 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2274 // The field is Klass::_super_check_offset. Return its (constant) value.
2275 // (Folds up type checking code.)
2276 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2277 return TypeInt::make(klass->super_check_offset());
2278 }
2279 if (UseCompactObjectHeaders) {
2280 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2281 // The field is Klass::_prototype_header. Return its (constant) value.
2282 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2283 return TypeX::make(klass->prototype_header());
2284 }
2285 }
2286 // Compute index into primary_supers array
2287 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2288 // Check for overflowing; use unsigned compare to handle the negative case.
2289 if( depth < ciKlass::primary_super_limit() ) {
2290 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2291 // (Folds up type checking code.)
2292 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2293 ciKlass *ss = klass->super_of_depth(depth);
2294 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2295 }
2296 const Type* aift = load_array_final_field(tkls, klass);
2297 if (aift != nullptr) return aift;
2298 }
2299
2300 // We can still check if we are loading from the primary_supers array at a
2301 // shallow enough depth. Even though the klass is not exact, entries less
2302 // than or equal to its super depth are correct.
2303 if (tkls->is_loaded()) {
2304 ciKlass* klass = nullptr;
2338 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2339 // The key property of this type is that it folds up tests
2340 // for array-ness, since it proves that the layout_helper is positive.
2341 // Thus, a generic value like the basic object layout helper works fine.
2342 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2343 }
2344 }
2345
2346 // If we are loading from a freshly-allocated object/array, produce a zero.
2347 // Things to check:
2348 // 1. Load is beyond the header: headers are not guaranteed to be zero
2349 // 2. Load is not vectorized: vectors have no zero constant
2350 // 3. Load has no matching store, i.e. the input is the initial memory state
2351 const TypeOopPtr* tinst = tp->isa_oopptr();
2352 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2353 bool is_not_vect = (_type->isa_vect() == nullptr);
2354 if (is_not_header && is_not_vect) {
2355 Node* mem = in(MemNode::Memory);
2356 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2357 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2358 return Type::get_zero_type(_type->basic_type());
2359 }
2360 }
2361
2362 if (!UseCompactObjectHeaders) {
2363 Node* alloc = is_new_object_mark_load();
2364 if (alloc != nullptr) {
2365 return TypeX::make(markWord::prototype().value());
2366 }
2367 }
2368
2369 return _type;
2370 }
2371
2372 //------------------------------match_edge-------------------------------------
2373 // Do we Match on this edge index or not? Match only the address.
2374 uint LoadNode::match_edge(uint idx) const {
2375 return idx == MemNode::Address;
2376 }
2377
2378 //--------------------------LoadBNode::Ideal--------------------------------------
2379 //
2380 // If the previous store is to the same address as this load,
2381 // and the value stored was larger than a byte, replace this load
2382 // with the value stored truncated to a byte. If no truncation is
2383 // needed, the replacement is done in LoadNode::Identity().
2384 //
2385 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2494 }
2495 }
2496 // Identity call will handle the case where truncation is not needed.
2497 return LoadNode::Ideal(phase, can_reshape);
2498 }
2499
2500 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2501 Node* mem = in(MemNode::Memory);
2502 Node* value = can_see_stored_value_through_membars(mem, phase);
2503 if (value != nullptr && value->is_Con() &&
2504 !value->bottom_type()->higher_equal(_type)) {
2505 // If the input to the store does not fit with the load's result type,
2506 // it must be truncated. We can't delay until Ideal call since
2507 // a singleton Value is needed for split_thru_phi optimization.
2508 int con = value->get_int();
2509 return TypeInt::make((con << 16) >> 16);
2510 }
2511 return LoadNode::Value(phase);
2512 }
2513
2514 //=============================================================================
2515 //----------------------------LoadKlassNode::make------------------------------
2516 // Polymorphic factory method:
2517 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2518 // sanity check the alias category against the created node type
2519 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2520 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2521 #ifdef _LP64
2522 if (adr_type->is_ptr_to_narrowklass()) {
2523 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2524 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2525 }
2526 #endif
2527 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2528 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2529 }
2530
2531 //------------------------------Value------------------------------------------
2532 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2533 return klass_value_common(phase);
2566 }
2567 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2568 }
2569 if (!t->is_klass()) {
2570 // a primitive Class (e.g., int.class) has null for a klass field
2571 return TypePtr::NULL_PTR;
2572 }
2573 // Fold up the load of the hidden field
2574 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2575 }
2576 // non-constant mirror, so we can't tell what's going on
2577 }
2578 if (!tinst->is_loaded())
2579 return _type; // Bail out if not loaded
2580 if (offset == oopDesc::klass_offset_in_bytes()) {
2581 return tinst->as_klass_type(true);
2582 }
2583 }
2584
2585 // Check for loading klass from an array
2586 const TypeAryPtr *tary = tp->isa_aryptr();
2587 if (tary != nullptr &&
2588 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2589 return tary->as_klass_type(true);
2590 }
2591
2592 // Check for loading klass from an array klass
2593 const TypeKlassPtr *tkls = tp->isa_klassptr();
2594 if (tkls != nullptr && !StressReflectiveCode) {
2595 if (!tkls->is_loaded())
2596 return _type; // Bail out if not loaded
2597 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2598 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2599 // // Always returning precise element type is incorrect,
2600 // // e.g., element type could be object and array may contain strings
2601 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2602
2603 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2604 // according to the element type's subclassing.
2605 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2606 }
2607 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2608 tkls->offset() == in_bytes(Klass::super_offset())) {
2609 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2610 // The field is Klass::_super. Return its (constant) value.
2611 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2612 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2613 }
2614 }
2615
2616 if (tkls != nullptr && !UseSecondarySupersCache
2617 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2618 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2619 return TypePtr::NULL_PTR;
2620 }
2621
2622 // Bailout case
2623 return LoadNode::Value(phase);
2624 }
2625
2626 //------------------------------Identity---------------------------------------
2649 base = bs->step_over_gc_barrier(base);
2650 }
2651
2652 // We can fetch the klass directly through an AllocateNode.
2653 // This works even if the klass is not constant (clone or newArray).
2654 if (offset == oopDesc::klass_offset_in_bytes()) {
2655 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2656 if (allocated_klass != nullptr) {
2657 return allocated_klass;
2658 }
2659 }
2660
2661 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2662 // See inline_native_Class_query for occurrences of these patterns.
2663 // Java Example: x.getClass().isAssignableFrom(y)
2664 //
2665 // This improves reflective code, often making the Class
2666 // mirror go completely dead. (Current exception: Class
2667 // mirrors may appear in debug info, but we could clean them out by
2668 // introducing a new debug info operator for Klass.java_mirror).
2669
2670 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2671 && offset == java_lang_Class::klass_offset()) {
2672 if (base->is_Load()) {
2673 Node* base2 = base->in(MemNode::Address);
2674 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2675 Node* adr2 = base2->in(MemNode::Address);
2676 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2677 if (tkls != nullptr && !tkls->empty()
2678 && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2679 && adr2->is_AddP()
2680 ) {
2681 int mirror_field = in_bytes(Klass::java_mirror_offset());
2682 if (tkls->offset() == mirror_field) {
2683 #ifdef ASSERT
2684 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2685 assert(tkls2->offset() == 0, "not a load of java_mirror");
2686 #endif
2687 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2688 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2689 return adr2->in(AddPNode::Address);
2690 }
2691 }
2692 }
2693 }
2694 }
2695
2696 return this;
2697 }
2698
2699 LoadNode* LoadNode::clone_pinned() const {
2700 LoadNode* ld = clone()->as_Load();
2827 // Polymorphic factory method:
2828 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2829 assert((mo == unordered || mo == release), "unexpected");
2830 Compile* C = gvn.C;
2831 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
2832 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2833 ctl != nullptr, "raw memory operations should have control edge");
2834
2835 switch (bt) {
2836 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2837 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2838 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2839 case T_CHAR:
2840 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2841 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2842 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2843 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2844 case T_METADATA:
2845 case T_ADDRESS:
2846 case T_OBJECT:
2847 #ifdef _LP64
2848 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2849 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2850 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2851 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2852 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
2853 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2854 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2855 }
2856 #endif
2857 {
2858 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2859 }
2860 default:
2861 ShouldNotReachHere();
2862 return (StoreNode*)nullptr;
2863 }
2864 }
2865
2866 //--------------------------bottom_type----------------------------------------
2867 const Type *StoreNode::bottom_type() const {
2868 return Type::MEMORY;
2869 }
2870
2871 //------------------------------hash-------------------------------------------
2872 uint StoreNode::hash() const {
2873 // unroll addition of interesting fields
2874 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2875
2876 // Since they are not commoned, do not hash them:
2877 return NO_HASH;
2878 }
2879
2880 // Link together multiple stores (B/S/C/I) into a longer one.
2881 //
3503 }
3504 ss.print_cr("[TraceMergeStores]: with");
3505 merged_input_value->dump("\n", false, &ss);
3506 merged_store->dump("\n", false, &ss);
3507 tty->print("%s", ss.as_string());
3508 }
3509 #endif
3510
3511 //------------------------------Ideal------------------------------------------
3512 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3513 // When a store immediately follows a relevant allocation/initialization,
3514 // try to capture it into the initialization, or hoist it above.
3515 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3516 Node* p = MemNode::Ideal_common(phase, can_reshape);
3517 if (p) return (p == NodeSentinel) ? nullptr : p;
3518
3519 Node* mem = in(MemNode::Memory);
3520 Node* address = in(MemNode::Address);
3521 Node* value = in(MemNode::ValueIn);
3522 // Back-to-back stores to same address? Fold em up. Generally
3523 // unsafe if I have intervening uses.
3524 {
3525 Node* st = mem;
3526 // If Store 'st' has more than one use, we cannot fold 'st' away.
3527 // For example, 'st' might be the final state at a conditional
3528 // return. Or, 'st' might be used by some node which is live at
3529 // the same time 'st' is live, which might be unschedulable. So,
3530 // require exactly ONE user until such time as we clone 'mem' for
3531 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3532 // true).
3533 while (st->is_Store() && st->outcnt() == 1) {
3534 // Looking at a dead closed cycle of memory?
3535 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3536 assert(Opcode() == st->Opcode() ||
3537 st->Opcode() == Op_StoreVector ||
3538 Opcode() == Op_StoreVector ||
3539 st->Opcode() == Op_StoreVectorScatter ||
3540 Opcode() == Op_StoreVectorScatter ||
3541 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3542 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3543 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3544 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3545 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3546
3547 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3548 st->as_Store()->memory_size() <= this->memory_size()) {
3549 Node* use = st->raw_out(0);
3550 if (phase->is_IterGVN()) {
3551 phase->is_IterGVN()->rehash_node_delayed(use);
3552 }
3553 // It's OK to do this in the parser, since DU info is always accurate,
3554 // and the parser always refers to nodes via SafePointNode maps.
3555 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3556 return this;
3557 }
3558 st = st->in(MemNode::Memory);
3559 }
3560 }
3561
3562
3563 // Capture an unaliased, unconditional, simple store into an initializer.
3664 const StoreVectorNode* store_vector = as_StoreVector();
3665 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3666 const Node* store_indices = store_vector->indices();
3667 const Node* mem_indices = mem_vector->indices();
3668 const Node* store_mask = store_vector->mask();
3669 const Node* mem_mask = mem_vector->mask();
3670 // Ensure types, indices, and masks match
3671 if (store_vector->vect_type() == mem_vector->vect_type() &&
3672 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3673 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3674 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3675 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3676 result = mem;
3677 }
3678 }
3679 }
3680
3681 // Store of zero anywhere into a freshly-allocated object?
3682 // Then the store is useless.
3683 // (It must already have been captured by the InitializeNode.)
3684 if (result == this &&
3685 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3686 // a newly allocated object is already all-zeroes everywhere
3687 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3688 result = mem;
3689 }
3690
3691 if (result == this) {
3692 // the store may also apply to zero-bits in an earlier object
3693 Node* prev_mem = find_previous_store(phase);
3694 // Steps (a), (b): Walk past independent stores to find an exact match.
3695 if (prev_mem != nullptr) {
3696 if (prev_mem->is_top()) {
3697 // find_previous_store returns top when the access is dead
3698 return prev_mem;
3699 }
3700 Node* prev_val = can_see_stored_value(prev_mem, phase);
3701 if (prev_val != nullptr && prev_val == val) {
3702 // prev_val and val might differ by a cast; it would be good
3703 // to keep the more informative of the two.
3704 result = mem;
3705 }
3706 }
3707 }
3708 }
3709
3710 PhaseIterGVN* igvn = phase->is_IterGVN();
3711 if (result != this && igvn != nullptr) {
4184 // Clearing a short array is faster with stores
4185 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4186 // Already know this is a large node, do not try to ideal it
4187 if (_is_large) return nullptr;
4188
4189 const int unit = BytesPerLong;
4190 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4191 if (!t) return nullptr;
4192 if (!t->is_con()) return nullptr;
4193 intptr_t raw_count = t->get_con();
4194 intptr_t size = raw_count;
4195 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4196 // Clearing nothing uses the Identity call.
4197 // Negative clears are possible on dead ClearArrays
4198 // (see jck test stmt114.stmt11402.val).
4199 if (size <= 0 || size % unit != 0) return nullptr;
4200 intptr_t count = size / unit;
4201 // Length too long; communicate this to matchers and assemblers.
4202 // Assemblers are responsible to produce fast hardware clears for it.
4203 if (size > InitArrayShortSize) {
4204 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4205 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4206 return nullptr;
4207 }
4208 if (!IdealizeClearArrayNode) return nullptr;
4209 Node *mem = in(1);
4210 if( phase->type(mem)==Type::TOP ) return nullptr;
4211 Node *adr = in(3);
4212 const Type* at = phase->type(adr);
4213 if( at==Type::TOP ) return nullptr;
4214 const TypePtr* atp = at->isa_ptr();
4215 // adjust atp to be the correct array element address type
4216 if (atp == nullptr) atp = TypePtr::BOTTOM;
4217 else atp = atp->add_offset(Type::OffsetBot);
4218 // Get base for derived pointer purposes
4219 if( adr->Opcode() != Op_AddP ) Unimplemented();
4220 Node *base = adr->in(1);
4221
4222 Node *zero = phase->makecon(TypeLong::ZERO);
4223 Node *off = phase->MakeConX(BytesPerLong);
4224 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4225 count--;
4226 while (count--) {
4227 mem = phase->transform(mem);
4228 adr = phase->transform(AddPNode::make_with_base(base, adr, off));
4229 mem = new StoreLNode(in(0), mem, adr, atp, zero, MemNode::unordered, false);
4230 }
4231 return mem;
4232 }
4233
4234 //----------------------------step_through----------------------------------
4235 // Return allocation input memory edge if it is different instance
4236 // or itself if it is the one we are looking for.
4237 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4238 Node* n = *np;
4239 assert(n->is_ClearArray(), "sanity");
4240 intptr_t offset;
4241 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4242 // This method is called only before Allocate nodes are expanded
4243 // during macro nodes expansion. Before that ClearArray nodes are
4244 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4245 // Allocate nodes are expanded) which follows allocations.
4246 assert(alloc != nullptr, "should have allocation");
4247 if (alloc->_idx == instance_id) {
4248 // Can not bypass initialization of the instance we are looking for.
4249 return false;
4252 InitializeNode* init = alloc->initialization();
4253 if (init != nullptr)
4254 *np = init->in(TypeFunc::Memory);
4255 else
4256 *np = alloc->in(TypeFunc::Memory);
4257 return true;
4258 }
4259
4260 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4261 Node* base = dest;
4262 if (raw_base) {
4263 // May be called as part of the initialization of a just allocated object
4264 base = phase->C->top();
4265 }
4266 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4267 }
4268
4269 //----------------------------clear_memory-------------------------------------
4270 // Generate code to initialize object storage to zero.
4271 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4272 intptr_t start_offset,
4273 Node* end_offset,
4274 bool raw_base,
4275 PhaseGVN* phase) {
4276 intptr_t offset = start_offset;
4277
4278 int unit = BytesPerLong;
4279 if ((offset % unit) != 0) {
4280 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4281 const TypePtr* atp = TypeRawPtr::BOTTOM;
4282 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4283 mem = phase->transform(mem);
4284 offset += BytesPerInt;
4285 }
4286 assert((offset % unit) == 0, "");
4287
4288 // Initialize the remaining stuff, if any, with a ClearArray.
4289 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, raw_base, phase);
4290 }
4291
4292 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4293 Node* start_offset,
4294 Node* end_offset,
4295 bool raw_base,
4296 PhaseGVN* phase) {
4297 if (start_offset == end_offset) {
4298 // nothing to do
4299 return mem;
4300 }
4301
4302 int unit = BytesPerLong;
4303 Node* zbase = start_offset;
4304 Node* zend = end_offset;
4305
4306 // Scale to the unit required by the CPU:
4307 if (!Matcher::init_array_count_is_in_bytes) {
4308 Node* shift = phase->intcon(exact_log2(unit));
4309 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4310 zend = phase->transform(new URShiftXNode(zend, shift) );
4311 }
4312
4313 // Bulk clear double-words
4314 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4315 Node* adr = make_address(dest, start_offset, raw_base, phase);
4316 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4317 return phase->transform(mem);
4318 }
4319
4320 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4321 intptr_t start_offset,
4322 intptr_t end_offset,
4323 bool raw_base,
4324 PhaseGVN* phase) {
4325 if (start_offset == end_offset) {
4326 // nothing to do
4327 return mem;
4328 }
4329
4330 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4331 intptr_t done_offset = end_offset;
4332 if ((done_offset % BytesPerLong) != 0) {
4333 done_offset -= BytesPerInt;
4334 }
4335 if (done_offset > start_offset) {
4336 mem = clear_memory(ctl, mem, dest,
4337 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4338 }
4339 if (done_offset < end_offset) { // emit the final 32-bit store
4340 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4341 const TypePtr* atp = TypeRawPtr::BOTTOM;
4342 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4343 mem = phase->transform(mem);
4344 done_offset += BytesPerInt;
4345 }
4346 assert(done_offset == end_offset, "");
4347 return mem;
4348 }
4349
4350 //=============================================================================
4351 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4352 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4353 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4354 #ifdef ASSERT
4355 , _pair_idx(0)
4356 #endif
4357 {
4358 init_class_id(Class_MemBar);
4359 Node* top = C->top();
4360 init_req(TypeFunc::I_O,top);
4361 init_req(TypeFunc::FramePtr,top);
4362 init_req(TypeFunc::ReturnAdr,top);
4471 PhaseIterGVN* igvn = phase->is_IterGVN();
4472 remove(igvn);
4473 // Must return either the original node (now dead) or a new node
4474 // (Do not return a top here, since that would break the uniqueness of top.)
4475 return new ConINode(TypeInt::ZERO);
4476 }
4477 }
4478 return progress ? this : nullptr;
4479 }
4480
4481 //------------------------------Value------------------------------------------
4482 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4483 if( !in(0) ) return Type::TOP;
4484 if( phase->type(in(0)) == Type::TOP )
4485 return Type::TOP;
4486 return TypeTuple::MEMBAR;
4487 }
4488
4489 //------------------------------match------------------------------------------
4490 // Construct projections for memory.
4491 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4492 switch (proj->_con) {
4493 case TypeFunc::Control:
4494 case TypeFunc::Memory:
4495 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4496 }
4497 ShouldNotReachHere();
4498 return nullptr;
4499 }
4500
4501 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4502 trailing->_kind = TrailingStore;
4503 leading->_kind = LeadingStore;
4504 #ifdef ASSERT
4505 trailing->_pair_idx = leading->_idx;
4506 leading->_pair_idx = leading->_idx;
4507 #endif
4508 }
4509
4510 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4511 trailing->_kind = TrailingLoadStore;
4758 return (req() > RawStores);
4759 }
4760
4761 void InitializeNode::set_complete(PhaseGVN* phase) {
4762 assert(!is_complete(), "caller responsibility");
4763 _is_complete = Complete;
4764
4765 // After this node is complete, it contains a bunch of
4766 // raw-memory initializations. There is no need for
4767 // it to have anything to do with non-raw memory effects.
4768 // Therefore, tell all non-raw users to re-optimize themselves,
4769 // after skipping the memory effects of this initialization.
4770 PhaseIterGVN* igvn = phase->is_IterGVN();
4771 if (igvn) igvn->add_users_to_worklist(this);
4772 }
4773
4774 // convenience function
4775 // return false if the init contains any stores already
4776 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4777 InitializeNode* init = initialization();
4778 if (init == nullptr || init->is_complete()) return false;
4779 init->remove_extra_zeroes();
4780 // for now, if this allocation has already collected any inits, bail:
4781 if (init->is_non_zero()) return false;
4782 init->set_complete(phase);
4783 return true;
4784 }
4785
4786 void InitializeNode::remove_extra_zeroes() {
4787 if (req() == RawStores) return;
4788 Node* zmem = zero_memory();
4789 uint fill = RawStores;
4790 for (uint i = fill; i < req(); i++) {
4791 Node* n = in(i);
4792 if (n->is_top() || n == zmem) continue; // skip
4793 if (fill < i) set_req(fill, n); // compact
4794 ++fill;
4795 }
4796 // delete any empty spaces created:
4797 while (fill < req()) {
4798 del_req(fill);
4942 // store node that we'd like to capture. We need to check
4943 // the uses of the MergeMemNode.
4944 mems.push(n);
4945 }
4946 } else if (n->is_Mem()) {
4947 Node* other_adr = n->in(MemNode::Address);
4948 if (other_adr == adr) {
4949 failed = true;
4950 break;
4951 } else {
4952 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4953 if (other_t_adr != nullptr) {
4954 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4955 if (other_alias_idx == alias_idx) {
4956 // A load from the same memory slice as the store right
4957 // after the InitializeNode. We check the control of the
4958 // object/array that is loaded from. If it's the same as
4959 // the store control then we cannot capture the store.
4960 assert(!n->is_Store(), "2 stores to same slice on same control?");
4961 Node* base = other_adr;
4962 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4963 base = base->in(AddPNode::Base);
4964 if (base != nullptr) {
4965 base = base->uncast();
4966 if (base->is_Proj() && base->in(0) == alloc) {
4967 failed = true;
4968 break;
4969 }
4970 }
4971 }
4972 }
4973 }
4974 } else {
4975 failed = true;
4976 break;
4977 }
4978 }
4979 }
4980 }
4981 if (failed) {
5527 // z's_done 12 16 16 16 12 16 12
5528 // z's_needed 12 16 16 16 16 16 16
5529 // zsize 0 0 0 0 4 0 4
5530 if (next_full_store < 0) {
5531 // Conservative tack: Zero to end of current word.
5532 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5533 } else {
5534 // Zero to beginning of next fully initialized word.
5535 // Or, don't zero at all, if we are already in that word.
5536 assert(next_full_store >= zeroes_needed, "must go forward");
5537 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5538 zeroes_needed = next_full_store;
5539 }
5540 }
5541
5542 if (zeroes_needed > zeroes_done) {
5543 intptr_t zsize = zeroes_needed - zeroes_done;
5544 // Do some incremental zeroing on rawmem, in parallel with inits.
5545 zeroes_done = align_down(zeroes_done, BytesPerInt);
5546 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5547 zeroes_done, zeroes_needed,
5548 true,
5549 phase);
5550 zeroes_done = zeroes_needed;
5551 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5552 do_zeroing = false; // leave the hole, next time
5553 }
5554 }
5555
5556 // Collect the store and move on:
5557 phase->replace_input_of(st, MemNode::Memory, inits);
5558 inits = st; // put it on the linearized chain
5559 set_req(i, zmem); // unhook from previous position
5560
5561 if (zeroes_done == st_off)
5562 zeroes_done = next_init_off;
5563
5564 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5565
5566 #ifdef ASSERT
5587 remove_extra_zeroes(); // clear out all the zmems left over
5588 add_req(inits);
5589
5590 if (!(UseTLAB && ZeroTLAB)) {
5591 // If anything remains to be zeroed, zero it all now.
5592 zeroes_done = align_down(zeroes_done, BytesPerInt);
5593 // if it is the last unused 4 bytes of an instance, forget about it
5594 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5595 if (zeroes_done + BytesPerLong >= size_limit) {
5596 AllocateNode* alloc = allocation();
5597 assert(alloc != nullptr, "must be present");
5598 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5599 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5600 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5601 if (zeroes_done == k->layout_helper())
5602 zeroes_done = size_limit;
5603 }
5604 }
5605 if (zeroes_done < size_limit) {
5606 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5607 zeroes_done, size_in_bytes, true, phase);
5608 }
5609 }
5610
5611 set_complete(phase);
5612 return rawmem;
5613 }
5614
5615 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5616 auto replace_proj = [&](ProjNode* proj) {
5617 C->gvn_replace_by(proj, mem);
5618 return CONTINUE;
5619 };
5620 apply_to_projs(replace_proj, TypeFunc::Memory);
5621 }
5622
5623 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5624 DUIterator_Fast imax, i = fast_outs(imax);
5625 auto replace_proj = [&](ProjNode* proj) {
5626 igvn->replace_node(proj, mem);
5824 //------------------------------Identity---------------------------------------
5825 Node* MergeMemNode::Identity(PhaseGVN* phase) {
5826 // Identity if this merge point does not record any interesting memory
5827 // disambiguations.
5828 Node* base_mem = base_memory();
5829 Node* empty_mem = empty_memory();
5830 if (base_mem != empty_mem) { // Memory path is not dead?
5831 for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
5832 Node* mem = in(i);
5833 if (mem != empty_mem && mem != base_mem) {
5834 return this; // Many memory splits; no change
5835 }
5836 }
5837 }
5838 return base_mem; // No memory splits; ID on the one true input
5839 }
5840
5841 //------------------------------Ideal------------------------------------------
5842 // This method is invoked recursively on chains of MergeMem nodes
5843 Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) {
5844 // Remove chain'd MergeMems
5845 //
5846 // This is delicate, because the each "in(i)" (i >= Raw) is interpreted
5847 // relative to the "in(Bot)". Since we are patching both at the same time,
5848 // we have to be careful to read each "in(i)" relative to the old "in(Bot)",
5849 // but rewrite each "in(i)" relative to the new "in(Bot)".
5850 Node *progress = nullptr;
5851
5852
5853 Node* old_base = base_memory();
5854 Node* empty_mem = empty_memory();
5855 if (old_base == empty_mem)
5856 return nullptr; // Dead memory path.
5857
5858 MergeMemNode* old_mbase;
5859 if (old_base != nullptr && old_base->is_MergeMem())
5860 old_mbase = old_base->as_MergeMem();
5861 else
5862 old_mbase = nullptr;
5863 Node* new_base = old_base;
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "ci/ciInlineKlass.hpp"
28 #include "ci/ciInstanceKlass.hpp"
29 #include "classfile/javaClasses.hpp"
30 #include "classfile/systemDictionary.hpp"
31 #include "classfile/vmIntrinsics.hpp"
32 #include "compiler/compileLog.hpp"
33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/c2/barrierSetC2.hpp"
35 #include "gc/shared/tlab_globals.hpp"
36 #include "memory/allocation.inline.hpp"
37 #include "memory/resourceArea.hpp"
38 #include "oops/flatArrayKlass.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "opto/addnode.hpp"
41 #include "opto/arraycopynode.hpp"
42 #include "opto/callnode.hpp"
43 #include "opto/cfgnode.hpp"
44 #include "opto/compile.hpp"
45 #include "opto/connode.hpp"
46 #include "opto/convertnode.hpp"
47 #include "opto/inlinetypenode.hpp"
48 #include "opto/loopnode.hpp"
49 #include "opto/machnode.hpp"
50 #include "opto/matcher.hpp"
51 #include "opto/memnode.hpp"
52 #include "opto/mempointer.hpp"
53 #include "opto/mulnode.hpp"
54 #include "opto/narrowptrnode.hpp"
55 #include "opto/opcodes.hpp"
56 #include "opto/phaseX.hpp"
57 #include "opto/regalloc.hpp"
58 #include "opto/regmask.hpp"
59 #include "opto/rootnode.hpp"
60 #include "opto/traceMergeStoresTag.hpp"
61 #include "opto/vectornode.hpp"
62 #include "runtime/arguments.hpp"
63 #include "utilities/align.hpp"
64 #include "utilities/copy.hpp"
65 #include "utilities/globalDefinitions.hpp"
66 #include "utilities/macros.hpp"
67 #include "utilities/powerOfTwo.hpp"
68 #include "utilities/vmError.hpp"
69
70 // Portions of code courtesy of Clifford Click
71
72 // Optimization - Graph Style
73
74 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
75
76 //=============================================================================
77 uint MemNode::size_of() const { return sizeof(*this); }
78
79 const TypePtr *MemNode::adr_type() const {
80 Node* adr = in(Address);
81 if (adr == nullptr) return nullptr; // node is dead
82 const TypePtr* cross_check = nullptr;
134 st->print(", idx=Bot;");
135 else if (atp->index() == Compile::AliasIdxTop)
136 st->print(", idx=Top;");
137 else if (atp->index() == Compile::AliasIdxRaw)
138 st->print(", idx=Raw;");
139 else {
140 ciField* field = atp->field();
141 if (field) {
142 st->print(", name=");
143 field->print_name_on(st);
144 }
145 st->print(", idx=%d;", atp->index());
146 }
147 }
148 }
149
150 extern void print_alias_types();
151
152 #endif
153
154 // Find the memory output corresponding to the fall-through path of a call
155 static Node* find_call_fallthrough_mem_output(CallNode* call) {
156 ResourceMark rm;
157 CallProjections* projs = call->extract_projections(false, false);
158 Node* res = projs->fallthrough_memproj;
159 assert(res != nullptr, "must have a fallthrough mem output");
160 return res;
161 }
162
163 // Try to find a better memory input for a load from a strict final field
164 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
165 intptr_t offset = 0;
166 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
167 if (base == nullptr) {
168 return nullptr;
169 }
170
171 Node* base_uncasted = base->uncast();
172 if (base_uncasted->is_Proj()) {
173 Node* multi = base_uncasted->in(0);
174 if (multi->is_top()) {
175 // The pointer dies, make the memory die, too
176 return multi;
177 } else if (multi->is_Allocate()) {
178 base_local = base_uncasted->as_Proj();
179 return nullptr;
180 } else if (multi->is_Call()) {
181 if (!multi->is_CallJava() || multi->as_CallJava()->method() == nullptr || !multi->as_CallJava()->method()->return_value_is_larval()) {
182 // The oop is returned from a call, the memory can be the fallthrough output of the call
183 return find_call_fallthrough_mem_output(multi->as_Call());
184 }
185 } else if (multi->is_Start()) {
186 // The oop is a parameter
187 if (base_uncasted->as_Proj()->_con == TypeFunc::Parms && phase->C->method()->receiver_maybe_larval()) {
188 // The receiver of a constructor is similar to the result of an AllocateNode
189 base_local = base_uncasted->as_Proj();
190 return nullptr;
191 } else {
192 // Use the start memory otherwise
193 return multi->as_Start()->proj_out(TypeFunc::Memory);
194 }
195 }
196 }
197
198 return nullptr;
199 }
200
201 // Whether a call can modify a strict final field, given that the object is allocated inside the
202 // current compilation unit, or is the first parameter when the compilation root is a constructor.
203 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
204 // the target method is a subclass of the class declaring 'field'.
205 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
206 if (!call->is_CallJava()) {
207 return false;
208 }
209
210 ciMethod* target = call->as_CallJava()->method();
211 if (target == nullptr) {
212 return false;
213 } else if (target->intrinsic_id() == vmIntrinsicID::_linkToSpecial) {
214 // linkToSpecial can be used to call a constructor, used in the construction of objects in the
215 // reflection API
216 return true;
217 } else if (!target->is_object_constructor()) {
218 return false;
219 }
220
221 // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
222 // then the field is set inside the constructor, else the field must be set before the
223 // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
224 // while a field Sub.y must be set before Super::<init> is invoked.
225 // We can try to be more heroic and decide if the receiver of the constructor invocation is the
226 // object from which we are loading from. This, however, may be problematic as deciding if 2
227 // nodes are definitely different may not be trivial, especially if the graph is not canonical.
228 // As a result, it is made more conservative for now.
229 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
230 return target->holder()->is_subclass_of(field->holder());
231 }
232
233 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
234 assert(t_oop != nullptr, "sanity");
235 bool is_known_instance = t_oop->is_known_instance_field();
236 bool is_strict_final_load = false;
237
238 // After macro expansion, an allocation may become a call, changing the memory input to the
239 // memory output of that call would be illegal. As a result, disallow this transformation after
240 // macro expansion.
241 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
242 is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
243 #ifdef ASSERT
244 if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
245 assert(is_strict_final_load, "sanity check for basic cases");
246 }
247 #endif // ASSERT
248 }
249
250 if (!is_known_instance && !is_strict_final_load) {
251 return mchain;
252 }
253
254 Node* result = mchain;
255 ProjNode* base_local = nullptr;
256
257 ciField* field = nullptr;
258 if (is_strict_final_load) {
259 field = phase->C->alias_type(t_oop)->field();
260 assert(field != nullptr, "must point to a field");
261
262 Node* adr = load->in(MemNode::Address);
263 assert(phase->type(adr) == t_oop, "inconsistent type");
264 Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
265 if (tmp != nullptr) {
266 result = tmp;
267 }
268 }
269
270 uint instance_id = t_oop->instance_id();
271 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
272 Node* prev = nullptr;
273 while (prev != result) {
274 prev = result;
275 if (result == start_mem) {
276 // start_mem is the earliest memory possible
277 break;
278 }
279
280 // skip over a call which does not affect this memory slice
281 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
282 Node* proj_in = result->in(0);
283 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
284 // This is the allocation that creates the object from which we are loading from
285 break;
286 } else if (proj_in->is_Call()) {
287 // ArrayCopyNodes processed here as well
288 CallNode* call = proj_in->as_Call();
289 if (!call->may_modify(t_oop, phase)) {
290 result = call->in(TypeFunc::Memory);
291 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
292 result = call->in(TypeFunc::Memory);
293 }
294 } else if (proj_in->Opcode() == Op_Tuple) {
295 // The call will be folded, skip over it.
296 break;
297 } else if (proj_in->is_Initialize()) {
298 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
299 // Stop if this is the initialization for the object instance which
300 // contains this memory slice, otherwise skip over it.
301 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
302 break;
303 }
304 if (is_known_instance) {
305 result = proj_in->in(TypeFunc::Memory);
306 } else if (is_strict_final_load) {
307 Node* klass = alloc->in(AllocateNode::KlassNode);
308 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
309 if (tklass->klass_is_exact() && !tklass->exact_klass()->is_subclass_of(t_oop->is_instptr()->instance_klass())) {
310 // Allocation of an unrelated type, must be another object
311 result = proj_in->in(TypeFunc::Memory);
312 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
313 // Allocation of another object
314 result = proj_in->in(TypeFunc::Memory);
315 }
316 }
317 } else if (proj_in->is_MemBar()) {
318 ArrayCopyNode* ac = nullptr;
319 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
320 break;
321 }
322 result = proj_in->in(TypeFunc::Memory);
323 } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
324 if (is_strict_final_load) {
325 // LoadFlat and StoreFlat cannot happen to strict final fields
326 result = proj_in->in(TypeFunc::Memory);
327 }
328 } else if (proj_in->is_top()) {
329 break; // dead code
330 } else {
331 assert(false, "unexpected projection of %s", proj_in->Name());
332 }
333 } else if (result->is_ClearArray()) {
334 if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
335 // Can not bypass initialization of the instance
336 // we are looking for.
337 break;
338 }
339 // Otherwise skip it (the call updated 'result' value).
340 } else if (result->is_MergeMem()) {
341 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
342 }
343 }
344 return result;
345 }
346
347 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
348 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
349 if (t_oop == nullptr)
350 return mchain; // don't try to optimize non-oop types
351 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
352 bool is_instance = t_oop->is_known_instance_field();
353 PhaseIterGVN *igvn = phase->is_IterGVN();
354 if (is_instance && igvn != nullptr && result->is_Phi()) {
355 PhiNode *mphi = result->as_Phi();
356 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
357 const TypePtr *t = mphi->adr_type();
358 bool do_split = false;
359 // In the following cases, Load memory input can be further optimized based on
360 // its precise address type
361 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
362 do_split = true;
363 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
364 const TypeOopPtr* mem_t =
365 t->is_oopptr()->cast_to_exactness(true)
366 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
367 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
368 if (t_oop->isa_aryptr()) {
369 mem_t = mem_t->is_aryptr()
370 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
371 ->cast_to_size(t_oop->is_aryptr()->size())
372 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
373 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
374 ->with_offset(t_oop->is_aryptr()->offset())
375 ->is_aryptr();
376 }
377 do_split = mem_t == t_oop;
378 }
379 if (do_split) {
380 // clone the Phi with our address type
381 result = mphi->split_out_instance(t_adr, igvn);
382 } else {
383 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
384 }
385 }
386 return result;
387 }
388
389 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
390 uint alias_idx = phase->C->get_alias_index(tp);
391 Node *mem = mmem;
392 #ifdef ASSERT
393 {
394 // Check that current type is consistent with the alias index used during graph construction
395 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
396 bool consistent = adr_check == nullptr || adr_check->empty() ||
397 phase->C->must_alias(adr_check, alias_idx );
398 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
399 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
400 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
401 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
402 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
403 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
404 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
405 // don't assert if it is dead code.
406 consistent = true;
407 }
408 if( !consistent ) {
409 st->print("alias_idx==%d, adr_check==", alias_idx);
410 if( adr_check == nullptr ) {
411 st->print("null");
412 } else {
413 adr_check->dump();
414 }
415 st->cr();
416 print_alias_types();
417 assert(consistent, "adr_check must match alias idx");
418 }
419 }
420 #endif
723 }
724
725 // Find an arraycopy ac that produces the memory state represented by parameter mem.
726 // Return ac if
727 // (a) can_see_stored_value=true and ac must have set the value for this load or if
728 // (b) can_see_stored_value=false and ac could have set the value for this load or if
729 // (c) can_see_stored_value=false and ac cannot have set the value for this load.
730 // In case (c) change the parameter mem to the memory input of ac to skip it
731 // when searching stored value.
732 // Otherwise return null.
733 Node* LoadNode::find_previous_arraycopy(PhaseValues* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const {
734 ArrayCopyNode* ac = find_array_copy_clone(ld_alloc, mem);
735 if (ac != nullptr) {
736 Node* ld_addp = in(MemNode::Address);
737 Node* src = ac->in(ArrayCopyNode::Src);
738 const TypeAryPtr* ary_t = phase->type(src)->isa_aryptr();
739
740 // This is a load from a cloned array. The corresponding arraycopy ac must
741 // have set the value for the load and we can return ac but only if the load
742 // is known to be within bounds. This is checked below.
743 // TODO 8350865: Support flat arrays in LoadNode::find_previous_arraycopy
744 if (ary_t != nullptr && ary_t->is_not_flat() && ld_addp->is_AddP()) {
745 Node* ld_offs = ld_addp->in(AddPNode::Offset);
746 BasicType ary_elem = ary_t->elem()->array_element_basic_type();
747 jlong header = arrayOopDesc::base_offset_in_bytes(ary_elem);
748 jlong elemsize = type2aelembytes(ary_elem);
749
750 const TypeX* ld_offs_t = phase->type(ld_offs)->isa_intptr_t();
751 const TypeInt* sizetype = ary_t->size();
752
753 if (ld_offs_t->_lo >= header && ld_offs_t->_hi < (sizetype->_lo * elemsize + header)) {
754 // The load is known to be within bounds. It receives its value from ac.
755 return ac;
756 }
757 // The load is known to be out-of-bounds.
758 }
759 // The load could be out-of-bounds. It must not be hoisted but must remain
760 // dependent on the runtime range check. This is achieved by returning null.
761 } else if (mem->is_Proj() && mem->in(0) != nullptr && mem->in(0)->is_ArrayCopy()) {
762 ArrayCopyNode* ac = mem->in(0)->as_ArrayCopy();
763
764 if (ac->is_arraycopy_validated() ||
1134 in_bytes(JavaThread::vthread_offset()),
1135 in_bytes(JavaThread::scopedValueCache_offset()),
1136 };
1137
1138 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1139 if (offset == offsets[i]) {
1140 return true;
1141 }
1142 }
1143 }
1144
1145 return false;
1146 }
1147 #endif
1148
1149 //----------------------------LoadNode::make-----------------------------------
1150 // Polymorphic factory method:
1151 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1152 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1153 Compile* C = gvn.C;
1154 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr(), true) == C->get_alias_index(adr_type, true), "adr and adr_type must agree");
1155
1156 // sanity check the alias category against the created node type
1157 assert(!(adr_type->isa_oopptr() &&
1158 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1159 "use LoadKlassNode instead");
1160 assert(!(adr_type->isa_aryptr() &&
1161 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1162 "use LoadRangeNode instead");
1163 // Check control edge of raw loads
1164 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1165 // oop will be recorded in oop map if load crosses safepoint
1166 rt->isa_oopptr() || is_immutable_value(adr),
1167 "raw memory operations should have control edge");
1168 LoadNode* load = nullptr;
1169 switch (bt) {
1170 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1171 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1172 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1173 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1174 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1175 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1176 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1177 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1178 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1179 case T_ARRAY:
1180 case T_OBJECT:
1181 case T_NARROWOOP:
1182 #ifdef _LP64
1183 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1184 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1185 } else
1186 #endif
1187 {
1188 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1189 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1190 }
1191 break;
1192 default:
1193 guarantee(false, "unexpected basic type %s", type2name(bt));
1194 break;
1195 }
1196 assert(load != nullptr, "LoadNode should have been created");
1197 if (unaligned) {
1198 load->set_unaligned_access();
1199 }
1200 if (mismatched) {
1201 load->set_mismatched_access();
1202 }
1203 if (unsafe) {
1204 load->set_unsafe_access();
1205 }
1206 load->set_barrier_data(barrier_data);
1207 if (load->Opcode() == Op_LoadN) {
1208 Node* ld = gvn.transform(load);
1209 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1210 }
1211
1212 return load;
1213 }
1214
1215 //------------------------------hash-------------------------------------------
1216 uint LoadNode::hash() const {
1217 // unroll addition of interesting fields
1218 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1219 }
1220
1221 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1222 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1223 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1224 bool is_stable_ary = FoldStableValues &&
1225 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1226 tp->isa_aryptr()->is_stable();
1227
1228 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1229 }
1230
1231 return false;
1232 }
1233
1234 // Is the value loaded previously stored by an arraycopy? If so return
1235 // a load node that reads from the source array so we may be able to
1236 // optimize out the ArrayCopy node later.
1237 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1238 Node* ld_adr = in(MemNode::Address);
1239 intptr_t ld_off = 0;
1240 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1241 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1242 if (ac != nullptr) {
1243 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1244
1245 Node* mem = ac->in(TypeFunc::Memory);
1246 Node* ctl = ac->in(0);
1247 Node* src = ac->in(ArrayCopyNode::Src);
1248
1256 if (ac->as_ArrayCopy()->is_clonebasic()) {
1257 assert(ld_alloc != nullptr, "need an alloc");
1258 assert(addp->is_AddP(), "address must be addp");
1259 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1260 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1261 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1262 addp->set_req(AddPNode::Base, src);
1263 addp->set_req(AddPNode::Address, src);
1264 } else {
1265 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1266 ac->as_ArrayCopy()->is_copyof_validated() ||
1267 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1268 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1269 addp->set_req(AddPNode::Base, src);
1270 addp->set_req(AddPNode::Address, src);
1271
1272 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1273 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1274 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1275
1276 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1277
1278 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1279 #ifdef _LP64
1280 diff = phase->transform(new ConvI2LNode(diff));
1281 #endif
1282 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1283
1284 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1285 addp->set_req(AddPNode::Offset, offset);
1286 }
1287 addp = phase->transform(addp);
1288 #ifdef ASSERT
1289 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1290 ld->_adr_type = adr_type;
1291 #endif
1292 ld->set_req(MemNode::Address, addp);
1293 ld->set_req(0, ctl);
1294 ld->set_req(MemNode::Memory, mem);
1295 return ld;
1296 }
1297 return nullptr;
1298 }
1299
1300 static Node* see_through_inline_type(PhaseValues* phase, const LoadNode* load, Node* base, int offset) {
1301 if (load->is_mismatched_access() || base == nullptr) {
1302 return nullptr;
1303 }
1304
1305 InlineTypeNode* vt = base->isa_InlineType();
1306 if (vt == nullptr || offset < vt->type()->inline_klass()->payload_offset()) {
1307 return nullptr;
1308 }
1309
1310 Node* value = vt->field_value_by_offset(offset, true);
1311 assert(value != nullptr, "must see some value");
1312 return value;
1313 }
1314
1315 // This routine exists to make sure this set of tests is done the same
1316 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1317 // will change the graph shape in a way which makes memory alive twice at the
1318 // same time (uses the Oracle model of aliasing), then some
1319 // LoadXNode::Identity will fold things back to the equivalence-class model
1320 // of aliasing.
1321 // This method may find an unencoded node instead of the corresponding encoded one.
1322 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1323 Node* ld_adr = in(MemNode::Address);
1324 intptr_t ld_off = 0;
1325 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1326 // Try to see through an InlineTypeNode
1327 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1328 if (value != nullptr) {
1329 return value;
1330 }
1331
1332 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1333 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1334
1335 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1336 uint alias_idx = atp->index();
1337 Node* result = nullptr;
1338 Node* current = st;
1339 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1340 // this load. Stop once any other kind of node is encountered.
1341 //
1342 // In principle, folding a load is moving it up until it meets a matching store.
1343 //
1344 // store(ptr, v); store(ptr, v); store(ptr, v);
1345 // membar1; -> membar1; -> load(ptr);
1346 // membar2; load(ptr); membar1;
1347 // load(ptr); membar2; membar2;
1348 //
1349 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1350 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1351 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1363 MergeMemNode* merge = mem->as_MergeMem();
1364 Node* new_st = merge->memory_at(alias_idx);
1365 if (new_st == merge->base_memory()) {
1366 // Keep searching
1367 current = new_st;
1368 continue;
1369 }
1370 // Save the new memory state for the slice and fall through
1371 // to exit.
1372 result = new_st;
1373 }
1374 }
1375 break;
1376 }
1377 if (result != nullptr) {
1378 st = result;
1379 }
1380 }
1381
1382 Node* res = can_see_stored_value(st, phase);
1383 // TODO: reimplement assert, see: JDK-8386157
1384 //assert(res == nullptr || is_java_primitive(value_basic_type()) || res->bottom_type()->higher_equal(type()), "the fold is unsafe");
1385 return res;
1386 }
1387
1388 // If st is a store to the same location as this, return the stored value
1389 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1390 Node* ld_adr = in(MemNode::Address);
1391 intptr_t ld_off = 0;
1392 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1393 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1394 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1395
1396 // Loop around twice in the case Load -> Initialize -> Store.
1397 // (See PhaseIterGVN::add_users_to_worklist, which knows about this case.)
1398 for (int trip = 0; trip <= 1; trip++) {
1399
1400 if (st->is_Store()) {
1401 Node* st_adr = st->in(MemNode::Address);
1402 if (st_adr != ld_adr) {
1403 // Try harder before giving up. Unify base pointers with casts (e.g., raw/non-raw pointers).
1404 intptr_t st_off = 0;
1449 if (is_Store() || is_java_primitive(value_basic_type()) || res->bottom_type()->higher_equal(bottom_type())) {
1450 return res;
1451 }
1452
1453 // Type-unsafe stores must be due to array polymorphism
1454 const TypePtr* adr_type = this->adr_type();
1455 assert(adr_type == nullptr || adr_type->isa_aryptr() != nullptr, "unexpected type-unsafe store");
1456 return nullptr;
1457 }
1458
1459 // A load from a freshly-created object always returns zero.
1460 // (This can happen after LoadNode::Ideal resets the load's memory input
1461 // to find_captured_store, which returned InitializeNode::zero_memory.)
1462 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1463 (st->in(0) == ld_alloc) &&
1464 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1465 // return a zero value for the load's basic type
1466 // (This is one of the few places where a generic PhaseTransform
1467 // can create new nodes. Think of it as lazily manifesting
1468 // virtually pre-existing constants.)
1469 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1470 if (init_value != nullptr) {
1471 const TypeAryPtr* ld_adr_type = phase->type(ld_adr)->isa_aryptr();
1472 if (ld_adr_type == nullptr) {
1473 return nullptr;
1474 }
1475
1476 // We know that this is not a flat array, the load should return the whole oop
1477 if (ld_adr_type->is_not_flat()) {
1478 return init_value;
1479 }
1480
1481 // If this is a flat array, try to see through init_value
1482 if (init_value->is_EncodeP()) {
1483 init_value = init_value->in(1);
1484 }
1485 if (!init_value->is_InlineType() || ld_adr_type->field_offset() == Type::Offset::bottom) {
1486 return nullptr;
1487 }
1488
1489 ciInlineKlass* vk = phase->type(init_value)->inline_klass();
1490 int field_offset_in_payload = ld_adr_type->field_offset().get();
1491 if (field_offset_in_payload == vk->null_marker_offset_in_payload()) {
1492 return init_value->as_InlineType()->get_null_marker();
1493 } else {
1494 return init_value->as_InlineType()->field_value_by_offset(field_offset_in_payload + vk->payload_offset(), true);
1495 }
1496 }
1497 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1498 if (value_basic_type() != T_VOID) {
1499 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1500 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1501 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1502 // by the ArrayCopyNode.
1503 return phase->zerocon(value_basic_type());
1504 }
1505 } else {
1506 // TODO: materialize all-zero vector constant
1507 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1508 }
1509 }
1510
1511 // A load from an initialization barrier can match a captured store.
1512 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1513 InitializeNode* init = st->in(0)->as_Initialize();
1514 AllocateNode* alloc = init->allocation();
1515 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1516 // examine a captured store value
1517 st = init->find_captured_store(ld_off, memory_size(), phase);
1530 base = bs->step_over_gc_barrier(base);
1531 if (base != nullptr && base->is_Proj() &&
1532 base->as_Proj()->_con == TypeFunc::Parms &&
1533 base->in(0)->is_CallStaticJava() &&
1534 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1535 return base->in(0)->in(TypeFunc::Parms);
1536 }
1537 }
1538
1539 break;
1540 }
1541
1542 return nullptr;
1543 }
1544
1545 //----------------------is_instance_field_load_with_local_phi------------------
1546 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1547 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1548 in(Address)->is_AddP() ) {
1549 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1550 // Only known instances and immutable fields
1551 if( t_oop != nullptr &&
1552 (t_oop->is_ptr_to_strict_final_field() ||
1553 t_oop->is_known_instance_field()) &&
1554 t_oop->offset() != Type::OffsetBot &&
1555 t_oop->offset() != Type::OffsetTop) {
1556 return true;
1557 }
1558 }
1559 return false;
1560 }
1561
1562 //------------------------------Identity---------------------------------------
1563 // Loads are identity if previous store is to same address
1564 Node* LoadNode::Identity(PhaseGVN* phase) {
1565 // If the previous store-maker is the right kind of Store, and the store is
1566 // to the same address, then we are equal to the value stored.
1567 Node* mem = in(Memory);
1568 Node* value = can_see_stored_value_through_membars(mem, phase);
1569 if( value ) {
1570 // byte, short & char stores truncate naturally.
1571 // A load has to load the truncated value which requires
1572 // some sort of masking operation and that requires an
1573 // Ideal call instead of an Identity call.
1574 if (memory_size() < BytesPerInt) {
1575 // If the input to the store does not fit with the load's result type,
1576 // it must be truncated via an Ideal call.
1577 if (!phase->type(value)->higher_equal(phase->type(this)))
1578 return this;
1579 }
1580
1581 if (phase->type(value)->isa_ptr() && phase->type(this)->isa_narrowoop()) {
1582 return this;
1583 }
1584 // (This works even when value is a Con, but LoadNode::Value
1585 // usually runs first, producing the singleton type of the Con.)
1586 if (!has_pinned_control_dependency() || value->is_Con()) {
1587 return value;
1588 } else {
1589 return this;
1590 }
1591 }
1592
1593 if (has_pinned_control_dependency()) {
1594 return this;
1595 }
1596 // Search for an existing data phi which was generated before for the same
1597 // instance's field to avoid infinite generation of phis in a loop.
1598 Node *region = mem->in(0);
1599 if (is_instance_field_load_with_local_phi(region)) {
1600 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1601 int this_index = phase->C->get_alias_index(addr_t);
1602 int this_offset = addr_t->offset();
1603 int this_iid = addr_t->instance_id();
1604 if (!addr_t->is_known_instance() &&
1605 addr_t->is_ptr_to_strict_final_field()) {
1606 // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1607 intptr_t ignore = 0;
1608 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1609 if (base == nullptr) {
1610 return this;
1611 }
1612 this_iid = base->_idx;
1613 }
1614 const Type* this_type = bottom_type();
1615 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1616 Node* phi = region->fast_out(i);
1617 if (phi->is_Phi() && phi != mem &&
1618 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1619 return phi;
1620 }
1621 }
1622 }
1623
1624 return this;
1625 }
1626
2161 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2162 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2163
2164 // Skip up past a SafePoint control. Cannot do this for Stores because
2165 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2166 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2167 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2168 !addr_mark &&
2169 (depends_only_on_test() || has_unknown_control_dependency())) {
2170 ctrl = ctrl->in(0);
2171 set_req(MemNode::Control,ctrl);
2172 return this;
2173 }
2174
2175 intptr_t ignore = 0;
2176 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2177 if (base != nullptr
2178 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2179 // Check for useless control edge in some common special cases
2180 if (in(MemNode::Control) != nullptr
2181 // TODO 8350865 Can we re-enable this?
2182 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2183 && can_remove_control()
2184 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2185 && all_controls_dominate(base, phase->C->start(), phase)) {
2186 // A method-invariant, non-null address (constant or 'this' argument).
2187 set_req(MemNode::Control, nullptr);
2188 return this;
2189 }
2190 }
2191
2192 Node* mem = in(MemNode::Memory);
2193 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2194
2195 if (can_reshape && (addr_t != nullptr)) {
2196 // try to optimize our memory input
2197 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2198 if (opt_mem != mem) {
2199 set_req_X(MemNode::Memory, opt_mem, phase);
2200 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2201 return this;
2202 }
2315 // No match.
2316 return nullptr;
2317 }
2318
2319 //------------------------------Value-----------------------------------------
2320 const Type* LoadNode::Value(PhaseGVN* phase) const {
2321 // Either input is TOP ==> the result is TOP
2322 Node* mem = in(MemNode::Memory);
2323 const Type *t1 = phase->type(mem);
2324 if (t1 == Type::TOP) return Type::TOP;
2325 Node* adr = in(MemNode::Address);
2326 const TypePtr* tp = phase->type(adr)->isa_ptr();
2327 if (tp == nullptr || tp->empty()) return Type::TOP;
2328 int off = tp->offset();
2329 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2330 Compile* C = phase->C;
2331
2332 // If load can see a previous constant store, use that.
2333 Node* value = can_see_stored_value_through_membars(mem, phase);
2334 if (value != nullptr && value->is_Con()) {
2335 if (phase->type(value)->isa_ptr() && _type->isa_narrowoop()) {
2336 return phase->type(value)->make_narrowoop();
2337 } else {
2338 assert(value->bottom_type()->higher_equal(_type), "sanity");
2339 return phase->type(value);
2340 }
2341 }
2342 // Try to guess loaded type from pointer type
2343 if (tp->isa_aryptr()) {
2344 const TypeAryPtr* ary = tp->is_aryptr();
2345 const Type* t = ary->elem();
2346
2347 // Determine whether the reference is beyond the header or not, by comparing
2348 // the offset against the offset of the start of the array's data.
2349 // Different array types begin at slightly different offsets (12 vs. 16).
2350 // We choose T_BYTE as an example base type that is least restrictive
2351 // as to alignment, which will therefore produce the smallest
2352 // possible base offset.
2353 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2354 const bool off_beyond_header = (off >= min_base_off);
2355
2356 // Try to constant-fold a stable array element.
2357 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2358 // Make sure the reference is not into the header and the offset is constant
2359 ciObject* aobj = ary->const_oop();
2360 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2361 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2362 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off, ary->field_offset().get(),
2363 stable_dimension,
2364 value_basic_type(), is_unsigned());
2365 if (con_type != nullptr) {
2366 return con_type;
2367 }
2368 }
2369 }
2370
2371 // Don't do this for integer types. There is only potential profit if
2372 // the element type t is lower than _type; that is, for int types, if _type is
2373 // more restrictive than t. This only happens here if one is short and the other
2374 // char (both 16 bits), and in those cases we've made an intentional decision
2375 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2376 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2377 //
2378 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2379 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2380 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2381 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2382 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2383 // In fact, that could have been the original type of p1, and p1 could have
2384 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2385 // expression (LShiftL quux 3) independently optimized to the constant 8.
2386 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2387 && (_type->isa_vect() == nullptr)
2388 && !ary->is_flat()
2389 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2390 // t might actually be lower than _type, if _type is a unique
2391 // concrete subclass of abstract class t.
2392 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2393 const Type* jt = t->join_speculative(_type);
2394 // In any case, do not allow the join, per se, to empty out the type.
2395 if (jt->empty() && !t->empty()) {
2396 // This can happen if a interface-typed array narrows to a class type.
2397 jt = _type;
2398 }
2399 #ifdef ASSERT
2400 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2401 // The pointers in the autobox arrays are always non-null
2402 Node* base = adr->in(AddPNode::Base);
2403 if ((base != nullptr) && base->is_DecodeN()) {
2404 // Get LoadN node which loads IntegerCache.cache field
2405 base = base->in(1);
2406 }
2407 if ((base != nullptr) && base->is_Con()) {
2408 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2409 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2410 // It could be narrow oop
2411 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2412 }
2413 }
2414 }
2415 #endif
2416 return jt;
2417 }
2418 }
2419 } else if (tp->base() == Type::InstPtr) {
2420 assert( off != Type::OffsetBot ||
2421 // arrays can be cast to Objects
2422 !tp->isa_instptr() ||
2423 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2424 // Default value load
2425 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2426 // unsafe field access may not have a constant offset
2427 is_unsafe_access(),
2428 "Field accesses must be precise" );
2429 // For oop loads, we expect the _type to be precise.
2430
2431 const TypeInstPtr* tinst = tp->is_instptr();
2432 BasicType bt = value_basic_type();
2433
2434 // Fold loads of the field map
2435 if (tinst != nullptr) {
2436 ciInstanceKlass* ik = tinst->instance_klass();
2437 int offset = tinst->offset();
2438 if (ik == phase->C->env()->Class_klass()) {
2439 ciType* t = tinst->java_mirror_type();
2440 if (t != nullptr && t->is_inlinetype() && offset == t->as_inline_klass()->field_map_offset()) {
2441 ciConstant map = t->as_inline_klass()->get_field_map();
2442 bool is_narrow_oop = (bt == T_NARROWOOP);
2443 return Type::make_from_constant(map, true, 1, is_narrow_oop);
2444 }
2445 }
2446 }
2447
2448 // Optimize loads from constant fields.
2449 ciObject* const_oop = tinst->const_oop();
2450 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2451 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2452 if (con_type != nullptr) {
2453 return con_type;
2454 }
2455 }
2456 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2457 assert(off != Type::OffsetBot ||
2458 !tp->isa_instklassptr() ||
2459 // arrays can be cast to Objects
2460 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2461 // also allow array-loading from the primary supertype
2462 // array during subtype checks
2463 Opcode() == Op_LoadKlass,
2464 "Field accesses must be precise");
2465 // For klass/static loads, we expect the _type to be precise
2466 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2467 /* With mirrors being an indirect in the Klass*
2468 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2469 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2470 *
2471 * So check the type and klass of the node before the LoadP.
2478 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2479 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2480 return TypeInstPtr::make(klass->java_mirror());
2481 }
2482 }
2483 }
2484
2485 const TypeKlassPtr *tkls = tp->isa_klassptr();
2486 if (tkls != nullptr) {
2487 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2488 ciKlass* klass = tkls->exact_klass();
2489 // We are loading a field from a Klass metaobject whose identity
2490 // is known at compile time (the type is "exact" or "precise").
2491 // Check for fields we know are maintained as constants by the VM.
2492 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2493 // The field is Klass::_super_check_offset. Return its (constant) value.
2494 // (Folds up type checking code.)
2495 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2496 return TypeInt::make(klass->super_check_offset());
2497 }
2498 if (klass->is_inlinetype() && tkls->offset() == in_bytes(InstanceKlass::acmp_maps_offset_offset())) {
2499 return TypeInt::make(klass->as_inline_klass()->field_map_offset());
2500 }
2501 if (klass->is_obj_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset())) {
2502 // Fold loads from LibraryCallKit::load_default_refined_array_klass
2503 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr();
2504 }
2505 if (klass->is_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::properties_offset())) {
2506 assert(klass->is_type_array_klass() || tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2507 return TypeInt::make((jint)klass->as_array_klass()->properties().value());
2508 }
2509 if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2510 assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2511 return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2512 }
2513 if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2514 // The field is Klass::_prototype_header. Return its (constant) value.
2515 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2516 return TypeX::make(klass->prototype_header());
2517 }
2518 // Compute index into primary_supers array
2519 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2520 // Check for overflowing; use unsigned compare to handle the negative case.
2521 if( depth < ciKlass::primary_super_limit() ) {
2522 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2523 // (Folds up type checking code.)
2524 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2525 ciKlass *ss = klass->super_of_depth(depth);
2526 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2527 }
2528 const Type* aift = load_array_final_field(tkls, klass);
2529 if (aift != nullptr) return aift;
2530 }
2531
2532 // We can still check if we are loading from the primary_supers array at a
2533 // shallow enough depth. Even though the klass is not exact, entries less
2534 // than or equal to its super depth are correct.
2535 if (tkls->is_loaded()) {
2536 ciKlass* klass = nullptr;
2570 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2571 // The key property of this type is that it folds up tests
2572 // for array-ness, since it proves that the layout_helper is positive.
2573 // Thus, a generic value like the basic object layout helper works fine.
2574 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2575 }
2576 }
2577
2578 // If we are loading from a freshly-allocated object/array, produce a zero.
2579 // Things to check:
2580 // 1. Load is beyond the header: headers are not guaranteed to be zero
2581 // 2. Load is not vectorized: vectors have no zero constant
2582 // 3. Load has no matching store, i.e. the input is the initial memory state
2583 const TypeOopPtr* tinst = tp->isa_oopptr();
2584 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2585 bool is_not_vect = (_type->isa_vect() == nullptr);
2586 if (is_not_header && is_not_vect) {
2587 Node* mem = in(MemNode::Memory);
2588 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2589 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2590 // TODO 8350865 Scalar replacement does not work well for flat arrays.
2591 // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2592 // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2593 // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2594 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2595 intptr_t offset = 0;
2596 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2597 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2598 if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2599 return _type;
2600 }
2601 }
2602 return Type::get_zero_type(_type->basic_type());
2603 }
2604 }
2605 if (!UseCompactObjectHeaders) {
2606 Node* alloc = is_new_object_mark_load();
2607 if (alloc != nullptr) {
2608 if (Arguments::is_valhalla_enabled()) {
2609 // The mark word may contain property bits (inline, flat, null-free)
2610 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2611 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2612 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2613 return TypeX::make(tkls->exact_klass()->prototype_header());
2614 }
2615 } else {
2616 return TypeX::make(markWord::prototype().value());
2617 }
2618 }
2619 }
2620
2621 return _type;
2622 }
2623
2624 //------------------------------match_edge-------------------------------------
2625 // Do we Match on this edge index or not? Match only the address.
2626 uint LoadNode::match_edge(uint idx) const {
2627 return idx == MemNode::Address;
2628 }
2629
2630 //--------------------------LoadBNode::Ideal--------------------------------------
2631 //
2632 // If the previous store is to the same address as this load,
2633 // and the value stored was larger than a byte, replace this load
2634 // with the value stored truncated to a byte. If no truncation is
2635 // needed, the replacement is done in LoadNode::Identity().
2636 //
2637 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2746 }
2747 }
2748 // Identity call will handle the case where truncation is not needed.
2749 return LoadNode::Ideal(phase, can_reshape);
2750 }
2751
2752 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2753 Node* mem = in(MemNode::Memory);
2754 Node* value = can_see_stored_value_through_membars(mem, phase);
2755 if (value != nullptr && value->is_Con() &&
2756 !value->bottom_type()->higher_equal(_type)) {
2757 // If the input to the store does not fit with the load's result type,
2758 // it must be truncated. We can't delay until Ideal call since
2759 // a singleton Value is needed for split_thru_phi optimization.
2760 int con = value->get_int();
2761 return TypeInt::make((con << 16) >> 16);
2762 }
2763 return LoadNode::Value(phase);
2764 }
2765
2766 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2767 // Can see the corresponding value, may need to add an EncodeP
2768 Node* value = can_see_stored_value_through_membars(in(Memory), phase);
2769 if (value != nullptr && phase->type(value)->isa_ptr() && type()->isa_narrowoop()) {
2770 return new EncodePNode(value, type());
2771 }
2772
2773 // Identity call will handle the case where EncodeP is unnecessary
2774 return LoadNode::Ideal(phase, can_reshape);
2775 }
2776
2777 //=============================================================================
2778 //----------------------------LoadKlassNode::make------------------------------
2779 // Polymorphic factory method:
2780 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2781 // sanity check the alias category against the created node type
2782 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2783 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2784 #ifdef _LP64
2785 if (adr_type->is_ptr_to_narrowklass()) {
2786 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2787 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2788 }
2789 #endif
2790 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2791 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2792 }
2793
2794 //------------------------------Value------------------------------------------
2795 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2796 return klass_value_common(phase);
2829 }
2830 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2831 }
2832 if (!t->is_klass()) {
2833 // a primitive Class (e.g., int.class) has null for a klass field
2834 return TypePtr::NULL_PTR;
2835 }
2836 // Fold up the load of the hidden field
2837 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2838 }
2839 // non-constant mirror, so we can't tell what's going on
2840 }
2841 if (!tinst->is_loaded())
2842 return _type; // Bail out if not loaded
2843 if (offset == oopDesc::klass_offset_in_bytes()) {
2844 return tinst->as_klass_type(true);
2845 }
2846 }
2847
2848 // Check for loading klass from an array
2849 const TypeAryPtr* tary = tp->isa_aryptr();
2850 if (tary != nullptr &&
2851 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2852 return tary->as_klass_type(true)->is_aryklassptr();
2853 }
2854
2855 // Check for loading klass from an array klass
2856 const TypeKlassPtr *tkls = tp->isa_klassptr();
2857 if (tkls != nullptr && !StressReflectiveCode) {
2858 if (!tkls->is_loaded())
2859 return _type; // Bail out if not loaded
2860 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2861 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2862 // // Always returning precise element type is incorrect,
2863 // // e.g., element type could be object and array may contain strings
2864 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2865
2866 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2867 // according to the element type's subclassing.
2868 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2869 }
2870 if (tkls->isa_aryklassptr() != nullptr && tkls->klass_is_exact() &&
2871 !tkls->exact_klass()->is_type_array_klass() &&
2872 tkls->offset() == in_bytes(Klass::super_offset())) {
2873 // We are loading the super klass of a refined array klass, return the non-refined klass pointer
2874 assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2875 return tkls->is_aryklassptr()->with_offset(0)->cast_to_non_refined();
2876 }
2877 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2878 tkls->offset() == in_bytes(Klass::super_offset())) {
2879 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2880 // The field is Klass::_super. Return its (constant) value.
2881 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2882 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2883 }
2884 }
2885
2886 if (tkls != nullptr && !UseSecondarySupersCache
2887 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2888 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2889 return TypePtr::NULL_PTR;
2890 }
2891
2892 // Bailout case
2893 return LoadNode::Value(phase);
2894 }
2895
2896 //------------------------------Identity---------------------------------------
2919 base = bs->step_over_gc_barrier(base);
2920 }
2921
2922 // We can fetch the klass directly through an AllocateNode.
2923 // This works even if the klass is not constant (clone or newArray).
2924 if (offset == oopDesc::klass_offset_in_bytes()) {
2925 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2926 if (allocated_klass != nullptr) {
2927 return allocated_klass;
2928 }
2929 }
2930
2931 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2932 // See inline_native_Class_query for occurrences of these patterns.
2933 // Java Example: x.getClass().isAssignableFrom(y)
2934 //
2935 // This improves reflective code, often making the Class
2936 // mirror go completely dead. (Current exception: Class
2937 // mirrors may appear in debug info, but we could clean them out by
2938 // introducing a new debug info operator for Klass.java_mirror).
2939 //
2940 // This optimization does not apply to arrays because if k is not a
2941 // constant, it was obtained via load_klass which returns the refined type
2942 // and '.java_mirror.as_klass' should return the Java type instead.
2943
2944 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2945 && offset == java_lang_Class::klass_offset()) {
2946 if (base->is_Load()) {
2947 Node* base2 = base->in(MemNode::Address);
2948 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2949 Node* adr2 = base2->in(MemNode::Address);
2950 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2951 if (tkls != nullptr && !tkls->empty()
2952 && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2953 && adr2->is_AddP()) {
2954 int mirror_field = in_bytes(Klass::java_mirror_offset());
2955 if (tkls->offset() == mirror_field) {
2956 #ifdef ASSERT
2957 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2958 assert(tkls2->offset() == 0, "not a load of java_mirror");
2959 #endif
2960 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2961 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2962 return adr2->in(AddPNode::Address);
2963 }
2964 }
2965 }
2966 }
2967 }
2968
2969 return this;
2970 }
2971
2972 LoadNode* LoadNode::clone_pinned() const {
2973 LoadNode* ld = clone()->as_Load();
3100 // Polymorphic factory method:
3101 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
3102 assert((mo == unordered || mo == release), "unexpected");
3103 Compile* C = gvn.C;
3104 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
3105 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
3106 ctl != nullptr, "raw memory operations should have control edge");
3107
3108 switch (bt) {
3109 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
3110 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
3111 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
3112 case T_CHAR:
3113 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
3114 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3115 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
3116 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3117 case T_METADATA:
3118 case T_ADDRESS:
3119 case T_OBJECT:
3120 case T_ARRAY:
3121 #ifdef _LP64
3122 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3123 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
3124 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
3125 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
3126 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
3127 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
3128 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
3129 }
3130 #endif
3131 {
3132 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
3133 }
3134 default:
3135 guarantee(false, "unexpected basic type %s", type2name(bt));
3136 return (StoreNode*)nullptr;
3137 }
3138 }
3139
3140 //--------------------------bottom_type----------------------------------------
3141 const Type *StoreNode::bottom_type() const {
3142 return Type::MEMORY;
3143 }
3144
3145 //------------------------------hash-------------------------------------------
3146 uint StoreNode::hash() const {
3147 // unroll addition of interesting fields
3148 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
3149
3150 // Since they are not commoned, do not hash them:
3151 return NO_HASH;
3152 }
3153
3154 // Link together multiple stores (B/S/C/I) into a longer one.
3155 //
3777 }
3778 ss.print_cr("[TraceMergeStores]: with");
3779 merged_input_value->dump("\n", false, &ss);
3780 merged_store->dump("\n", false, &ss);
3781 tty->print("%s", ss.as_string());
3782 }
3783 #endif
3784
3785 //------------------------------Ideal------------------------------------------
3786 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3787 // When a store immediately follows a relevant allocation/initialization,
3788 // try to capture it into the initialization, or hoist it above.
3789 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3790 Node* p = MemNode::Ideal_common(phase, can_reshape);
3791 if (p) return (p == NodeSentinel) ? nullptr : p;
3792
3793 Node* mem = in(MemNode::Memory);
3794 Node* address = in(MemNode::Address);
3795 Node* value = in(MemNode::ValueIn);
3796 // Back-to-back stores to same address? Fold em up. Generally
3797 // unsafe if I have intervening uses...
3798 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3799 Node* st = mem;
3800 // If Store 'st' has more than one use, we cannot fold 'st' away.
3801 // For example, 'st' might be the final state at a conditional
3802 // return. Or, 'st' might be used by some node which is live at
3803 // the same time 'st' is live, which might be unschedulable. So,
3804 // require exactly ONE user until such time as we clone 'mem' for
3805 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3806 // true).
3807 while (st->is_Store() && st->outcnt() == 1) {
3808 // Looking at a dead closed cycle of memory?
3809 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3810 assert(Opcode() == st->Opcode() ||
3811 st->Opcode() == Op_StoreVector ||
3812 Opcode() == Op_StoreVector ||
3813 st->Opcode() == Op_StoreVectorScatter ||
3814 Opcode() == Op_StoreVectorScatter ||
3815 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3816 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3817 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3818 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3819 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3820 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3821
3822 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3823 st->as_Store()->memory_size() <= this->memory_size()) {
3824 Node* use = st->raw_out(0);
3825 if (phase->is_IterGVN()) {
3826 phase->is_IterGVN()->rehash_node_delayed(use);
3827 }
3828 // It's OK to do this in the parser, since DU info is always accurate,
3829 // and the parser always refers to nodes via SafePointNode maps.
3830 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3831 return this;
3832 }
3833 st = st->in(MemNode::Memory);
3834 }
3835 }
3836
3837
3838 // Capture an unaliased, unconditional, simple store into an initializer.
3939 const StoreVectorNode* store_vector = as_StoreVector();
3940 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3941 const Node* store_indices = store_vector->indices();
3942 const Node* mem_indices = mem_vector->indices();
3943 const Node* store_mask = store_vector->mask();
3944 const Node* mem_mask = mem_vector->mask();
3945 // Ensure types, indices, and masks match
3946 if (store_vector->vect_type() == mem_vector->vect_type() &&
3947 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3948 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3949 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3950 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3951 result = mem;
3952 }
3953 }
3954 }
3955
3956 // Store of zero anywhere into a freshly-allocated object?
3957 // Then the store is useless.
3958 // (It must already have been captured by the InitializeNode.)
3959 if (result == this && ReduceFieldZeroing) {
3960 // a newly allocated object is already all-zeroes everywhere
3961 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3962 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3963 result = mem;
3964 }
3965
3966 if (result == this && phase->type(val)->is_zero_type()) {
3967 // the store may also apply to zero-bits in an earlier object
3968 Node* prev_mem = find_previous_store(phase);
3969 // Steps (a), (b): Walk past independent stores to find an exact match.
3970 if (prev_mem != nullptr) {
3971 if (prev_mem->is_top()) {
3972 // find_previous_store returns top when the access is dead
3973 return prev_mem;
3974 }
3975 Node* prev_val = can_see_stored_value(prev_mem, phase);
3976 if (prev_val != nullptr && prev_val == val) {
3977 // prev_val and val might differ by a cast; it would be good
3978 // to keep the more informative of the two.
3979 result = mem;
3980 }
3981 }
3982 }
3983 }
3984
3985 PhaseIterGVN* igvn = phase->is_IterGVN();
3986 if (result != this && igvn != nullptr) {
4459 // Clearing a short array is faster with stores
4460 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4461 // Already know this is a large node, do not try to ideal it
4462 if (_is_large) return nullptr;
4463
4464 const int unit = BytesPerLong;
4465 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4466 if (!t) return nullptr;
4467 if (!t->is_con()) return nullptr;
4468 intptr_t raw_count = t->get_con();
4469 intptr_t size = raw_count;
4470 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4471 // Clearing nothing uses the Identity call.
4472 // Negative clears are possible on dead ClearArrays
4473 // (see jck test stmt114.stmt11402.val).
4474 if (size <= 0 || size % unit != 0) return nullptr;
4475 intptr_t count = size / unit;
4476 // Length too long; communicate this to matchers and assemblers.
4477 // Assemblers are responsible to produce fast hardware clears for it.
4478 if (size > InitArrayShortSize) {
4479 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4480 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4481 return nullptr;
4482 }
4483 if (!IdealizeClearArrayNode) return nullptr;
4484 Node *mem = in(1);
4485 if( phase->type(mem)==Type::TOP ) return nullptr;
4486 Node *adr = in(3);
4487 const Type* at = phase->type(adr);
4488 if( at==Type::TOP ) return nullptr;
4489 const TypePtr* atp = at->isa_ptr();
4490 // adjust atp to be the correct array element address type
4491 if (atp == nullptr) atp = TypePtr::BOTTOM;
4492 else atp = atp->add_offset(Type::OffsetBot);
4493 // Get base for derived pointer purposes
4494 if( adr->Opcode() != Op_AddP ) Unimplemented();
4495 Node *base = adr->in(1);
4496
4497 Node *val = in(4);
4498 Node *off = phase->MakeConX(BytesPerLong);
4499 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4500 count--;
4501 while (count--) {
4502 mem = phase->transform(mem);
4503 adr = phase->transform(AddPNode::make_with_base(base,adr,off));
4504 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4505 }
4506 return mem;
4507 }
4508
4509 //----------------------------step_through----------------------------------
4510 // Return allocation input memory edge if it is different instance
4511 // or itself if it is the one we are looking for.
4512 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4513 Node* n = *np;
4514 assert(n->is_ClearArray(), "sanity");
4515 intptr_t offset;
4516 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4517 // This method is called only before Allocate nodes are expanded
4518 // during macro nodes expansion. Before that ClearArray nodes are
4519 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4520 // Allocate nodes are expanded) which follows allocations.
4521 assert(alloc != nullptr, "should have allocation");
4522 if (alloc->_idx == instance_id) {
4523 // Can not bypass initialization of the instance we are looking for.
4524 return false;
4527 InitializeNode* init = alloc->initialization();
4528 if (init != nullptr)
4529 *np = init->in(TypeFunc::Memory);
4530 else
4531 *np = alloc->in(TypeFunc::Memory);
4532 return true;
4533 }
4534
4535 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4536 Node* base = dest;
4537 if (raw_base) {
4538 // May be called as part of the initialization of a just allocated object
4539 base = phase->C->top();
4540 }
4541 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4542 }
4543
4544 //----------------------------clear_memory-------------------------------------
4545 // Generate code to initialize object storage to zero.
4546 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4547 Node* val,
4548 Node* raw_val,
4549 intptr_t start_offset,
4550 Node* end_offset,
4551 bool raw_base,
4552 PhaseGVN* phase) {
4553 intptr_t offset = start_offset;
4554
4555 int unit = BytesPerLong;
4556 if ((offset % unit) != 0) {
4557 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4558 const TypePtr* atp = TypeRawPtr::BOTTOM;
4559 if (val != nullptr) {
4560 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4561 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4562 } else {
4563 assert(raw_val == nullptr, "val may not be null");
4564 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4565 }
4566 mem = phase->transform(mem);
4567 offset += BytesPerInt;
4568 }
4569 assert((offset % unit) == 0, "");
4570
4571 // Initialize the remaining stuff, if any, with a ClearArray.
4572 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, raw_base, phase);
4573 }
4574
4575 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4576 Node* raw_val,
4577 Node* start_offset,
4578 Node* end_offset,
4579 bool raw_base,
4580 PhaseGVN* phase) {
4581 if (start_offset == end_offset) {
4582 // nothing to do
4583 return mem;
4584 }
4585
4586 int unit = BytesPerLong;
4587 Node* zbase = start_offset;
4588 Node* zend = end_offset;
4589
4590 // Scale to the unit required by the CPU:
4591 if (!Matcher::init_array_count_is_in_bytes) {
4592 Node* shift = phase->intcon(exact_log2(unit));
4593 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4594 zend = phase->transform(new URShiftXNode(zend, shift) );
4595 }
4596
4597 // Bulk clear double-words
4598 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4599 Node* adr = make_address(dest, start_offset, raw_base, phase);
4600 if (raw_val == nullptr) {
4601 raw_val = phase->MakeConX(0);
4602 }
4603 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4604 return phase->transform(mem);
4605 }
4606
4607 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4608 Node* val,
4609 Node* raw_val,
4610 intptr_t start_offset,
4611 intptr_t end_offset,
4612 bool raw_base,
4613 PhaseGVN* phase) {
4614 if (start_offset == end_offset) {
4615 // nothing to do
4616 return mem;
4617 }
4618
4619 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4620 intptr_t done_offset = end_offset;
4621 if ((done_offset % BytesPerLong) != 0) {
4622 done_offset -= BytesPerInt;
4623 }
4624 if (done_offset > start_offset) {
4625 mem = clear_memory(ctl, mem, dest, val, raw_val,
4626 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4627 }
4628 if (done_offset < end_offset) { // emit the final 32-bit store
4629 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4630 const TypePtr* atp = TypeRawPtr::BOTTOM;
4631 if (val != nullptr) {
4632 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4633 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4634 } else {
4635 assert(raw_val == nullptr, "val may not be null");
4636 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4637 }
4638 mem = phase->transform(mem);
4639 done_offset += BytesPerInt;
4640 }
4641 assert(done_offset == end_offset, "");
4642 return mem;
4643 }
4644
4645 //=============================================================================
4646 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4647 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4648 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4649 #ifdef ASSERT
4650 , _pair_idx(0)
4651 #endif
4652 {
4653 init_class_id(Class_MemBar);
4654 Node* top = C->top();
4655 init_req(TypeFunc::I_O,top);
4656 init_req(TypeFunc::FramePtr,top);
4657 init_req(TypeFunc::ReturnAdr,top);
4766 PhaseIterGVN* igvn = phase->is_IterGVN();
4767 remove(igvn);
4768 // Must return either the original node (now dead) or a new node
4769 // (Do not return a top here, since that would break the uniqueness of top.)
4770 return new ConINode(TypeInt::ZERO);
4771 }
4772 }
4773 return progress ? this : nullptr;
4774 }
4775
4776 //------------------------------Value------------------------------------------
4777 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4778 if( !in(0) ) return Type::TOP;
4779 if( phase->type(in(0)) == Type::TOP )
4780 return Type::TOP;
4781 return TypeTuple::MEMBAR;
4782 }
4783
4784 //------------------------------match------------------------------------------
4785 // Construct projections for memory.
4786 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4787 switch (proj->_con) {
4788 case TypeFunc::Control:
4789 case TypeFunc::Memory:
4790 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4791 }
4792 ShouldNotReachHere();
4793 return nullptr;
4794 }
4795
4796 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4797 trailing->_kind = TrailingStore;
4798 leading->_kind = LeadingStore;
4799 #ifdef ASSERT
4800 trailing->_pair_idx = leading->_idx;
4801 leading->_pair_idx = leading->_idx;
4802 #endif
4803 }
4804
4805 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4806 trailing->_kind = TrailingLoadStore;
5053 return (req() > RawStores);
5054 }
5055
5056 void InitializeNode::set_complete(PhaseGVN* phase) {
5057 assert(!is_complete(), "caller responsibility");
5058 _is_complete = Complete;
5059
5060 // After this node is complete, it contains a bunch of
5061 // raw-memory initializations. There is no need for
5062 // it to have anything to do with non-raw memory effects.
5063 // Therefore, tell all non-raw users to re-optimize themselves,
5064 // after skipping the memory effects of this initialization.
5065 PhaseIterGVN* igvn = phase->is_IterGVN();
5066 if (igvn) igvn->add_users_to_worklist(this);
5067 }
5068
5069 // convenience function
5070 // return false if the init contains any stores already
5071 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
5072 InitializeNode* init = initialization();
5073 if (init == nullptr || init->is_complete()) {
5074 return false;
5075 }
5076 init->remove_extra_zeroes();
5077 // for now, if this allocation has already collected any inits, bail:
5078 if (init->is_non_zero()) return false;
5079 init->set_complete(phase);
5080 return true;
5081 }
5082
5083 void InitializeNode::remove_extra_zeroes() {
5084 if (req() == RawStores) return;
5085 Node* zmem = zero_memory();
5086 uint fill = RawStores;
5087 for (uint i = fill; i < req(); i++) {
5088 Node* n = in(i);
5089 if (n->is_top() || n == zmem) continue; // skip
5090 if (fill < i) set_req(fill, n); // compact
5091 ++fill;
5092 }
5093 // delete any empty spaces created:
5094 while (fill < req()) {
5095 del_req(fill);
5239 // store node that we'd like to capture. We need to check
5240 // the uses of the MergeMemNode.
5241 mems.push(n);
5242 }
5243 } else if (n->is_Mem()) {
5244 Node* other_adr = n->in(MemNode::Address);
5245 if (other_adr == adr) {
5246 failed = true;
5247 break;
5248 } else {
5249 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5250 if (other_t_adr != nullptr) {
5251 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5252 if (other_alias_idx == alias_idx) {
5253 // A load from the same memory slice as the store right
5254 // after the InitializeNode. We check the control of the
5255 // object/array that is loaded from. If it's the same as
5256 // the store control then we cannot capture the store.
5257 assert(!n->is_Store(), "2 stores to same slice on same control?");
5258 Node* base = other_adr;
5259 if (base->is_Phi()) {
5260 // In rare case, base may be a PhiNode and it may read
5261 // the same memory slice between InitializeNode and store.
5262 failed = true;
5263 break;
5264 }
5265 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5266 base = base->in(AddPNode::Base);
5267 if (base != nullptr) {
5268 base = base->uncast();
5269 if (base->is_Proj() && base->in(0) == alloc) {
5270 failed = true;
5271 break;
5272 }
5273 }
5274 }
5275 }
5276 }
5277 } else {
5278 failed = true;
5279 break;
5280 }
5281 }
5282 }
5283 }
5284 if (failed) {
5830 // z's_done 12 16 16 16 12 16 12
5831 // z's_needed 12 16 16 16 16 16 16
5832 // zsize 0 0 0 0 4 0 4
5833 if (next_full_store < 0) {
5834 // Conservative tack: Zero to end of current word.
5835 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5836 } else {
5837 // Zero to beginning of next fully initialized word.
5838 // Or, don't zero at all, if we are already in that word.
5839 assert(next_full_store >= zeroes_needed, "must go forward");
5840 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5841 zeroes_needed = next_full_store;
5842 }
5843 }
5844
5845 if (zeroes_needed > zeroes_done) {
5846 intptr_t zsize = zeroes_needed - zeroes_done;
5847 // Do some incremental zeroing on rawmem, in parallel with inits.
5848 zeroes_done = align_down(zeroes_done, BytesPerInt);
5849 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5850 allocation()->in(AllocateNode::InitValue),
5851 allocation()->in(AllocateNode::RawInitValue),
5852 zeroes_done, zeroes_needed,
5853 true,
5854 phase);
5855 zeroes_done = zeroes_needed;
5856 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5857 do_zeroing = false; // leave the hole, next time
5858 }
5859 }
5860
5861 // Collect the store and move on:
5862 phase->replace_input_of(st, MemNode::Memory, inits);
5863 inits = st; // put it on the linearized chain
5864 set_req(i, zmem); // unhook from previous position
5865
5866 if (zeroes_done == st_off)
5867 zeroes_done = next_init_off;
5868
5869 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5870
5871 #ifdef ASSERT
5892 remove_extra_zeroes(); // clear out all the zmems left over
5893 add_req(inits);
5894
5895 if (!(UseTLAB && ZeroTLAB)) {
5896 // If anything remains to be zeroed, zero it all now.
5897 zeroes_done = align_down(zeroes_done, BytesPerInt);
5898 // if it is the last unused 4 bytes of an instance, forget about it
5899 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5900 if (zeroes_done + BytesPerLong >= size_limit) {
5901 AllocateNode* alloc = allocation();
5902 assert(alloc != nullptr, "must be present");
5903 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5904 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5905 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5906 if (zeroes_done == k->layout_helper())
5907 zeroes_done = size_limit;
5908 }
5909 }
5910 if (zeroes_done < size_limit) {
5911 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5912 allocation()->in(AllocateNode::InitValue),
5913 allocation()->in(AllocateNode::RawInitValue),
5914 zeroes_done, size_in_bytes, true, phase);
5915 }
5916 }
5917
5918 set_complete(phase);
5919 return rawmem;
5920 }
5921
5922 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5923 auto replace_proj = [&](ProjNode* proj) {
5924 C->gvn_replace_by(proj, mem);
5925 return CONTINUE;
5926 };
5927 apply_to_projs(replace_proj, TypeFunc::Memory);
5928 }
5929
5930 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5931 DUIterator_Fast imax, i = fast_outs(imax);
5932 auto replace_proj = [&](ProjNode* proj) {
5933 igvn->replace_node(proj, mem);
6131 //------------------------------Identity---------------------------------------
6132 Node* MergeMemNode::Identity(PhaseGVN* phase) {
6133 // Identity if this merge point does not record any interesting memory
6134 // disambiguations.
6135 Node* base_mem = base_memory();
6136 Node* empty_mem = empty_memory();
6137 if (base_mem != empty_mem) { // Memory path is not dead?
6138 for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
6139 Node* mem = in(i);
6140 if (mem != empty_mem && mem != base_mem) {
6141 return this; // Many memory splits; no change
6142 }
6143 }
6144 }
6145 return base_mem; // No memory splits; ID on the one true input
6146 }
6147
6148 //------------------------------Ideal------------------------------------------
6149 // This method is invoked recursively on chains of MergeMem nodes
6150 Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) {
6151 if (Identity(phase) != this) {
6152 // Let Identity handle this case
6153 return nullptr;
6154 }
6155
6156 // Remove chain'd MergeMems
6157 //
6158 // This is delicate, because the each "in(i)" (i >= Raw) is interpreted
6159 // relative to the "in(Bot)". Since we are patching both at the same time,
6160 // we have to be careful to read each "in(i)" relative to the old "in(Bot)",
6161 // but rewrite each "in(i)" relative to the new "in(Bot)".
6162 Node *progress = nullptr;
6163
6164
6165 Node* old_base = base_memory();
6166 Node* empty_mem = empty_memory();
6167 if (old_base == empty_mem)
6168 return nullptr; // Dead memory path.
6169
6170 MergeMemNode* old_mbase;
6171 if (old_base != nullptr && old_base->is_MergeMem())
6172 old_mbase = old_base->as_MergeMem();
6173 else
6174 old_mbase = nullptr;
6175 Node* new_base = old_base;
|