6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/compile.hpp"
38 #include "opto/connode.hpp"
39 #include "opto/convertnode.hpp"
40 #include "opto/loopnode.hpp"
41 #include "opto/machnode.hpp"
42 #include "opto/matcher.hpp"
43 #include "opto/memnode.hpp"
44 #include "opto/mempointer.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/opcodes.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regalloc.hpp"
50 #include "opto/regmask.hpp"
51 #include "opto/rootnode.hpp"
52 #include "opto/traceMergeStoresTag.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/globalDefinitions.hpp"
57 #include "utilities/macros.hpp"
58 #include "utilities/powerOfTwo.hpp"
59 #include "utilities/vmError.hpp"
60
61 // Portions of code courtesy of Clifford Click
62
63 // Optimization - Graph Style
64
65 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
66
67 //=============================================================================
68 uint MemNode::size_of() const { return sizeof(*this); }
69
70 const TypePtr *MemNode::adr_type() const {
71 Node* adr = in(Address);
72 if (adr == nullptr) return nullptr; // node is dead
73 const TypePtr* cross_check = nullptr;
125 st->print(", idx=Bot;");
126 else if (atp->index() == Compile::AliasIdxTop)
127 st->print(", idx=Top;");
128 else if (atp->index() == Compile::AliasIdxRaw)
129 st->print(", idx=Raw;");
130 else {
131 ciField* field = atp->field();
132 if (field) {
133 st->print(", name=");
134 field->print_name_on(st);
135 }
136 st->print(", idx=%d;", atp->index());
137 }
138 }
139 }
140
141 extern void print_alias_types();
142
143 #endif
144
145 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
146 assert((t_oop != nullptr), "sanity");
147 bool is_instance = t_oop->is_known_instance_field();
148 bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
149 (load != nullptr) && load->is_Load() &&
150 (phase->is_IterGVN() != nullptr);
151 if (!(is_instance || is_boxed_value_load))
152 return mchain; // don't try to optimize non-instance types
153 uint instance_id = t_oop->instance_id();
154 Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
155 Node *prev = nullptr;
156 Node *result = mchain;
157 while (prev != result) {
158 prev = result;
159 if (result == start_mem)
160 break; // hit one of our sentinels
161 // skip over a call which does not affect this memory slice
162 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
163 Node *proj_in = result->in(0);
164 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
165 break; // hit one of our sentinels
166 } else if (proj_in->is_Call()) {
167 // ArrayCopyNodes processed here as well
168 CallNode *call = proj_in->as_Call();
169 if (!call->may_modify(t_oop, phase)) { // returns false for instances
170 result = call->in(TypeFunc::Memory);
171 }
172 } else if (proj_in->is_Initialize()) {
173 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
174 // Stop if this is the initialization for the object instance which
175 // contains this memory slice, otherwise skip over it.
176 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
177 break;
178 }
179 if (is_instance) {
180 result = proj_in->in(TypeFunc::Memory);
181 } else if (is_boxed_value_load) {
182 Node* klass = alloc->in(AllocateNode::KlassNode);
183 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
184 if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
185 result = proj_in->in(TypeFunc::Memory); // not related allocation
186 }
187 }
188 } else if (proj_in->is_MemBar()) {
189 ArrayCopyNode* ac = nullptr;
190 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
191 break;
192 }
193 result = proj_in->in(TypeFunc::Memory);
194 } else if (proj_in->is_top()) {
195 break; // dead code
196 } else {
197 assert(false, "unexpected projection");
198 }
199 } else if (result->is_ClearArray()) {
200 if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
201 // Can not bypass initialization of the instance
202 // we are looking for.
203 break;
204 }
205 // Otherwise skip it (the call updated 'result' value).
206 } else if (result->is_MergeMem()) {
207 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
208 }
209 }
210 return result;
211 }
212
213 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
214 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
215 if (t_oop == nullptr)
216 return mchain; // don't try to optimize non-oop types
217 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
218 bool is_instance = t_oop->is_known_instance_field();
219 PhaseIterGVN *igvn = phase->is_IterGVN();
220 if (is_instance && igvn != nullptr && result->is_Phi()) {
221 PhiNode *mphi = result->as_Phi();
222 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
223 const TypePtr *t = mphi->adr_type();
224 bool do_split = false;
225 // In the following cases, Load memory input can be further optimized based on
226 // its precise address type
227 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
228 do_split = true;
229 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
230 const TypeOopPtr* mem_t =
231 t->is_oopptr()->cast_to_exactness(true)
232 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
233 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
234 if (t_oop->isa_aryptr()) {
235 mem_t = mem_t->is_aryptr()
236 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
237 ->cast_to_size(t_oop->is_aryptr()->size())
238 ->with_offset(t_oop->is_aryptr()->offset())
239 ->is_aryptr();
240 }
241 do_split = mem_t == t_oop;
242 }
243 if (do_split) {
244 // clone the Phi with our address type
245 result = mphi->split_out_instance(t_adr, igvn);
246 } else {
247 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
248 }
249 }
250 return result;
251 }
252
253 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
254 uint alias_idx = phase->C->get_alias_index(tp);
255 Node *mem = mmem;
256 #ifdef ASSERT
257 {
258 // Check that current type is consistent with the alias index used during graph construction
259 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
260 bool consistent = adr_check == nullptr || adr_check->empty() ||
261 phase->C->must_alias(adr_check, alias_idx );
262 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
263 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
264 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
265 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
266 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
267 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
268 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
269 // don't assert if it is dead code.
270 consistent = true;
271 }
272 if( !consistent ) {
273 st->print("alias_idx==%d, adr_check==", alias_idx);
274 if( adr_check == nullptr ) {
275 st->print("null");
276 } else {
277 adr_check->dump();
278 }
279 st->cr();
280 print_alias_types();
281 assert(consistent, "adr_check must match alias idx");
282 }
283 }
284 #endif
587 }
588
589 // Find an arraycopy ac that produces the memory state represented by parameter mem.
590 // Return ac if
591 // (a) can_see_stored_value=true and ac must have set the value for this load or if
592 // (b) can_see_stored_value=false and ac could have set the value for this load or if
593 // (c) can_see_stored_value=false and ac cannot have set the value for this load.
594 // In case (c) change the parameter mem to the memory input of ac to skip it
595 // when searching stored value.
596 // Otherwise return null.
597 Node* LoadNode::find_previous_arraycopy(PhaseValues* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const {
598 ArrayCopyNode* ac = find_array_copy_clone(ld_alloc, mem);
599 if (ac != nullptr) {
600 Node* ld_addp = in(MemNode::Address);
601 Node* src = ac->in(ArrayCopyNode::Src);
602 const TypeAryPtr* ary_t = phase->type(src)->isa_aryptr();
603
604 // This is a load from a cloned array. The corresponding arraycopy ac must
605 // have set the value for the load and we can return ac but only if the load
606 // is known to be within bounds. This is checked below.
607 if (ary_t != nullptr && ld_addp->is_AddP()) {
608 Node* ld_offs = ld_addp->in(AddPNode::Offset);
609 BasicType ary_elem = ary_t->elem()->array_element_basic_type();
610 jlong header = arrayOopDesc::base_offset_in_bytes(ary_elem);
611 jlong elemsize = type2aelembytes(ary_elem);
612
613 const TypeX* ld_offs_t = phase->type(ld_offs)->isa_intptr_t();
614 const TypeInt* sizetype = ary_t->size();
615
616 if (ld_offs_t->_lo >= header && ld_offs_t->_hi < (sizetype->_lo * elemsize + header)) {
617 // The load is known to be within bounds. It receives its value from ac.
618 return ac;
619 }
620 // The load is known to be out-of-bounds.
621 }
622 // The load could be out-of-bounds. It must not be hoisted but must remain
623 // dependent on the runtime range check. This is achieved by returning null.
624 } else if (mem->is_Proj() && mem->in(0) != nullptr && mem->in(0)->is_ArrayCopy()) {
625 ArrayCopyNode* ac = mem->in(0)->as_ArrayCopy();
626
627 if (ac->is_arraycopy_validated() ||
997 in_bytes(JavaThread::vthread_offset()),
998 in_bytes(JavaThread::scopedValueCache_offset()),
999 };
1000
1001 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1002 if (offset == offsets[i]) {
1003 return true;
1004 }
1005 }
1006 }
1007
1008 return false;
1009 }
1010 #endif
1011
1012 //----------------------------LoadNode::make-----------------------------------
1013 // Polymorphic factory method:
1014 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1015 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1016 Compile* C = gvn.C;
1017 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
1018
1019 // sanity check the alias category against the created node type
1020 assert(!(adr_type->isa_oopptr() &&
1021 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1022 "use LoadKlassNode instead");
1023 assert(!(adr_type->isa_aryptr() &&
1024 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1025 "use LoadRangeNode instead");
1026 // Check control edge of raw loads
1027 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1028 // oop will be recorded in oop map if load crosses safepoint
1029 rt->isa_oopptr() || is_immutable_value(adr),
1030 "raw memory operations should have control edge");
1031 LoadNode* load = nullptr;
1032 switch (bt) {
1033 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1034 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1035 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1036 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1037 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1038 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1039 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1040 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1041 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1042 case T_OBJECT:
1043 case T_NARROWOOP:
1044 #ifdef _LP64
1045 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1046 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1047 } else
1048 #endif
1049 {
1050 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1051 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1052 }
1053 break;
1054 default:
1055 ShouldNotReachHere();
1056 break;
1057 }
1058 assert(load != nullptr, "LoadNode should have been created");
1059 if (unaligned) {
1060 load->set_unaligned_access();
1061 }
1062 if (mismatched) {
1063 load->set_mismatched_access();
1064 }
1065 if (unsafe) {
1066 load->set_unsafe_access();
1067 }
1068 load->set_barrier_data(barrier_data);
1069 if (load->Opcode() == Op_LoadN) {
1070 Node* ld = gvn.transform(load);
1071 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1072 }
1073
1074 return load;
1075 }
1076
1077 //------------------------------hash-------------------------------------------
1078 uint LoadNode::hash() const {
1079 // unroll addition of interesting fields
1080 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1081 }
1082
1083 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1084 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1085 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1086 bool is_stable_ary = FoldStableValues &&
1087 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1088 tp->isa_aryptr()->is_stable();
1089
1090 return (eliminate_boxing && non_volatile) || is_stable_ary;
1091 }
1092
1093 return false;
1094 }
1095
1096 // Is the value loaded previously stored by an arraycopy? If so return
1097 // a load node that reads from the source array so we may be able to
1098 // optimize out the ArrayCopy node later.
1099 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1100 Node* ld_adr = in(MemNode::Address);
1101 intptr_t ld_off = 0;
1102 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1103 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1104 if (ac != nullptr) {
1105 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1106
1107 Node* mem = ac->in(TypeFunc::Memory);
1108 Node* ctl = ac->in(0);
1109 Node* src = ac->in(ArrayCopyNode::Src);
1110
1118 if (ac->as_ArrayCopy()->is_clonebasic()) {
1119 assert(ld_alloc != nullptr, "need an alloc");
1120 assert(addp->is_AddP(), "address must be addp");
1121 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1122 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1123 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1124 addp->set_req(AddPNode::Base, src);
1125 addp->set_req(AddPNode::Address, src);
1126 } else {
1127 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1128 ac->as_ArrayCopy()->is_copyof_validated() ||
1129 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1130 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1131 addp->set_req(AddPNode::Base, src);
1132 addp->set_req(AddPNode::Address, src);
1133
1134 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1135 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1136 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1137
1138 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1139 uint shift = exact_log2(type2aelembytes(ary_elem));
1140
1141 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1142 #ifdef _LP64
1143 diff = phase->transform(new ConvI2LNode(diff));
1144 #endif
1145 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1146
1147 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1148 addp->set_req(AddPNode::Offset, offset);
1149 }
1150 addp = phase->transform(addp);
1151 #ifdef ASSERT
1152 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1153 ld->_adr_type = adr_type;
1154 #endif
1155 ld->set_req(MemNode::Address, addp);
1156 ld->set_req(0, ctl);
1157 ld->set_req(MemNode::Memory, mem);
1158 return ld;
1159 }
1160 return nullptr;
1161 }
1162
1163 // This routine exists to make sure this set of tests is done the same
1164 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1165 // will change the graph shape in a way which makes memory alive twice at the
1166 // same time (uses the Oracle model of aliasing), then some
1167 // LoadXNode::Identity will fold things back to the equivalence-class model
1168 // of aliasing.
1169 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1170 Node* ld_adr = in(MemNode::Address);
1171 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1172 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1173
1174 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1175 uint alias_idx = atp->index();
1176 Node* result = nullptr;
1177 Node* current = st;
1178 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1179 // this load. Stop once any other kind of node is encountered.
1180 //
1181 // In principle, folding a load is moving it up until it meets a matching store.
1182 //
1183 // store(ptr, v); store(ptr, v); store(ptr, v);
1184 // membar1; -> membar1; -> load(ptr);
1185 // membar2; load(ptr); membar1;
1186 // load(ptr); membar2; membar2;
1187 //
1188 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1189 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1190 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1202 MergeMemNode* merge = mem->as_MergeMem();
1203 Node* new_st = merge->memory_at(alias_idx);
1204 if (new_st == merge->base_memory()) {
1205 // Keep searching
1206 current = new_st;
1207 continue;
1208 }
1209 // Save the new memory state for the slice and fall through
1210 // to exit.
1211 result = new_st;
1212 }
1213 }
1214 break;
1215 }
1216 if (result != nullptr) {
1217 st = result;
1218 }
1219 }
1220
1221 Node* res = can_see_stored_value(st, phase);
1222 assert(res == nullptr || is_java_primitive(value_basic_type()) || res->bottom_type()->higher_equal(type()), "the fold is unsafe");
1223 return res;
1224 }
1225
1226 // If st is a store to the same location as this, return the stored value
1227 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1228 Node* ld_adr = in(MemNode::Address);
1229 intptr_t ld_off = 0;
1230 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1231 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1232 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1233
1234 // Loop around twice in the case Load -> Initialize -> Store.
1235 // (See PhaseIterGVN::add_users_to_worklist, which knows about this case.)
1236 for (int trip = 0; trip <= 1; trip++) {
1237
1238 if (st->is_Store()) {
1239 Node* st_adr = st->in(MemNode::Address);
1240 if (st_adr != ld_adr) {
1241 // Try harder before giving up. Unify base pointers with casts (e.g., raw/non-raw pointers).
1242 intptr_t st_off = 0;
1287 if (is_Store() || is_java_primitive(value_basic_type()) || res->bottom_type()->higher_equal(bottom_type())) {
1288 return res;
1289 }
1290
1291 // Type-unsafe stores must be due to array polymorphism
1292 const TypePtr* adr_type = this->adr_type();
1293 assert(adr_type == nullptr || adr_type->isa_aryptr() != nullptr, "unexpected type-unsafe store");
1294 return nullptr;
1295 }
1296
1297 // A load from a freshly-created object always returns zero.
1298 // (This can happen after LoadNode::Ideal resets the load's memory input
1299 // to find_captured_store, which returned InitializeNode::zero_memory.)
1300 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1301 (st->in(0) == ld_alloc) &&
1302 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1303 // return a zero value for the load's basic type
1304 // (This is one of the few places where a generic PhaseTransform
1305 // can create new nodes. Think of it as lazily manifesting
1306 // virtually pre-existing constants.)
1307 if (value_basic_type() != T_VOID) {
1308 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1309 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1310 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1311 // by the ArrayCopyNode.
1312 return phase->zerocon(value_basic_type());
1313 }
1314 } else {
1315 // TODO: materialize all-zero vector constant
1316 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1317 }
1318 }
1319
1320 // A load from an initialization barrier can match a captured store.
1321 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1322 InitializeNode* init = st->in(0)->as_Initialize();
1323 AllocateNode* alloc = init->allocation();
1324 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1325 // examine a captured store value
1326 st = init->find_captured_store(ld_off, memory_size(), phase);
1339 base = bs->step_over_gc_barrier(base);
1340 if (base != nullptr && base->is_Proj() &&
1341 base->as_Proj()->_con == TypeFunc::Parms &&
1342 base->in(0)->is_CallStaticJava() &&
1343 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1344 return base->in(0)->in(TypeFunc::Parms);
1345 }
1346 }
1347
1348 break;
1349 }
1350
1351 return nullptr;
1352 }
1353
1354 //----------------------is_instance_field_load_with_local_phi------------------
1355 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1356 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1357 in(Address)->is_AddP() ) {
1358 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1359 // Only instances and boxed values.
1360 if( t_oop != nullptr &&
1361 (t_oop->is_ptr_to_boxed_value() ||
1362 t_oop->is_known_instance_field()) &&
1363 t_oop->offset() != Type::OffsetBot &&
1364 t_oop->offset() != Type::OffsetTop) {
1365 return true;
1366 }
1367 }
1368 return false;
1369 }
1370
1371 //------------------------------Identity---------------------------------------
1372 // Loads are identity if previous store is to same address
1373 Node* LoadNode::Identity(PhaseGVN* phase) {
1374 // If the previous store-maker is the right kind of Store, and the store is
1375 // to the same address, then we are equal to the value stored.
1376 Node* mem = in(Memory);
1377 Node* value = can_see_stored_value_through_membars(mem, phase);
1378 if( value ) {
1379 // byte, short & char stores truncate naturally.
1380 // A load has to load the truncated value which requires
1381 // some sort of masking operation and that requires an
1382 // Ideal call instead of an Identity call.
1383 if (memory_size() < BytesPerInt) {
1384 // If the input to the store does not fit with the load's result type,
1385 // it must be truncated via an Ideal call.
1386 if (!phase->type(value)->higher_equal(phase->type(this)))
1387 return this;
1388 }
1389 // (This works even when value is a Con, but LoadNode::Value
1390 // usually runs first, producing the singleton type of the Con.)
1391 if (!has_pinned_control_dependency() || value->is_Con()) {
1392 return value;
1393 } else {
1394 return this;
1395 }
1396 }
1397
1398 if (has_pinned_control_dependency()) {
1399 return this;
1400 }
1401 // Search for an existing data phi which was generated before for the same
1402 // instance's field to avoid infinite generation of phis in a loop.
1403 Node *region = mem->in(0);
1404 if (is_instance_field_load_with_local_phi(region)) {
1405 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1406 int this_index = phase->C->get_alias_index(addr_t);
1407 int this_offset = addr_t->offset();
1408 int this_iid = addr_t->instance_id();
1409 if (!addr_t->is_known_instance() &&
1410 addr_t->is_ptr_to_boxed_value()) {
1411 // Use _idx of address base (could be Phi node) for boxed values.
1412 intptr_t ignore = 0;
1413 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1414 if (base == nullptr) {
1415 return this;
1416 }
1417 this_iid = base->_idx;
1418 }
1419 const Type* this_type = bottom_type();
1420 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1421 Node* phi = region->fast_out(i);
1422 if (phi->is_Phi() && phi != mem &&
1423 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1424 return phi;
1425 }
1426 }
1427 }
1428
1429 return this;
1430 }
1431
1966 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1967 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1968
1969 // Skip up past a SafePoint control. Cannot do this for Stores because
1970 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1971 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1972 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1973 !addr_mark &&
1974 (depends_only_on_test() || has_unknown_control_dependency())) {
1975 ctrl = ctrl->in(0);
1976 set_req(MemNode::Control,ctrl);
1977 return this;
1978 }
1979
1980 intptr_t ignore = 0;
1981 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1982 if (base != nullptr
1983 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1984 // Check for useless control edge in some common special cases
1985 if (in(MemNode::Control) != nullptr
1986 && can_remove_control()
1987 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1988 && all_controls_dominate(base, phase->C->start(), phase)) {
1989 // A method-invariant, non-null address (constant or 'this' argument).
1990 set_req(MemNode::Control, nullptr);
1991 return this;
1992 }
1993 }
1994
1995 Node* mem = in(MemNode::Memory);
1996 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1997
1998 if (can_reshape && (addr_t != nullptr)) {
1999 // try to optimize our memory input
2000 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2001 if (opt_mem != mem) {
2002 set_req_X(MemNode::Memory, opt_mem, phase);
2003 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2004 return this;
2005 }
2118 // No match.
2119 return nullptr;
2120 }
2121
2122 //------------------------------Value-----------------------------------------
2123 const Type* LoadNode::Value(PhaseGVN* phase) const {
2124 // Either input is TOP ==> the result is TOP
2125 Node* mem = in(MemNode::Memory);
2126 const Type *t1 = phase->type(mem);
2127 if (t1 == Type::TOP) return Type::TOP;
2128 Node* adr = in(MemNode::Address);
2129 const TypePtr* tp = phase->type(adr)->isa_ptr();
2130 if (tp == nullptr || tp->empty()) return Type::TOP;
2131 int off = tp->offset();
2132 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2133 Compile* C = phase->C;
2134
2135 // If load can see a previous constant store, use that.
2136 Node* value = can_see_stored_value_through_membars(mem, phase);
2137 if (value != nullptr && value->is_Con()) {
2138 assert(value->bottom_type()->higher_equal(_type), "sanity");
2139 return value->bottom_type();
2140 }
2141
2142 // Try to guess loaded type from pointer type
2143 if (tp->isa_aryptr()) {
2144 const TypeAryPtr* ary = tp->is_aryptr();
2145 const Type* t = ary->elem();
2146
2147 // Determine whether the reference is beyond the header or not, by comparing
2148 // the offset against the offset of the start of the array's data.
2149 // Different array types begin at slightly different offsets (12 vs. 16).
2150 // We choose T_BYTE as an example base type that is least restrictive
2151 // as to alignment, which will therefore produce the smallest
2152 // possible base offset.
2153 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2154 const bool off_beyond_header = (off >= min_base_off);
2155
2156 // Try to constant-fold a stable array element.
2157 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2158 // Make sure the reference is not into the header and the offset is constant
2159 ciObject* aobj = ary->const_oop();
2160 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2161 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2162 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off,
2163 stable_dimension,
2164 value_basic_type(), is_unsigned());
2165 if (con_type != nullptr) {
2166 return con_type;
2167 }
2168 }
2169 }
2170
2171 // Don't do this for integer types. There is only potential profit if
2172 // the element type t is lower than _type; that is, for int types, if _type is
2173 // more restrictive than t. This only happens here if one is short and the other
2174 // char (both 16 bits), and in those cases we've made an intentional decision
2175 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2176 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2177 //
2178 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2179 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2180 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2181 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2182 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2183 // In fact, that could have been the original type of p1, and p1 could have
2184 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2185 // expression (LShiftL quux 3) independently optimized to the constant 8.
2186 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2187 && (_type->isa_vect() == nullptr)
2188 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2189 // t might actually be lower than _type, if _type is a unique
2190 // concrete subclass of abstract class t.
2191 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2192 const Type* jt = t->join_speculative(_type);
2193 // In any case, do not allow the join, per se, to empty out the type.
2194 if (jt->empty() && !t->empty()) {
2195 // This can happen if a interface-typed array narrows to a class type.
2196 jt = _type;
2197 }
2198 #ifdef ASSERT
2199 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2200 // The pointers in the autobox arrays are always non-null
2201 Node* base = adr->in(AddPNode::Base);
2202 if ((base != nullptr) && base->is_DecodeN()) {
2203 // Get LoadN node which loads IntegerCache.cache field
2204 base = base->in(1);
2205 }
2206 if ((base != nullptr) && base->is_Con()) {
2207 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2208 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2209 // It could be narrow oop
2210 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2211 }
2212 }
2213 }
2214 #endif
2215 return jt;
2216 }
2217 }
2218 } else if (tp->base() == Type::InstPtr) {
2219 assert( off != Type::OffsetBot ||
2220 // arrays can be cast to Objects
2221 !tp->isa_instptr() ||
2222 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2223 // unsafe field access may not have a constant offset
2224 C->has_unsafe_access(),
2225 "Field accesses must be precise" );
2226 // For oop loads, we expect the _type to be precise.
2227
2228 // Optimize loads from constant fields.
2229 const TypeInstPtr* tinst = tp->is_instptr();
2230 ciObject* const_oop = tinst->const_oop();
2231 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2232 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2233 if (con_type != nullptr) {
2234 return con_type;
2235 }
2236 }
2237 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2238 assert(off != Type::OffsetBot ||
2239 !tp->isa_instklassptr() ||
2240 // arrays can be cast to Objects
2241 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2242 // also allow array-loading from the primary supertype
2243 // array during subtype checks
2244 Opcode() == Op_LoadKlass,
2245 "Field accesses must be precise");
2246 // For klass/static loads, we expect the _type to be precise
2247 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2248 /* With mirrors being an indirect in the Klass*
2249 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2250 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2251 *
2252 * So check the type and klass of the node before the LoadP.
2259 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2260 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2261 return TypeInstPtr::make(klass->java_mirror());
2262 }
2263 }
2264 }
2265
2266 const TypeKlassPtr *tkls = tp->isa_klassptr();
2267 if (tkls != nullptr) {
2268 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2269 ciKlass* klass = tkls->exact_klass();
2270 // We are loading a field from a Klass metaobject whose identity
2271 // is known at compile time (the type is "exact" or "precise").
2272 // Check for fields we know are maintained as constants by the VM.
2273 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2274 // The field is Klass::_super_check_offset. Return its (constant) value.
2275 // (Folds up type checking code.)
2276 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2277 return TypeInt::make(klass->super_check_offset());
2278 }
2279 if (UseCompactObjectHeaders) {
2280 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2281 // The field is Klass::_prototype_header. Return its (constant) value.
2282 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2283 return TypeX::make(klass->prototype_header());
2284 }
2285 }
2286 // Compute index into primary_supers array
2287 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2288 // Check for overflowing; use unsigned compare to handle the negative case.
2289 if( depth < ciKlass::primary_super_limit() ) {
2290 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2291 // (Folds up type checking code.)
2292 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2293 ciKlass *ss = klass->super_of_depth(depth);
2294 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2295 }
2296 const Type* aift = load_array_final_field(tkls, klass);
2297 if (aift != nullptr) return aift;
2298 }
2299
2300 // We can still check if we are loading from the primary_supers array at a
2301 // shallow enough depth. Even though the klass is not exact, entries less
2302 // than or equal to its super depth are correct.
2303 if (tkls->is_loaded()) {
2304 ciKlass* klass = nullptr;
2338 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2339 // The key property of this type is that it folds up tests
2340 // for array-ness, since it proves that the layout_helper is positive.
2341 // Thus, a generic value like the basic object layout helper works fine.
2342 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2343 }
2344 }
2345
2346 // If we are loading from a freshly-allocated object/array, produce a zero.
2347 // Things to check:
2348 // 1. Load is beyond the header: headers are not guaranteed to be zero
2349 // 2. Load is not vectorized: vectors have no zero constant
2350 // 3. Load has no matching store, i.e. the input is the initial memory state
2351 const TypeOopPtr* tinst = tp->isa_oopptr();
2352 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2353 bool is_not_vect = (_type->isa_vect() == nullptr);
2354 if (is_not_header && is_not_vect) {
2355 Node* mem = in(MemNode::Memory);
2356 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2357 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2358 return Type::get_zero_type(_type->basic_type());
2359 }
2360 }
2361
2362 if (!UseCompactObjectHeaders) {
2363 Node* alloc = is_new_object_mark_load();
2364 if (alloc != nullptr) {
2365 return TypeX::make(markWord::prototype().value());
2366 }
2367 }
2368
2369 return _type;
2370 }
2371
2372 //------------------------------match_edge-------------------------------------
2373 // Do we Match on this edge index or not? Match only the address.
2374 uint LoadNode::match_edge(uint idx) const {
2375 return idx == MemNode::Address;
2376 }
2377
2378 //--------------------------LoadBNode::Ideal--------------------------------------
2379 //
2380 // If the previous store is to the same address as this load,
2381 // and the value stored was larger than a byte, replace this load
2382 // with the value stored truncated to a byte. If no truncation is
2383 // needed, the replacement is done in LoadNode::Identity().
2384 //
2385 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2494 }
2495 }
2496 // Identity call will handle the case where truncation is not needed.
2497 return LoadNode::Ideal(phase, can_reshape);
2498 }
2499
2500 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2501 Node* mem = in(MemNode::Memory);
2502 Node* value = can_see_stored_value_through_membars(mem, phase);
2503 if (value != nullptr && value->is_Con() &&
2504 !value->bottom_type()->higher_equal(_type)) {
2505 // If the input to the store does not fit with the load's result type,
2506 // it must be truncated. We can't delay until Ideal call since
2507 // a singleton Value is needed for split_thru_phi optimization.
2508 int con = value->get_int();
2509 return TypeInt::make((con << 16) >> 16);
2510 }
2511 return LoadNode::Value(phase);
2512 }
2513
2514 //=============================================================================
2515 //----------------------------LoadKlassNode::make------------------------------
2516 // Polymorphic factory method:
2517 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2518 // sanity check the alias category against the created node type
2519 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2520 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2521 #ifdef _LP64
2522 if (adr_type->is_ptr_to_narrowklass()) {
2523 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2524 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2525 }
2526 #endif
2527 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2528 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2529 }
2530
2531 //------------------------------Value------------------------------------------
2532 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2533 return klass_value_common(phase);
2566 }
2567 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2568 }
2569 if (!t->is_klass()) {
2570 // a primitive Class (e.g., int.class) has null for a klass field
2571 return TypePtr::NULL_PTR;
2572 }
2573 // Fold up the load of the hidden field
2574 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2575 }
2576 // non-constant mirror, so we can't tell what's going on
2577 }
2578 if (!tinst->is_loaded())
2579 return _type; // Bail out if not loaded
2580 if (offset == oopDesc::klass_offset_in_bytes()) {
2581 return tinst->as_klass_type(true);
2582 }
2583 }
2584
2585 // Check for loading klass from an array
2586 const TypeAryPtr *tary = tp->isa_aryptr();
2587 if (tary != nullptr &&
2588 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2589 return tary->as_klass_type(true);
2590 }
2591
2592 // Check for loading klass from an array klass
2593 const TypeKlassPtr *tkls = tp->isa_klassptr();
2594 if (tkls != nullptr && !StressReflectiveCode) {
2595 if (!tkls->is_loaded())
2596 return _type; // Bail out if not loaded
2597 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2598 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2599 // // Always returning precise element type is incorrect,
2600 // // e.g., element type could be object and array may contain strings
2601 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2602
2603 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2604 // according to the element type's subclassing.
2605 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2606 }
2607 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2608 tkls->offset() == in_bytes(Klass::super_offset())) {
2609 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2610 // The field is Klass::_super. Return its (constant) value.
2611 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2612 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2613 }
2614 }
2615
2616 if (tkls != nullptr && !UseSecondarySupersCache
2617 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2618 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2619 return TypePtr::NULL_PTR;
2620 }
2621
2622 // Bailout case
2623 return LoadNode::Value(phase);
2624 }
2625
2626 //------------------------------Identity---------------------------------------
2649 base = bs->step_over_gc_barrier(base);
2650 }
2651
2652 // We can fetch the klass directly through an AllocateNode.
2653 // This works even if the klass is not constant (clone or newArray).
2654 if (offset == oopDesc::klass_offset_in_bytes()) {
2655 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2656 if (allocated_klass != nullptr) {
2657 return allocated_klass;
2658 }
2659 }
2660
2661 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2662 // See inline_native_Class_query for occurrences of these patterns.
2663 // Java Example: x.getClass().isAssignableFrom(y)
2664 //
2665 // This improves reflective code, often making the Class
2666 // mirror go completely dead. (Current exception: Class
2667 // mirrors may appear in debug info, but we could clean them out by
2668 // introducing a new debug info operator for Klass.java_mirror).
2669
2670 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2671 && offset == java_lang_Class::klass_offset()) {
2672 if (base->is_Load()) {
2673 Node* base2 = base->in(MemNode::Address);
2674 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2675 Node* adr2 = base2->in(MemNode::Address);
2676 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2677 if (tkls != nullptr && !tkls->empty()
2678 && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2679 && adr2->is_AddP()
2680 ) {
2681 int mirror_field = in_bytes(Klass::java_mirror_offset());
2682 if (tkls->offset() == mirror_field) {
2683 #ifdef ASSERT
2684 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2685 assert(tkls2->offset() == 0, "not a load of java_mirror");
2686 #endif
2687 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2688 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2689 return adr2->in(AddPNode::Address);
2690 }
2691 }
2692 }
2693 }
2694 }
2695
2696 return this;
2697 }
2698
2699 LoadNode* LoadNode::clone_pinned() const {
2700 LoadNode* ld = clone()->as_Load();
2827 // Polymorphic factory method:
2828 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2829 assert((mo == unordered || mo == release), "unexpected");
2830 Compile* C = gvn.C;
2831 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
2832 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2833 ctl != nullptr, "raw memory operations should have control edge");
2834
2835 switch (bt) {
2836 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2837 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2838 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2839 case T_CHAR:
2840 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2841 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2842 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2843 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2844 case T_METADATA:
2845 case T_ADDRESS:
2846 case T_OBJECT:
2847 #ifdef _LP64
2848 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2849 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2850 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2851 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2852 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
2853 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2854 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2855 }
2856 #endif
2857 {
2858 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2859 }
2860 default:
2861 ShouldNotReachHere();
2862 return (StoreNode*)nullptr;
2863 }
2864 }
2865
2866 //--------------------------bottom_type----------------------------------------
2867 const Type *StoreNode::bottom_type() const {
2868 return Type::MEMORY;
2869 }
2870
2871 //------------------------------hash-------------------------------------------
2872 uint StoreNode::hash() const {
2873 // unroll addition of interesting fields
2874 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2875
2876 // Since they are not commoned, do not hash them:
2877 return NO_HASH;
2878 }
2879
2880 // Link together multiple stores (B/S/C/I) into a longer one.
2881 //
3503 }
3504 ss.print_cr("[TraceMergeStores]: with");
3505 merged_input_value->dump("\n", false, &ss);
3506 merged_store->dump("\n", false, &ss);
3507 tty->print("%s", ss.as_string());
3508 }
3509 #endif
3510
3511 //------------------------------Ideal------------------------------------------
3512 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3513 // When a store immediately follows a relevant allocation/initialization,
3514 // try to capture it into the initialization, or hoist it above.
3515 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3516 Node* p = MemNode::Ideal_common(phase, can_reshape);
3517 if (p) return (p == NodeSentinel) ? nullptr : p;
3518
3519 Node* mem = in(MemNode::Memory);
3520 Node* address = in(MemNode::Address);
3521 Node* value = in(MemNode::ValueIn);
3522 // Back-to-back stores to same address? Fold em up. Generally
3523 // unsafe if I have intervening uses.
3524 {
3525 Node* st = mem;
3526 // If Store 'st' has more than one use, we cannot fold 'st' away.
3527 // For example, 'st' might be the final state at a conditional
3528 // return. Or, 'st' might be used by some node which is live at
3529 // the same time 'st' is live, which might be unschedulable. So,
3530 // require exactly ONE user until such time as we clone 'mem' for
3531 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3532 // true).
3533 while (st->is_Store() && st->outcnt() == 1) {
3534 // Looking at a dead closed cycle of memory?
3535 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3536 assert(Opcode() == st->Opcode() ||
3537 st->Opcode() == Op_StoreVector ||
3538 Opcode() == Op_StoreVector ||
3539 st->Opcode() == Op_StoreVectorScatter ||
3540 Opcode() == Op_StoreVectorScatter ||
3541 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3542 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3543 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3544 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3545 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3546
3547 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3548 st->as_Store()->memory_size() <= this->memory_size()) {
3549 Node* use = st->raw_out(0);
3550 if (phase->is_IterGVN()) {
3551 phase->is_IterGVN()->rehash_node_delayed(use);
3552 }
3553 // It's OK to do this in the parser, since DU info is always accurate,
3554 // and the parser always refers to nodes via SafePointNode maps.
3555 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3556 return this;
3557 }
3558 st = st->in(MemNode::Memory);
3559 }
3560 }
3561
3562
3563 // Capture an unaliased, unconditional, simple store into an initializer.
3664 const StoreVectorNode* store_vector = as_StoreVector();
3665 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3666 const Node* store_indices = store_vector->indices();
3667 const Node* mem_indices = mem_vector->indices();
3668 const Node* store_mask = store_vector->mask();
3669 const Node* mem_mask = mem_vector->mask();
3670 // Ensure types, indices, and masks match
3671 if (store_vector->vect_type() == mem_vector->vect_type() &&
3672 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3673 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3674 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3675 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3676 result = mem;
3677 }
3678 }
3679 }
3680
3681 // Store of zero anywhere into a freshly-allocated object?
3682 // Then the store is useless.
3683 // (It must already have been captured by the InitializeNode.)
3684 if (result == this &&
3685 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3686 // a newly allocated object is already all-zeroes everywhere
3687 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3688 result = mem;
3689 }
3690
3691 if (result == this) {
3692 // the store may also apply to zero-bits in an earlier object
3693 Node* prev_mem = find_previous_store(phase);
3694 // Steps (a), (b): Walk past independent stores to find an exact match.
3695 if (prev_mem != nullptr) {
3696 if (prev_mem->is_top()) {
3697 // find_previous_store returns top when the access is dead
3698 return prev_mem;
3699 }
3700 Node* prev_val = can_see_stored_value(prev_mem, phase);
3701 if (prev_val != nullptr && prev_val == val) {
3702 // prev_val and val might differ by a cast; it would be good
3703 // to keep the more informative of the two.
3704 result = mem;
3705 }
3706 }
3707 }
3708 }
3709
3710 PhaseIterGVN* igvn = phase->is_IterGVN();
3711 if (result != this && igvn != nullptr) {
4184 // Clearing a short array is faster with stores
4185 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4186 // Already know this is a large node, do not try to ideal it
4187 if (_is_large) return nullptr;
4188
4189 const int unit = BytesPerLong;
4190 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4191 if (!t) return nullptr;
4192 if (!t->is_con()) return nullptr;
4193 intptr_t raw_count = t->get_con();
4194 intptr_t size = raw_count;
4195 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4196 // Clearing nothing uses the Identity call.
4197 // Negative clears are possible on dead ClearArrays
4198 // (see jck test stmt114.stmt11402.val).
4199 if (size <= 0 || size % unit != 0) return nullptr;
4200 intptr_t count = size / unit;
4201 // Length too long; communicate this to matchers and assemblers.
4202 // Assemblers are responsible to produce fast hardware clears for it.
4203 if (size > InitArrayShortSize) {
4204 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4205 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4206 return nullptr;
4207 }
4208 if (!IdealizeClearArrayNode) return nullptr;
4209 Node *mem = in(1);
4210 if( phase->type(mem)==Type::TOP ) return nullptr;
4211 Node *adr = in(3);
4212 const Type* at = phase->type(adr);
4213 if( at==Type::TOP ) return nullptr;
4214 const TypePtr* atp = at->isa_ptr();
4215 // adjust atp to be the correct array element address type
4216 if (atp == nullptr) atp = TypePtr::BOTTOM;
4217 else atp = atp->add_offset(Type::OffsetBot);
4218 // Get base for derived pointer purposes
4219 if( adr->Opcode() != Op_AddP ) Unimplemented();
4220 Node *base = adr->in(1);
4221
4222 Node *zero = phase->makecon(TypeLong::ZERO);
4223 Node *off = phase->MakeConX(BytesPerLong);
4224 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4225 count--;
4226 while (count--) {
4227 mem = phase->transform(mem);
4228 adr = phase->transform(AddPNode::make_with_base(base, adr, off));
4229 mem = new StoreLNode(in(0), mem, adr, atp, zero, MemNode::unordered, false);
4230 }
4231 return mem;
4232 }
4233
4234 //----------------------------step_through----------------------------------
4235 // Return allocation input memory edge if it is different instance
4236 // or itself if it is the one we are looking for.
4237 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4238 Node* n = *np;
4239 assert(n->is_ClearArray(), "sanity");
4240 intptr_t offset;
4241 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4242 // This method is called only before Allocate nodes are expanded
4243 // during macro nodes expansion. Before that ClearArray nodes are
4244 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4245 // Allocate nodes are expanded) which follows allocations.
4246 assert(alloc != nullptr, "should have allocation");
4247 if (alloc->_idx == instance_id) {
4248 // Can not bypass initialization of the instance we are looking for.
4249 return false;
4252 InitializeNode* init = alloc->initialization();
4253 if (init != nullptr)
4254 *np = init->in(TypeFunc::Memory);
4255 else
4256 *np = alloc->in(TypeFunc::Memory);
4257 return true;
4258 }
4259
4260 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4261 Node* base = dest;
4262 if (raw_base) {
4263 // May be called as part of the initialization of a just allocated object
4264 base = phase->C->top();
4265 }
4266 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4267 }
4268
4269 //----------------------------clear_memory-------------------------------------
4270 // Generate code to initialize object storage to zero.
4271 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4272 intptr_t start_offset,
4273 Node* end_offset,
4274 bool raw_base,
4275 PhaseGVN* phase) {
4276 intptr_t offset = start_offset;
4277
4278 int unit = BytesPerLong;
4279 if ((offset % unit) != 0) {
4280 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4281 const TypePtr* atp = TypeRawPtr::BOTTOM;
4282 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4283 mem = phase->transform(mem);
4284 offset += BytesPerInt;
4285 }
4286 assert((offset % unit) == 0, "");
4287
4288 // Initialize the remaining stuff, if any, with a ClearArray.
4289 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, raw_base, phase);
4290 }
4291
4292 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4293 Node* start_offset,
4294 Node* end_offset,
4295 bool raw_base,
4296 PhaseGVN* phase) {
4297 if (start_offset == end_offset) {
4298 // nothing to do
4299 return mem;
4300 }
4301
4302 int unit = BytesPerLong;
4303 Node* zbase = start_offset;
4304 Node* zend = end_offset;
4305
4306 // Scale to the unit required by the CPU:
4307 if (!Matcher::init_array_count_is_in_bytes) {
4308 Node* shift = phase->intcon(exact_log2(unit));
4309 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4310 zend = phase->transform(new URShiftXNode(zend, shift) );
4311 }
4312
4313 // Bulk clear double-words
4314 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4315 Node* adr = make_address(dest, start_offset, raw_base, phase);
4316 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
4317 return phase->transform(mem);
4318 }
4319
4320 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4321 intptr_t start_offset,
4322 intptr_t end_offset,
4323 bool raw_base,
4324 PhaseGVN* phase) {
4325 if (start_offset == end_offset) {
4326 // nothing to do
4327 return mem;
4328 }
4329
4330 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4331 intptr_t done_offset = end_offset;
4332 if ((done_offset % BytesPerLong) != 0) {
4333 done_offset -= BytesPerInt;
4334 }
4335 if (done_offset > start_offset) {
4336 mem = clear_memory(ctl, mem, dest,
4337 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4338 }
4339 if (done_offset < end_offset) { // emit the final 32-bit store
4340 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4341 const TypePtr* atp = TypeRawPtr::BOTTOM;
4342 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4343 mem = phase->transform(mem);
4344 done_offset += BytesPerInt;
4345 }
4346 assert(done_offset == end_offset, "");
4347 return mem;
4348 }
4349
4350 //=============================================================================
4351 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4352 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4353 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4354 #ifdef ASSERT
4355 , _pair_idx(0)
4356 #endif
4357 {
4358 init_class_id(Class_MemBar);
4359 Node* top = C->top();
4360 init_req(TypeFunc::I_O,top);
4361 init_req(TypeFunc::FramePtr,top);
4362 init_req(TypeFunc::ReturnAdr,top);
4471 PhaseIterGVN* igvn = phase->is_IterGVN();
4472 remove(igvn);
4473 // Must return either the original node (now dead) or a new node
4474 // (Do not return a top here, since that would break the uniqueness of top.)
4475 return new ConINode(TypeInt::ZERO);
4476 }
4477 }
4478 return progress ? this : nullptr;
4479 }
4480
4481 //------------------------------Value------------------------------------------
4482 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4483 if( !in(0) ) return Type::TOP;
4484 if( phase->type(in(0)) == Type::TOP )
4485 return Type::TOP;
4486 return TypeTuple::MEMBAR;
4487 }
4488
4489 //------------------------------match------------------------------------------
4490 // Construct projections for memory.
4491 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4492 switch (proj->_con) {
4493 case TypeFunc::Control:
4494 case TypeFunc::Memory:
4495 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4496 }
4497 ShouldNotReachHere();
4498 return nullptr;
4499 }
4500
4501 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4502 trailing->_kind = TrailingStore;
4503 leading->_kind = LeadingStore;
4504 #ifdef ASSERT
4505 trailing->_pair_idx = leading->_idx;
4506 leading->_pair_idx = leading->_idx;
4507 #endif
4508 }
4509
4510 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4511 trailing->_kind = TrailingLoadStore;
4758 return (req() > RawStores);
4759 }
4760
4761 void InitializeNode::set_complete(PhaseGVN* phase) {
4762 assert(!is_complete(), "caller responsibility");
4763 _is_complete = Complete;
4764
4765 // After this node is complete, it contains a bunch of
4766 // raw-memory initializations. There is no need for
4767 // it to have anything to do with non-raw memory effects.
4768 // Therefore, tell all non-raw users to re-optimize themselves,
4769 // after skipping the memory effects of this initialization.
4770 PhaseIterGVN* igvn = phase->is_IterGVN();
4771 if (igvn) igvn->add_users_to_worklist(this);
4772 }
4773
4774 // convenience function
4775 // return false if the init contains any stores already
4776 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4777 InitializeNode* init = initialization();
4778 if (init == nullptr || init->is_complete()) return false;
4779 init->remove_extra_zeroes();
4780 // for now, if this allocation has already collected any inits, bail:
4781 if (init->is_non_zero()) return false;
4782 init->set_complete(phase);
4783 return true;
4784 }
4785
4786 void InitializeNode::remove_extra_zeroes() {
4787 if (req() == RawStores) return;
4788 Node* zmem = zero_memory();
4789 uint fill = RawStores;
4790 for (uint i = fill; i < req(); i++) {
4791 Node* n = in(i);
4792 if (n->is_top() || n == zmem) continue; // skip
4793 if (fill < i) set_req(fill, n); // compact
4794 ++fill;
4795 }
4796 // delete any empty spaces created:
4797 while (fill < req()) {
4798 del_req(fill);
4942 // store node that we'd like to capture. We need to check
4943 // the uses of the MergeMemNode.
4944 mems.push(n);
4945 }
4946 } else if (n->is_Mem()) {
4947 Node* other_adr = n->in(MemNode::Address);
4948 if (other_adr == adr) {
4949 failed = true;
4950 break;
4951 } else {
4952 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4953 if (other_t_adr != nullptr) {
4954 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4955 if (other_alias_idx == alias_idx) {
4956 // A load from the same memory slice as the store right
4957 // after the InitializeNode. We check the control of the
4958 // object/array that is loaded from. If it's the same as
4959 // the store control then we cannot capture the store.
4960 assert(!n->is_Store(), "2 stores to same slice on same control?");
4961 Node* base = other_adr;
4962 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4963 base = base->in(AddPNode::Base);
4964 if (base != nullptr) {
4965 base = base->uncast();
4966 if (base->is_Proj() && base->in(0) == alloc) {
4967 failed = true;
4968 break;
4969 }
4970 }
4971 }
4972 }
4973 }
4974 } else {
4975 failed = true;
4976 break;
4977 }
4978 }
4979 }
4980 }
4981 if (failed) {
5527 // z's_done 12 16 16 16 12 16 12
5528 // z's_needed 12 16 16 16 16 16 16
5529 // zsize 0 0 0 0 4 0 4
5530 if (next_full_store < 0) {
5531 // Conservative tack: Zero to end of current word.
5532 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5533 } else {
5534 // Zero to beginning of next fully initialized word.
5535 // Or, don't zero at all, if we are already in that word.
5536 assert(next_full_store >= zeroes_needed, "must go forward");
5537 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5538 zeroes_needed = next_full_store;
5539 }
5540 }
5541
5542 if (zeroes_needed > zeroes_done) {
5543 intptr_t zsize = zeroes_needed - zeroes_done;
5544 // Do some incremental zeroing on rawmem, in parallel with inits.
5545 zeroes_done = align_down(zeroes_done, BytesPerInt);
5546 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5547 zeroes_done, zeroes_needed,
5548 true,
5549 phase);
5550 zeroes_done = zeroes_needed;
5551 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5552 do_zeroing = false; // leave the hole, next time
5553 }
5554 }
5555
5556 // Collect the store and move on:
5557 phase->replace_input_of(st, MemNode::Memory, inits);
5558 inits = st; // put it on the linearized chain
5559 set_req(i, zmem); // unhook from previous position
5560
5561 if (zeroes_done == st_off)
5562 zeroes_done = next_init_off;
5563
5564 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5565
5566 #ifdef ASSERT
5587 remove_extra_zeroes(); // clear out all the zmems left over
5588 add_req(inits);
5589
5590 if (!(UseTLAB && ZeroTLAB)) {
5591 // If anything remains to be zeroed, zero it all now.
5592 zeroes_done = align_down(zeroes_done, BytesPerInt);
5593 // if it is the last unused 4 bytes of an instance, forget about it
5594 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5595 if (zeroes_done + BytesPerLong >= size_limit) {
5596 AllocateNode* alloc = allocation();
5597 assert(alloc != nullptr, "must be present");
5598 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5599 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5600 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5601 if (zeroes_done == k->layout_helper())
5602 zeroes_done = size_limit;
5603 }
5604 }
5605 if (zeroes_done < size_limit) {
5606 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5607 zeroes_done, size_in_bytes, true, phase);
5608 }
5609 }
5610
5611 set_complete(phase);
5612 return rawmem;
5613 }
5614
5615 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5616 auto replace_proj = [&](ProjNode* proj) {
5617 C->gvn_replace_by(proj, mem);
5618 return CONTINUE;
5619 };
5620 apply_to_projs(replace_proj, TypeFunc::Memory);
5621 }
5622
5623 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5624 DUIterator_Fast imax, i = fast_outs(imax);
5625 auto replace_proj = [&](ProjNode* proj) {
5626 igvn->replace_node(proj, mem);
5824 //------------------------------Identity---------------------------------------
5825 Node* MergeMemNode::Identity(PhaseGVN* phase) {
5826 // Identity if this merge point does not record any interesting memory
5827 // disambiguations.
5828 Node* base_mem = base_memory();
5829 Node* empty_mem = empty_memory();
5830 if (base_mem != empty_mem) { // Memory path is not dead?
5831 for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
5832 Node* mem = in(i);
5833 if (mem != empty_mem && mem != base_mem) {
5834 return this; // Many memory splits; no change
5835 }
5836 }
5837 }
5838 return base_mem; // No memory splits; ID on the one true input
5839 }
5840
5841 //------------------------------Ideal------------------------------------------
5842 // This method is invoked recursively on chains of MergeMem nodes
5843 Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) {
5844 // Remove chain'd MergeMems
5845 //
5846 // This is delicate, because the each "in(i)" (i >= Raw) is interpreted
5847 // relative to the "in(Bot)". Since we are patching both at the same time,
5848 // we have to be careful to read each "in(i)" relative to the old "in(Bot)",
5849 // but rewrite each "in(i)" relative to the new "in(Bot)".
5850 Node *progress = nullptr;
5851
5852
5853 Node* old_base = base_memory();
5854 Node* empty_mem = empty_memory();
5855 if (old_base == empty_mem)
5856 return nullptr; // Dead memory path.
5857
5858 MergeMemNode* old_mbase;
5859 if (old_base != nullptr && old_base->is_MergeMem())
5860 old_mbase = old_base->as_MergeMem();
5861 else
5862 old_mbase = nullptr;
5863 Node* new_base = old_base;
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "ci/ciInlineKlass.hpp"
28 #include "ci/ciInstanceKlass.hpp"
29 #include "classfile/javaClasses.hpp"
30 #include "classfile/systemDictionary.hpp"
31 #include "classfile/vmIntrinsics.hpp"
32 #include "compiler/compileLog.hpp"
33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/c2/barrierSetC2.hpp"
35 #include "gc/shared/tlab_globals.hpp"
36 #include "memory/allocation.inline.hpp"
37 #include "memory/resourceArea.hpp"
38 #include "oops/flatArrayKlass.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "opto/addnode.hpp"
41 #include "opto/arraycopynode.hpp"
42 #include "opto/callnode.hpp"
43 #include "opto/cfgnode.hpp"
44 #include "opto/compile.hpp"
45 #include "opto/connode.hpp"
46 #include "opto/convertnode.hpp"
47 #include "opto/inlinetypenode.hpp"
48 #include "opto/loopnode.hpp"
49 #include "opto/machnode.hpp"
50 #include "opto/matcher.hpp"
51 #include "opto/memnode.hpp"
52 #include "opto/mempointer.hpp"
53 #include "opto/mulnode.hpp"
54 #include "opto/narrowptrnode.hpp"
55 #include "opto/opcodes.hpp"
56 #include "opto/phaseX.hpp"
57 #include "opto/regalloc.hpp"
58 #include "opto/regmask.hpp"
59 #include "opto/rootnode.hpp"
60 #include "opto/traceMergeStoresTag.hpp"
61 #include "opto/vectornode.hpp"
62 #include "runtime/arguments.hpp"
63 #include "utilities/align.hpp"
64 #include "utilities/copy.hpp"
65 #include "utilities/globalDefinitions.hpp"
66 #include "utilities/macros.hpp"
67 #include "utilities/powerOfTwo.hpp"
68 #include "utilities/vmError.hpp"
69
70 // Portions of code courtesy of Clifford Click
71
72 // Optimization - Graph Style
73
74 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
75
76 //=============================================================================
77 uint MemNode::size_of() const { return sizeof(*this); }
78
79 const TypePtr *MemNode::adr_type() const {
80 Node* adr = in(Address);
81 if (adr == nullptr) return nullptr; // node is dead
82 const TypePtr* cross_check = nullptr;
134 st->print(", idx=Bot;");
135 else if (atp->index() == Compile::AliasIdxTop)
136 st->print(", idx=Top;");
137 else if (atp->index() == Compile::AliasIdxRaw)
138 st->print(", idx=Raw;");
139 else {
140 ciField* field = atp->field();
141 if (field) {
142 st->print(", name=");
143 field->print_name_on(st);
144 }
145 st->print(", idx=%d;", atp->index());
146 }
147 }
148 }
149
150 extern void print_alias_types();
151
152 #endif
153
154 // Find the memory output corresponding to the fall-through path of a call
155 static Node* find_call_fallthrough_mem_output(CallNode* call) {
156 ResourceMark rm;
157 CallProjections* projs = call->extract_projections(false, false);
158 Node* res = projs->fallthrough_memproj;
159 assert(res != nullptr, "must have a fallthrough mem output");
160 return res;
161 }
162
163 // Try to find a better memory input for a load from a strict final field
164 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
165 intptr_t offset = 0;
166 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
167 if (base == nullptr) {
168 return nullptr;
169 }
170
171 Node* base_uncasted = base->uncast();
172 if (base_uncasted->is_Proj()) {
173 Node* multi = base_uncasted->in(0);
174 if (multi->is_top()) {
175 // The pointer dies, make the memory die, too
176 return multi;
177 } else if (multi->is_Allocate()) {
178 base_local = base_uncasted->as_Proj();
179 return nullptr;
180 } else if (multi->is_Call()) {
181 if (!multi->is_CallJava() || multi->as_CallJava()->method() == nullptr || !multi->as_CallJava()->method()->return_value_is_larval()) {
182 // The oop is returned from a call, the memory can be the fallthrough output of the call
183 return find_call_fallthrough_mem_output(multi->as_Call());
184 }
185 } else if (multi->is_Start()) {
186 // The oop is a parameter
187 if (base_uncasted->as_Proj()->_con == TypeFunc::Parms && phase->C->method()->receiver_maybe_larval()) {
188 // The receiver of a constructor is similar to the result of an AllocateNode
189 base_local = base_uncasted->as_Proj();
190 return nullptr;
191 } else {
192 // Use the start memory otherwise
193 return multi->as_Start()->proj_out(TypeFunc::Memory);
194 }
195 }
196 }
197
198 return nullptr;
199 }
200
201 // Whether a call can modify a strict final field, given that the object is allocated inside the
202 // current compilation unit, or is the first parameter when the compilation root is a constructor.
203 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
204 // the target method is a subclass of the class declaring 'field'.
205 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
206 if (!call->is_CallJava()) {
207 return false;
208 }
209
210 ciMethod* target = call->as_CallJava()->method();
211 if (target == nullptr) {
212 return false;
213 } else if (target->intrinsic_id() == vmIntrinsicID::_linkToSpecial) {
214 // linkToSpecial can be used to call a constructor, used in the construction of objects in the
215 // reflection API
216 return true;
217 } else if (!target->is_object_constructor()) {
218 return false;
219 }
220
221 // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
222 // then the field is set inside the constructor, else the field must be set before the
223 // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
224 // while a field Sub.y must be set before Super::<init> is invoked.
225 // We can try to be more heroic and decide if the receiver of the constructor invocation is the
226 // object from which we are loading from. This, however, may be problematic as deciding if 2
227 // nodes are definitely different may not be trivial, especially if the graph is not canonical.
228 // As a result, it is made more conservative for now.
229 assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
230 return target->holder()->is_subclass_of(field->holder());
231 }
232
233 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
234 assert(t_oop != nullptr, "sanity");
235 bool is_known_instance = t_oop->is_known_instance_field();
236 bool is_strict_final_load = false;
237
238 // After macro expansion, an allocation may become a call, changing the memory input to the
239 // memory output of that call would be illegal. As a result, disallow this transformation after
240 // macro expansion.
241 if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
242 is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
243 #ifdef ASSERT
244 if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
245 assert(is_strict_final_load, "sanity check for basic cases");
246 }
247 #endif // ASSERT
248 }
249
250 if (!is_known_instance && !is_strict_final_load) {
251 return mchain;
252 }
253
254 Node* result = mchain;
255 ProjNode* base_local = nullptr;
256
257 ciField* field = nullptr;
258 if (is_strict_final_load) {
259 field = phase->C->alias_type(t_oop)->field();
260 assert(field != nullptr, "must point to a field");
261
262 Node* adr = load->in(MemNode::Address);
263 assert(phase->type(adr) == t_oop, "inconsistent type");
264 Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
265 if (tmp != nullptr) {
266 result = tmp;
267 }
268 }
269
270 uint instance_id = t_oop->instance_id();
271 Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
272 Node* prev = nullptr;
273 while (prev != result) {
274 prev = result;
275 if (result == start_mem) {
276 // start_mem is the earliest memory possible
277 break;
278 }
279
280 // skip over a call which does not affect this memory slice
281 if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
282 Node* proj_in = result->in(0);
283 if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
284 // This is the allocation that creates the object from which we are loading from
285 break;
286 } else if (proj_in->is_Call()) {
287 // ArrayCopyNodes processed here as well
288 CallNode* call = proj_in->as_Call();
289 if (!call->may_modify(t_oop, phase)) {
290 result = call->in(TypeFunc::Memory);
291 } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
292 result = call->in(TypeFunc::Memory);
293 }
294 } else if (proj_in->Opcode() == Op_Tuple) {
295 // The call will be folded, skip over it.
296 break;
297 } else if (proj_in->is_Initialize()) {
298 AllocateNode* alloc = proj_in->as_Initialize()->allocation();
299 // Stop if this is the initialization for the object instance which
300 // contains this memory slice, otherwise skip over it.
301 if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
302 break;
303 }
304 if (is_known_instance) {
305 result = proj_in->in(TypeFunc::Memory);
306 } else if (is_strict_final_load) {
307 Node* klass = alloc->in(AllocateNode::KlassNode);
308 const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
309 if (tklass->klass_is_exact() && !tklass->exact_klass()->is_subclass_of(t_oop->is_instptr()->instance_klass())) {
310 // Allocation of an unrelated type, must be another object
311 result = proj_in->in(TypeFunc::Memory);
312 } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
313 // Allocation of another object
314 result = proj_in->in(TypeFunc::Memory);
315 }
316 }
317 } else if (proj_in->is_MemBar()) {
318 ArrayCopyNode* ac = nullptr;
319 if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
320 break;
321 }
322 result = proj_in->in(TypeFunc::Memory);
323 } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
324 bool mismatched = proj_in->is_LoadFlat() ? proj_in->as_LoadFlat()->is_mismatched() : proj_in->as_StoreFlat()->is_mismatched();
325 if (is_strict_final_load || (is_known_instance && !mismatched)) {
326 // LoadFlat and StoreFlat cannot happen to strict final fields
327 // LoadFlat and StoreFlat to known instances are removed at the end of EA unless mismatched: this one is unrelated
328 result = proj_in->in(TypeFunc::Memory);
329 }
330 } else if (proj_in->is_top()) {
331 break; // dead code
332 } else {
333 assert(false, "unexpected projection of %s", proj_in->Name());
334 }
335 } else if (result->is_ClearArray()) {
336 if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
337 // Can not bypass initialization of the instance
338 // we are looking for.
339 break;
340 }
341 // Otherwise skip it (the call updated 'result' value).
342 } else if (result->is_MergeMem()) {
343 result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
344 }
345 }
346 return result;
347 }
348
349 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
350 const TypeOopPtr* t_oop = t_adr->isa_oopptr();
351 if (t_oop == nullptr)
352 return mchain; // don't try to optimize non-oop types
353 Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
354 bool is_instance = t_oop->is_known_instance_field();
355 PhaseIterGVN *igvn = phase->is_IterGVN();
356 if (is_instance && igvn != nullptr && result->is_Phi()) {
357 PhiNode *mphi = result->as_Phi();
358 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
359 const TypePtr *t = mphi->adr_type();
360 bool do_split = false;
361 // In the following cases, Load memory input can be further optimized based on
362 // its precise address type
363 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
364 do_split = true;
365 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
366 const TypeOopPtr* mem_t =
367 t->is_oopptr()->cast_to_exactness(true)
368 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
369 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
370 if (t_oop->isa_aryptr()) {
371 mem_t = mem_t->is_aryptr()
372 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
373 ->cast_to_size(t_oop->is_aryptr()->size())
374 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
375 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
376 ->with_offset(t_oop->is_aryptr()->offset())
377 ->is_aryptr();
378 }
379 do_split = mem_t == t_oop;
380 }
381 if (do_split) {
382 // clone the Phi with our address type
383 result = mphi->split_out_instance(t_adr, igvn);
384 } else {
385 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
386 }
387 }
388 return result;
389 }
390
391 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
392 uint alias_idx = phase->C->get_alias_index(tp);
393 Node *mem = mmem;
394 #ifdef ASSERT
395 {
396 // Check that current type is consistent with the alias index used during graph construction
397 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
398 bool consistent = adr_check == nullptr || adr_check->empty() ||
399 phase->C->must_alias(adr_check, alias_idx );
400 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
401 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
402 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
403 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
404 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
405 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
406 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
407 // don't assert if it is dead code.
408 consistent = true;
409 }
410 if( !consistent ) {
411 st->print("alias_idx==%d, adr_check==", alias_idx);
412 if( adr_check == nullptr ) {
413 st->print("null");
414 } else {
415 adr_check->dump();
416 }
417 st->cr();
418 print_alias_types();
419 assert(consistent, "adr_check must match alias idx");
420 }
421 }
422 #endif
725 }
726
727 // Find an arraycopy ac that produces the memory state represented by parameter mem.
728 // Return ac if
729 // (a) can_see_stored_value=true and ac must have set the value for this load or if
730 // (b) can_see_stored_value=false and ac could have set the value for this load or if
731 // (c) can_see_stored_value=false and ac cannot have set the value for this load.
732 // In case (c) change the parameter mem to the memory input of ac to skip it
733 // when searching stored value.
734 // Otherwise return null.
735 Node* LoadNode::find_previous_arraycopy(PhaseValues* phase, Node* ld_alloc, Node*& mem, bool can_see_stored_value) const {
736 ArrayCopyNode* ac = find_array_copy_clone(ld_alloc, mem);
737 if (ac != nullptr) {
738 Node* ld_addp = in(MemNode::Address);
739 Node* src = ac->in(ArrayCopyNode::Src);
740 const TypeAryPtr* ary_t = phase->type(src)->isa_aryptr();
741
742 // This is a load from a cloned array. The corresponding arraycopy ac must
743 // have set the value for the load and we can return ac but only if the load
744 // is known to be within bounds. This is checked below.
745 // TODO 8350865: Support flat arrays in LoadNode::find_previous_arraycopy
746 if (ary_t != nullptr && ary_t->is_not_flat() && ld_addp->is_AddP()) {
747 Node* ld_offs = ld_addp->in(AddPNode::Offset);
748 BasicType ary_elem = ary_t->elem()->array_element_basic_type();
749 jlong header = arrayOopDesc::base_offset_in_bytes(ary_elem);
750 jlong elemsize = type2aelembytes(ary_elem);
751
752 const TypeX* ld_offs_t = phase->type(ld_offs)->isa_intptr_t();
753 const TypeInt* sizetype = ary_t->size();
754
755 if (ld_offs_t->_lo >= header && ld_offs_t->_hi < (sizetype->_lo * elemsize + header)) {
756 // The load is known to be within bounds. It receives its value from ac.
757 return ac;
758 }
759 // The load is known to be out-of-bounds.
760 }
761 // The load could be out-of-bounds. It must not be hoisted but must remain
762 // dependent on the runtime range check. This is achieved by returning null.
763 } else if (mem->is_Proj() && mem->in(0) != nullptr && mem->in(0)->is_ArrayCopy()) {
764 ArrayCopyNode* ac = mem->in(0)->as_ArrayCopy();
765
766 if (ac->is_arraycopy_validated() ||
1136 in_bytes(JavaThread::vthread_offset()),
1137 in_bytes(JavaThread::scopedValueCache_offset()),
1138 };
1139
1140 for (size_t i = 0; i < sizeof offsets / sizeof offsets[0]; i++) {
1141 if (offset == offsets[i]) {
1142 return true;
1143 }
1144 }
1145 }
1146
1147 return false;
1148 }
1149 #endif
1150
1151 //----------------------------LoadNode::make-----------------------------------
1152 // Polymorphic factory method:
1153 Node* LoadNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, const Type* rt, BasicType bt, MemOrd mo,
1154 ControlDependency control_dependency, bool require_atomic_access, bool unaligned, bool mismatched, bool unsafe, uint8_t barrier_data) {
1155 Compile* C = gvn.C;
1156 assert(adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr(), true) == C->get_alias_index(adr_type, true), "adr and adr_type must agree");
1157
1158 // sanity check the alias category against the created node type
1159 assert(!(adr_type->isa_oopptr() &&
1160 adr_type->offset() == oopDesc::klass_offset_in_bytes()),
1161 "use LoadKlassNode instead");
1162 assert(!(adr_type->isa_aryptr() &&
1163 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1164 "use LoadRangeNode instead");
1165 // Check control edge of raw loads
1166 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1167 // oop will be recorded in oop map if load crosses safepoint
1168 rt->isa_oopptr() || is_immutable_value(adr),
1169 "raw memory operations should have control edge");
1170 LoadNode* load = nullptr;
1171 switch (bt) {
1172 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1173 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1174 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1175 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1176 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
1177 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1178 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
1179 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
1180 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
1181 case T_ARRAY:
1182 case T_OBJECT:
1183 case T_NARROWOOP:
1184 #ifdef _LP64
1185 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1186 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1187 } else
1188 #endif
1189 {
1190 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1191 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1192 }
1193 break;
1194 default:
1195 guarantee(false, "unexpected basic type %s", type2name(bt));
1196 break;
1197 }
1198 assert(load != nullptr, "LoadNode should have been created");
1199 if (unaligned) {
1200 load->set_unaligned_access();
1201 }
1202 if (mismatched) {
1203 load->set_mismatched_access();
1204 }
1205 if (unsafe) {
1206 load->set_unsafe_access();
1207 }
1208 load->set_barrier_data(barrier_data);
1209 if (load->Opcode() == Op_LoadN) {
1210 Node* ld = gvn.transform(load);
1211 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1212 }
1213
1214 return load;
1215 }
1216
1217 //------------------------------hash-------------------------------------------
1218 uint LoadNode::hash() const {
1219 // unroll addition of interesting fields
1220 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1221 }
1222
1223 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1224 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1225 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1226 bool is_stable_ary = FoldStableValues &&
1227 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1228 tp->isa_aryptr()->is_stable();
1229
1230 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1231 }
1232
1233 return false;
1234 }
1235
1236 // Is the value loaded previously stored by an arraycopy? If so return
1237 // a load node that reads from the source array so we may be able to
1238 // optimize out the ArrayCopy node later.
1239 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1240 Node* ld_adr = in(MemNode::Address);
1241 intptr_t ld_off = 0;
1242 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1243 Node* ac = find_previous_arraycopy(phase, ld_alloc, st, true);
1244 if (ac != nullptr) {
1245 assert(ac->is_ArrayCopy(), "what kind of node can this be?");
1246
1247 Node* mem = ac->in(TypeFunc::Memory);
1248 Node* ctl = ac->in(0);
1249 Node* src = ac->in(ArrayCopyNode::Src);
1250
1258 if (ac->as_ArrayCopy()->is_clonebasic()) {
1259 assert(ld_alloc != nullptr, "need an alloc");
1260 assert(addp->is_AddP(), "address must be addp");
1261 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1262 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1263 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1264 addp->set_req(AddPNode::Base, src);
1265 addp->set_req(AddPNode::Address, src);
1266 } else {
1267 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1268 ac->as_ArrayCopy()->is_copyof_validated() ||
1269 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1270 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1271 addp->set_req(AddPNode::Base, src);
1272 addp->set_req(AddPNode::Address, src);
1273
1274 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1275 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1276 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1277
1278 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1279
1280 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1281 #ifdef _LP64
1282 diff = phase->transform(new ConvI2LNode(diff));
1283 #endif
1284 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1285
1286 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1287 addp->set_req(AddPNode::Offset, offset);
1288 }
1289 addp = phase->transform(addp);
1290 #ifdef ASSERT
1291 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1292 ld->_adr_type = adr_type;
1293 #endif
1294 ld->set_req(MemNode::Address, addp);
1295 ld->set_req(0, ctl);
1296 ld->set_req(MemNode::Memory, mem);
1297 return ld;
1298 }
1299 return nullptr;
1300 }
1301
1302 static Node* see_through_inline_type(PhaseValues* phase, const LoadNode* load, Node* base, int offset) {
1303 if (load->is_mismatched_access() || base == nullptr) {
1304 return nullptr;
1305 }
1306
1307 InlineTypeNode* vt = base->isa_InlineType();
1308 if (vt == nullptr || offset < vt->type()->inline_klass()->payload_offset()) {
1309 return nullptr;
1310 }
1311
1312 Node* value = vt->field_value_by_offset(offset, true);
1313 assert(value != nullptr, "must see some value");
1314 return value;
1315 }
1316
1317 // This routine exists to make sure this set of tests is done the same
1318 // everywhere. We need to make a coordinated change: first LoadNode::Ideal
1319 // will change the graph shape in a way which makes memory alive twice at the
1320 // same time (uses the Oracle model of aliasing), then some
1321 // LoadXNode::Identity will fold things back to the equivalence-class model
1322 // of aliasing.
1323 // This method may find an unencoded node instead of the corresponding encoded one.
1324 Node* LoadNode::can_see_stored_value_through_membars(Node* st, PhaseValues* phase) const {
1325 Node* ld_adr = in(MemNode::Address);
1326 intptr_t ld_off = 0;
1327 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1328 // Try to see through an InlineTypeNode
1329 Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1330 if (value != nullptr) {
1331 return value;
1332 }
1333
1334 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1335 Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1336
1337 if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1338 uint alias_idx = atp->index();
1339 Node* result = nullptr;
1340 Node* current = st;
1341 // Skip through chains of MemBarNodes checking the MergeMems for new states for the slice of
1342 // this load. Stop once any other kind of node is encountered.
1343 //
1344 // In principle, folding a load is moving it up until it meets a matching store.
1345 //
1346 // store(ptr, v); store(ptr, v); store(ptr, v);
1347 // membar1; -> membar1; -> load(ptr);
1348 // membar2; load(ptr); membar1;
1349 // load(ptr); membar2; membar2;
1350 //
1351 // So, we can decide which kinds of barriers we can walk past. It is not safe to step over
1352 // MemBarCPUOrder, even if the memory is not rewritable, because alias info above them may be
1353 // inaccurate (e.g., due to mixed/mismatched unsafe accesses).
1365 MergeMemNode* merge = mem->as_MergeMem();
1366 Node* new_st = merge->memory_at(alias_idx);
1367 if (new_st == merge->base_memory()) {
1368 // Keep searching
1369 current = new_st;
1370 continue;
1371 }
1372 // Save the new memory state for the slice and fall through
1373 // to exit.
1374 result = new_st;
1375 }
1376 }
1377 break;
1378 }
1379 if (result != nullptr) {
1380 st = result;
1381 }
1382 }
1383
1384 Node* res = can_see_stored_value(st, phase);
1385 // TODO: reimplement assert, see: JDK-8386157
1386 //assert(res == nullptr || is_java_primitive(value_basic_type()) || res->bottom_type()->higher_equal(type()), "the fold is unsafe");
1387 return res;
1388 }
1389
1390 // If st is a store to the same location as this, return the stored value
1391 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1392 Node* ld_adr = in(MemNode::Address);
1393 intptr_t ld_off = 0;
1394 Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1395 Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1396 const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1397
1398 // Loop around twice in the case Load -> Initialize -> Store.
1399 // (See PhaseIterGVN::add_users_to_worklist, which knows about this case.)
1400 for (int trip = 0; trip <= 1; trip++) {
1401
1402 if (st->is_Store()) {
1403 Node* st_adr = st->in(MemNode::Address);
1404 if (st_adr != ld_adr) {
1405 // Try harder before giving up. Unify base pointers with casts (e.g., raw/non-raw pointers).
1406 intptr_t st_off = 0;
1451 if (is_Store() || is_java_primitive(value_basic_type()) || res->bottom_type()->higher_equal(bottom_type())) {
1452 return res;
1453 }
1454
1455 // Type-unsafe stores must be due to array polymorphism
1456 const TypePtr* adr_type = this->adr_type();
1457 assert(adr_type == nullptr || adr_type->isa_aryptr() != nullptr, "unexpected type-unsafe store");
1458 return nullptr;
1459 }
1460
1461 // A load from a freshly-created object always returns zero.
1462 // (This can happen after LoadNode::Ideal resets the load's memory input
1463 // to find_captured_store, which returned InitializeNode::zero_memory.)
1464 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1465 (st->in(0) == ld_alloc) &&
1466 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1467 // return a zero value for the load's basic type
1468 // (This is one of the few places where a generic PhaseTransform
1469 // can create new nodes. Think of it as lazily manifesting
1470 // virtually pre-existing constants.)
1471 Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1472 if (init_value != nullptr) {
1473 const TypeAryPtr* ld_adr_type = phase->type(ld_adr)->isa_aryptr();
1474 if (ld_adr_type == nullptr) {
1475 return nullptr;
1476 }
1477
1478 // We know that this is not a flat array, the load should return the whole oop
1479 if (ld_adr_type->is_not_flat()) {
1480 return init_value;
1481 }
1482
1483 // If this is a flat array, try to see through init_value
1484 if (init_value->is_EncodeP()) {
1485 init_value = init_value->in(1);
1486 }
1487 if (!init_value->is_InlineType() || ld_adr_type->field_offset() == Type::Offset::bottom) {
1488 return nullptr;
1489 }
1490
1491 ciInlineKlass* vk = phase->type(init_value)->inline_klass();
1492 int field_offset_in_payload = ld_adr_type->field_offset().get();
1493 if (field_offset_in_payload == vk->null_marker_offset_in_payload()) {
1494 return init_value->as_InlineType()->get_null_marker();
1495 } else {
1496 return init_value->as_InlineType()->field_value_by_offset(field_offset_in_payload + vk->payload_offset(), true);
1497 }
1498 }
1499 assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1500 if (value_basic_type() != T_VOID) {
1501 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1502 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1503 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1504 // by the ArrayCopyNode.
1505 return phase->zerocon(value_basic_type());
1506 }
1507 } else {
1508 // TODO: materialize all-zero vector constant
1509 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1510 }
1511 }
1512
1513 // A load from an initialization barrier can match a captured store.
1514 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1515 InitializeNode* init = st->in(0)->as_Initialize();
1516 AllocateNode* alloc = init->allocation();
1517 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1518 // examine a captured store value
1519 st = init->find_captured_store(ld_off, memory_size(), phase);
1532 base = bs->step_over_gc_barrier(base);
1533 if (base != nullptr && base->is_Proj() &&
1534 base->as_Proj()->_con == TypeFunc::Parms &&
1535 base->in(0)->is_CallStaticJava() &&
1536 base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1537 return base->in(0)->in(TypeFunc::Parms);
1538 }
1539 }
1540
1541 break;
1542 }
1543
1544 return nullptr;
1545 }
1546
1547 //----------------------is_instance_field_load_with_local_phi------------------
1548 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1549 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1550 in(Address)->is_AddP() ) {
1551 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1552 // Only known instances and immutable fields
1553 if( t_oop != nullptr &&
1554 (t_oop->is_ptr_to_strict_final_field() ||
1555 t_oop->is_known_instance_field()) &&
1556 t_oop->offset() != Type::OffsetBot &&
1557 t_oop->offset() != Type::OffsetTop) {
1558 return true;
1559 }
1560 }
1561 return false;
1562 }
1563
1564 //------------------------------Identity---------------------------------------
1565 // Loads are identity if previous store is to same address
1566 Node* LoadNode::Identity(PhaseGVN* phase) {
1567 // If the previous store-maker is the right kind of Store, and the store is
1568 // to the same address, then we are equal to the value stored.
1569 Node* mem = in(Memory);
1570 Node* value = can_see_stored_value_through_membars(mem, phase);
1571 if( value ) {
1572 // byte, short & char stores truncate naturally.
1573 // A load has to load the truncated value which requires
1574 // some sort of masking operation and that requires an
1575 // Ideal call instead of an Identity call.
1576 if (memory_size() < BytesPerInt) {
1577 // If the input to the store does not fit with the load's result type,
1578 // it must be truncated via an Ideal call.
1579 if (!phase->type(value)->higher_equal(phase->type(this)))
1580 return this;
1581 }
1582
1583 if (phase->type(value)->isa_ptr() && phase->type(this)->isa_narrowoop()) {
1584 return this;
1585 }
1586 // (This works even when value is a Con, but LoadNode::Value
1587 // usually runs first, producing the singleton type of the Con.)
1588 if (!has_pinned_control_dependency() || value->is_Con()) {
1589 return value;
1590 } else {
1591 return this;
1592 }
1593 }
1594
1595 if (has_pinned_control_dependency()) {
1596 return this;
1597 }
1598 // Search for an existing data phi which was generated before for the same
1599 // instance's field to avoid infinite generation of phis in a loop.
1600 Node *region = mem->in(0);
1601 if (is_instance_field_load_with_local_phi(region)) {
1602 const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1603 int this_index = phase->C->get_alias_index(addr_t);
1604 int this_offset = addr_t->offset();
1605 int this_iid = addr_t->instance_id();
1606 if (!addr_t->is_known_instance() &&
1607 addr_t->is_ptr_to_strict_final_field()) {
1608 // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1609 intptr_t ignore = 0;
1610 Node* base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1611 if (base == nullptr) {
1612 return this;
1613 }
1614 this_iid = base->_idx;
1615 }
1616 const Type* this_type = bottom_type();
1617 for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1618 Node* phi = region->fast_out(i);
1619 if (phi->is_Phi() && phi != mem &&
1620 phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1621 return phi;
1622 }
1623 }
1624 }
1625
1626 return this;
1627 }
1628
2163 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2164 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2165
2166 // Skip up past a SafePoint control. Cannot do this for Stores because
2167 // pointer stores & cardmarks must stay on the same side of a SafePoint.
2168 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2169 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
2170 !addr_mark &&
2171 (depends_only_on_test() || has_unknown_control_dependency())) {
2172 ctrl = ctrl->in(0);
2173 set_req(MemNode::Control,ctrl);
2174 return this;
2175 }
2176
2177 intptr_t ignore = 0;
2178 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2179 if (base != nullptr
2180 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2181 // Check for useless control edge in some common special cases
2182 if (in(MemNode::Control) != nullptr
2183 // TODO 8350865 Can we re-enable this?
2184 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2185 && can_remove_control()
2186 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2187 && all_controls_dominate(base, phase->C->start(), phase)) {
2188 // A method-invariant, non-null address (constant or 'this' argument).
2189 set_req(MemNode::Control, nullptr);
2190 return this;
2191 }
2192 }
2193
2194 Node* mem = in(MemNode::Memory);
2195 const TypePtr *addr_t = phase->type(address)->isa_ptr();
2196
2197 if (can_reshape && (addr_t != nullptr)) {
2198 // try to optimize our memory input
2199 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2200 if (opt_mem != mem) {
2201 set_req_X(MemNode::Memory, opt_mem, phase);
2202 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2203 return this;
2204 }
2317 // No match.
2318 return nullptr;
2319 }
2320
2321 //------------------------------Value-----------------------------------------
2322 const Type* LoadNode::Value(PhaseGVN* phase) const {
2323 // Either input is TOP ==> the result is TOP
2324 Node* mem = in(MemNode::Memory);
2325 const Type *t1 = phase->type(mem);
2326 if (t1 == Type::TOP) return Type::TOP;
2327 Node* adr = in(MemNode::Address);
2328 const TypePtr* tp = phase->type(adr)->isa_ptr();
2329 if (tp == nullptr || tp->empty()) return Type::TOP;
2330 int off = tp->offset();
2331 assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
2332 Compile* C = phase->C;
2333
2334 // If load can see a previous constant store, use that.
2335 Node* value = can_see_stored_value_through_membars(mem, phase);
2336 if (value != nullptr && value->is_Con()) {
2337 if (phase->type(value)->isa_ptr() && _type->isa_narrowoop()) {
2338 return phase->type(value)->make_narrowoop();
2339 } else {
2340 assert(value->bottom_type()->higher_equal(_type), "sanity");
2341 return phase->type(value);
2342 }
2343 }
2344 // Try to guess loaded type from pointer type
2345 if (tp->isa_aryptr()) {
2346 const TypeAryPtr* ary = tp->is_aryptr();
2347 const Type* t = ary->elem();
2348
2349 // Determine whether the reference is beyond the header or not, by comparing
2350 // the offset against the offset of the start of the array's data.
2351 // Different array types begin at slightly different offsets (12 vs. 16).
2352 // We choose T_BYTE as an example base type that is least restrictive
2353 // as to alignment, which will therefore produce the smallest
2354 // possible base offset.
2355 const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
2356 const bool off_beyond_header = (off >= min_base_off);
2357
2358 // Try to constant-fold a stable array element.
2359 if (FoldStableValues && !is_mismatched_access() && ary->is_stable()) {
2360 // Make sure the reference is not into the header and the offset is constant
2361 ciObject* aobj = ary->const_oop();
2362 if (aobj != nullptr && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
2363 int stable_dimension = (ary->stable_dimension() > 0 ? ary->stable_dimension() - 1 : 0);
2364 const Type* con_type = Type::make_constant_from_array_element(aobj->as_array(), off, ary->field_offset().get(),
2365 stable_dimension,
2366 value_basic_type(), is_unsigned());
2367 if (con_type != nullptr) {
2368 return con_type;
2369 }
2370 }
2371 }
2372
2373 // Don't do this for integer types. There is only potential profit if
2374 // the element type t is lower than _type; that is, for int types, if _type is
2375 // more restrictive than t. This only happens here if one is short and the other
2376 // char (both 16 bits), and in those cases we've made an intentional decision
2377 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2378 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2379 //
2380 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2381 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2382 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2383 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2384 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2385 // In fact, that could have been the original type of p1, and p1 could have
2386 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2387 // expression (LShiftL quux 3) independently optimized to the constant 8.
2388 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2389 && (_type->isa_vect() == nullptr)
2390 && !ary->is_flat()
2391 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2392 // t might actually be lower than _type, if _type is a unique
2393 // concrete subclass of abstract class t.
2394 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2395 const Type* jt = t->join_speculative(_type);
2396 // In any case, do not allow the join, per se, to empty out the type.
2397 if (jt->empty() && !t->empty()) {
2398 // This can happen if a interface-typed array narrows to a class type.
2399 jt = _type;
2400 }
2401 #ifdef ASSERT
2402 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2403 // The pointers in the autobox arrays are always non-null
2404 Node* base = adr->in(AddPNode::Base);
2405 if ((base != nullptr) && base->is_DecodeN()) {
2406 // Get LoadN node which loads IntegerCache.cache field
2407 base = base->in(1);
2408 }
2409 if ((base != nullptr) && base->is_Con()) {
2410 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2411 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2412 // It could be narrow oop
2413 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2414 }
2415 }
2416 }
2417 #endif
2418 return jt;
2419 }
2420 }
2421 } else if (tp->base() == Type::InstPtr) {
2422 assert( off != Type::OffsetBot ||
2423 // arrays can be cast to Objects
2424 !tp->isa_instptr() ||
2425 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2426 // Default value load
2427 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2428 // unsafe field access may not have a constant offset
2429 is_unsafe_access(),
2430 "Field accesses must be precise" );
2431 // For oop loads, we expect the _type to be precise.
2432
2433 const TypeInstPtr* tinst = tp->is_instptr();
2434 BasicType bt = value_basic_type();
2435
2436 // Fold loads of the field map
2437 if (tinst != nullptr) {
2438 ciInstanceKlass* ik = tinst->instance_klass();
2439 int offset = tinst->offset();
2440 if (ik == phase->C->env()->Class_klass()) {
2441 ciType* t = tinst->java_mirror_type();
2442 if (t != nullptr && t->is_inlinetype() && offset == t->as_inline_klass()->field_map_offset()) {
2443 ciConstant map = t->as_inline_klass()->get_field_map();
2444 bool is_narrow_oop = (bt == T_NARROWOOP);
2445 return Type::make_from_constant(map, true, 1, is_narrow_oop);
2446 }
2447 }
2448 }
2449
2450 // Optimize loads from constant fields.
2451 ciObject* const_oop = tinst->const_oop();
2452 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2453 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2454 if (con_type != nullptr) {
2455 return con_type;
2456 }
2457 }
2458 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2459 assert(off != Type::OffsetBot ||
2460 !tp->isa_instklassptr() ||
2461 // arrays can be cast to Objects
2462 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2463 // also allow array-loading from the primary supertype
2464 // array during subtype checks
2465 Opcode() == Op_LoadKlass,
2466 "Field accesses must be precise");
2467 // For klass/static loads, we expect the _type to be precise
2468 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2469 /* With mirrors being an indirect in the Klass*
2470 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2471 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2472 *
2473 * So check the type and klass of the node before the LoadP.
2480 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2481 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2482 return TypeInstPtr::make(klass->java_mirror());
2483 }
2484 }
2485 }
2486
2487 const TypeKlassPtr *tkls = tp->isa_klassptr();
2488 if (tkls != nullptr) {
2489 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2490 ciKlass* klass = tkls->exact_klass();
2491 // We are loading a field from a Klass metaobject whose identity
2492 // is known at compile time (the type is "exact" or "precise").
2493 // Check for fields we know are maintained as constants by the VM.
2494 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2495 // The field is Klass::_super_check_offset. Return its (constant) value.
2496 // (Folds up type checking code.)
2497 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2498 return TypeInt::make(klass->super_check_offset());
2499 }
2500 if (klass->is_inlinetype() && tkls->offset() == in_bytes(InstanceKlass::acmp_maps_offset_offset())) {
2501 return TypeInt::make(klass->as_inline_klass()->field_map_offset());
2502 }
2503 if (klass->is_obj_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset())) {
2504 // Fold loads from LibraryCallKit::load_default_refined_array_klass
2505 return tkls->is_aryklassptr()->cast_to_refined_array_klass_ptr();
2506 }
2507 if (klass->is_array_klass() && tkls->offset() == in_bytes(ObjArrayKlass::properties_offset())) {
2508 assert(klass->is_type_array_klass() || tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2509 return TypeInt::make((jint)klass->as_array_klass()->properties().value());
2510 }
2511 if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2512 assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2513 return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2514 }
2515 if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2516 // The field is Klass::_prototype_header. Return its (constant) value.
2517 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2518 return TypeX::make(klass->prototype_header());
2519 }
2520 // Compute index into primary_supers array
2521 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2522 // Check for overflowing; use unsigned compare to handle the negative case.
2523 if( depth < ciKlass::primary_super_limit() ) {
2524 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2525 // (Folds up type checking code.)
2526 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2527 ciKlass *ss = klass->super_of_depth(depth);
2528 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2529 }
2530 const Type* aift = load_array_final_field(tkls, klass);
2531 if (aift != nullptr) return aift;
2532 }
2533
2534 // We can still check if we are loading from the primary_supers array at a
2535 // shallow enough depth. Even though the klass is not exact, entries less
2536 // than or equal to its super depth are correct.
2537 if (tkls->is_loaded()) {
2538 ciKlass* klass = nullptr;
2572 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2573 // The key property of this type is that it folds up tests
2574 // for array-ness, since it proves that the layout_helper is positive.
2575 // Thus, a generic value like the basic object layout helper works fine.
2576 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2577 }
2578 }
2579
2580 // If we are loading from a freshly-allocated object/array, produce a zero.
2581 // Things to check:
2582 // 1. Load is beyond the header: headers are not guaranteed to be zero
2583 // 2. Load is not vectorized: vectors have no zero constant
2584 // 3. Load has no matching store, i.e. the input is the initial memory state
2585 const TypeOopPtr* tinst = tp->isa_oopptr();
2586 bool is_not_header = (tinst != nullptr) && tinst->is_known_instance_field();
2587 bool is_not_vect = (_type->isa_vect() == nullptr);
2588 if (is_not_header && is_not_vect) {
2589 Node* mem = in(MemNode::Memory);
2590 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2591 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2592 // TODO 8350865 Scalar replacement does not work well for flat arrays.
2593 // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2594 // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2595 // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2596 if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2597 intptr_t offset = 0;
2598 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2599 AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2600 if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2601 return _type;
2602 }
2603 }
2604 return Type::get_zero_type(_type->basic_type());
2605 }
2606 }
2607 if (!UseCompactObjectHeaders) {
2608 Node* alloc = is_new_object_mark_load();
2609 if (alloc != nullptr) {
2610 if (Arguments::is_valhalla_enabled()) {
2611 // The mark word may contain property bits (inline, flat, null-free)
2612 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2613 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2614 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2615 return TypeX::make(tkls->exact_klass()->prototype_header());
2616 }
2617 } else {
2618 return TypeX::make(markWord::prototype().value());
2619 }
2620 }
2621 }
2622
2623 return _type;
2624 }
2625
2626 //------------------------------match_edge-------------------------------------
2627 // Do we Match on this edge index or not? Match only the address.
2628 uint LoadNode::match_edge(uint idx) const {
2629 return idx == MemNode::Address;
2630 }
2631
2632 //--------------------------LoadBNode::Ideal--------------------------------------
2633 //
2634 // If the previous store is to the same address as this load,
2635 // and the value stored was larger than a byte, replace this load
2636 // with the value stored truncated to a byte. If no truncation is
2637 // needed, the replacement is done in LoadNode::Identity().
2638 //
2639 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2748 }
2749 }
2750 // Identity call will handle the case where truncation is not needed.
2751 return LoadNode::Ideal(phase, can_reshape);
2752 }
2753
2754 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2755 Node* mem = in(MemNode::Memory);
2756 Node* value = can_see_stored_value_through_membars(mem, phase);
2757 if (value != nullptr && value->is_Con() &&
2758 !value->bottom_type()->higher_equal(_type)) {
2759 // If the input to the store does not fit with the load's result type,
2760 // it must be truncated. We can't delay until Ideal call since
2761 // a singleton Value is needed for split_thru_phi optimization.
2762 int con = value->get_int();
2763 return TypeInt::make((con << 16) >> 16);
2764 }
2765 return LoadNode::Value(phase);
2766 }
2767
2768 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2769 // Can see the corresponding value, may need to add an EncodeP
2770 Node* value = can_see_stored_value_through_membars(in(Memory), phase);
2771 if (value != nullptr && phase->type(value)->isa_ptr() && type()->isa_narrowoop()) {
2772 return new EncodePNode(value, type());
2773 }
2774
2775 // Identity call will handle the case where EncodeP is unnecessary
2776 return LoadNode::Ideal(phase, can_reshape);
2777 }
2778
2779 //=============================================================================
2780 //----------------------------LoadKlassNode::make------------------------------
2781 // Polymorphic factory method:
2782 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2783 // sanity check the alias category against the created node type
2784 const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2785 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2786 #ifdef _LP64
2787 if (adr_type->is_ptr_to_narrowklass()) {
2788 Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2789 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2790 }
2791 #endif
2792 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2793 return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2794 }
2795
2796 //------------------------------Value------------------------------------------
2797 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2798 return klass_value_common(phase);
2831 }
2832 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2833 }
2834 if (!t->is_klass()) {
2835 // a primitive Class (e.g., int.class) has null for a klass field
2836 return TypePtr::NULL_PTR;
2837 }
2838 // Fold up the load of the hidden field
2839 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2840 }
2841 // non-constant mirror, so we can't tell what's going on
2842 }
2843 if (!tinst->is_loaded())
2844 return _type; // Bail out if not loaded
2845 if (offset == oopDesc::klass_offset_in_bytes()) {
2846 return tinst->as_klass_type(true);
2847 }
2848 }
2849
2850 // Check for loading klass from an array
2851 const TypeAryPtr* tary = tp->isa_aryptr();
2852 if (tary != nullptr &&
2853 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2854 return tary->as_klass_type(true)->is_aryklassptr();
2855 }
2856
2857 // Check for loading klass from an array klass
2858 const TypeKlassPtr *tkls = tp->isa_klassptr();
2859 if (tkls != nullptr && !StressReflectiveCode) {
2860 if (!tkls->is_loaded())
2861 return _type; // Bail out if not loaded
2862 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2863 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2864 // // Always returning precise element type is incorrect,
2865 // // e.g., element type could be object and array may contain strings
2866 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2867
2868 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2869 // according to the element type's subclassing.
2870 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2871 }
2872 if (tkls->isa_aryklassptr() != nullptr && tkls->klass_is_exact() &&
2873 !tkls->exact_klass()->is_type_array_klass() &&
2874 tkls->offset() == in_bytes(Klass::super_offset())) {
2875 // We are loading the super klass of a refined array klass, return the non-refined klass pointer
2876 assert(tkls->is_aryklassptr()->is_refined_type(), "Must be a refined array klass pointer");
2877 return tkls->is_aryklassptr()->with_offset(0)->cast_to_non_refined();
2878 }
2879 if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2880 tkls->offset() == in_bytes(Klass::super_offset())) {
2881 ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();
2882 // The field is Klass::_super. Return its (constant) value.
2883 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2884 return sup ? TypeKlassPtr::make(sup, Type::trust_interfaces) : TypePtr::NULL_PTR;
2885 }
2886 }
2887
2888 if (tkls != nullptr && !UseSecondarySupersCache
2889 && tkls->offset() == in_bytes(Klass::secondary_super_cache_offset())) {
2890 // Treat Klass::_secondary_super_cache as a constant when the cache is disabled.
2891 return TypePtr::NULL_PTR;
2892 }
2893
2894 // Bailout case
2895 return LoadNode::Value(phase);
2896 }
2897
2898 //------------------------------Identity---------------------------------------
2921 base = bs->step_over_gc_barrier(base);
2922 }
2923
2924 // We can fetch the klass directly through an AllocateNode.
2925 // This works even if the klass is not constant (clone or newArray).
2926 if (offset == oopDesc::klass_offset_in_bytes()) {
2927 Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2928 if (allocated_klass != nullptr) {
2929 return allocated_klass;
2930 }
2931 }
2932
2933 // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2934 // See inline_native_Class_query for occurrences of these patterns.
2935 // Java Example: x.getClass().isAssignableFrom(y)
2936 //
2937 // This improves reflective code, often making the Class
2938 // mirror go completely dead. (Current exception: Class
2939 // mirrors may appear in debug info, but we could clean them out by
2940 // introducing a new debug info operator for Klass.java_mirror).
2941 //
2942 // This optimization does not apply to arrays because if k is not a
2943 // constant, it was obtained via load_klass which returns the refined type
2944 // and '.java_mirror.as_klass' should return the Java type instead.
2945
2946 if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2947 && offset == java_lang_Class::klass_offset()) {
2948 if (base->is_Load()) {
2949 Node* base2 = base->in(MemNode::Address);
2950 if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2951 Node* adr2 = base2->in(MemNode::Address);
2952 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2953 if (tkls != nullptr && !tkls->empty()
2954 && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2955 && adr2->is_AddP()) {
2956 int mirror_field = in_bytes(Klass::java_mirror_offset());
2957 if (tkls->offset() == mirror_field) {
2958 #ifdef ASSERT
2959 const TypeKlassPtr* tkls2 = phase->type(adr2->in(AddPNode::Address))->is_klassptr();
2960 assert(tkls2->offset() == 0, "not a load of java_mirror");
2961 #endif
2962 assert(adr2->in(AddPNode::Base)->is_top(), "not an off heap load");
2963 assert(adr2->in(AddPNode::Offset)->find_intptr_t_con(-1) == in_bytes(Klass::java_mirror_offset()), "incorrect offset");
2964 return adr2->in(AddPNode::Address);
2965 }
2966 }
2967 }
2968 }
2969 }
2970
2971 return this;
2972 }
2973
2974 LoadNode* LoadNode::clone_pinned() const {
2975 LoadNode* ld = clone()->as_Load();
3102 // Polymorphic factory method:
3103 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
3104 assert((mo == unordered || mo == release), "unexpected");
3105 Compile* C = gvn.C;
3106 assert(adr_type == nullptr || adr->is_top() || C->get_alias_index(gvn.type(adr)->is_ptr()) == C->get_alias_index(adr_type), "adr and adr_type must agree");
3107 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
3108 ctl != nullptr, "raw memory operations should have control edge");
3109
3110 switch (bt) {
3111 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
3112 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
3113 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
3114 case T_CHAR:
3115 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
3116 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3117 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
3118 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
3119 case T_METADATA:
3120 case T_ADDRESS:
3121 case T_OBJECT:
3122 case T_ARRAY:
3123 #ifdef _LP64
3124 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
3125 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
3126 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
3127 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
3128 (val->bottom_type()->isa_klassptr() && adr->bottom_type()->isa_rawptr())) {
3129 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
3130 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
3131 }
3132 #endif
3133 {
3134 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
3135 }
3136 default:
3137 guarantee(false, "unexpected basic type %s", type2name(bt));
3138 return (StoreNode*)nullptr;
3139 }
3140 }
3141
3142 //--------------------------bottom_type----------------------------------------
3143 const Type *StoreNode::bottom_type() const {
3144 return Type::MEMORY;
3145 }
3146
3147 //------------------------------hash-------------------------------------------
3148 uint StoreNode::hash() const {
3149 // unroll addition of interesting fields
3150 //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
3151
3152 // Since they are not commoned, do not hash them:
3153 return NO_HASH;
3154 }
3155
3156 // Link together multiple stores (B/S/C/I) into a longer one.
3157 //
3779 }
3780 ss.print_cr("[TraceMergeStores]: with");
3781 merged_input_value->dump("\n", false, &ss);
3782 merged_store->dump("\n", false, &ss);
3783 tty->print("%s", ss.as_string());
3784 }
3785 #endif
3786
3787 //------------------------------Ideal------------------------------------------
3788 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3789 // When a store immediately follows a relevant allocation/initialization,
3790 // try to capture it into the initialization, or hoist it above.
3791 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3792 Node* p = MemNode::Ideal_common(phase, can_reshape);
3793 if (p) return (p == NodeSentinel) ? nullptr : p;
3794
3795 Node* mem = in(MemNode::Memory);
3796 Node* address = in(MemNode::Address);
3797 Node* value = in(MemNode::ValueIn);
3798 // Back-to-back stores to same address? Fold em up. Generally
3799 // unsafe if I have intervening uses...
3800 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3801 Node* st = mem;
3802 // If Store 'st' has more than one use, we cannot fold 'st' away.
3803 // For example, 'st' might be the final state at a conditional
3804 // return. Or, 'st' might be used by some node which is live at
3805 // the same time 'st' is live, which might be unschedulable. So,
3806 // require exactly ONE user until such time as we clone 'mem' for
3807 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3808 // true).
3809 while (st->is_Store() && st->outcnt() == 1) {
3810 // Looking at a dead closed cycle of memory?
3811 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3812 assert(Opcode() == st->Opcode() ||
3813 st->Opcode() == Op_StoreVector ||
3814 Opcode() == Op_StoreVector ||
3815 st->Opcode() == Op_StoreVectorScatter ||
3816 Opcode() == Op_StoreVectorScatter ||
3817 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3818 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3819 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3820 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3821 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3822 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3823
3824 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3825 st->as_Store()->memory_size() <= this->memory_size()) {
3826 Node* use = st->raw_out(0);
3827 if (phase->is_IterGVN()) {
3828 phase->is_IterGVN()->rehash_node_delayed(use);
3829 }
3830 // It's OK to do this in the parser, since DU info is always accurate,
3831 // and the parser always refers to nodes via SafePointNode maps.
3832 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3833 return this;
3834 }
3835 st = st->in(MemNode::Memory);
3836 }
3837 }
3838
3839
3840 // Capture an unaliased, unconditional, simple store into an initializer.
3941 const StoreVectorNode* store_vector = as_StoreVector();
3942 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3943 const Node* store_indices = store_vector->indices();
3944 const Node* mem_indices = mem_vector->indices();
3945 const Node* store_mask = store_vector->mask();
3946 const Node* mem_mask = mem_vector->mask();
3947 // Ensure types, indices, and masks match
3948 if (store_vector->vect_type() == mem_vector->vect_type() &&
3949 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3950 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3951 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3952 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3953 result = mem;
3954 }
3955 }
3956 }
3957
3958 // Store of zero anywhere into a freshly-allocated object?
3959 // Then the store is useless.
3960 // (It must already have been captured by the InitializeNode.)
3961 if (result == this && ReduceFieldZeroing) {
3962 // a newly allocated object is already all-zeroes everywhere
3963 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3964 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3965 result = mem;
3966 }
3967
3968 if (result == this && phase->type(val)->is_zero_type()) {
3969 // the store may also apply to zero-bits in an earlier object
3970 Node* prev_mem = find_previous_store(phase);
3971 // Steps (a), (b): Walk past independent stores to find an exact match.
3972 if (prev_mem != nullptr) {
3973 if (prev_mem->is_top()) {
3974 // find_previous_store returns top when the access is dead
3975 return prev_mem;
3976 }
3977 Node* prev_val = can_see_stored_value(prev_mem, phase);
3978 if (prev_val != nullptr && prev_val == val) {
3979 // prev_val and val might differ by a cast; it would be good
3980 // to keep the more informative of the two.
3981 result = mem;
3982 }
3983 }
3984 }
3985 }
3986
3987 PhaseIterGVN* igvn = phase->is_IterGVN();
3988 if (result != this && igvn != nullptr) {
4461 // Clearing a short array is faster with stores
4462 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4463 // Already know this is a large node, do not try to ideal it
4464 if (_is_large) return nullptr;
4465
4466 const int unit = BytesPerLong;
4467 const TypeX* t = phase->type(in(2))->isa_intptr_t();
4468 if (!t) return nullptr;
4469 if (!t->is_con()) return nullptr;
4470 intptr_t raw_count = t->get_con();
4471 intptr_t size = raw_count;
4472 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4473 // Clearing nothing uses the Identity call.
4474 // Negative clears are possible on dead ClearArrays
4475 // (see jck test stmt114.stmt11402.val).
4476 if (size <= 0 || size % unit != 0) return nullptr;
4477 intptr_t count = size / unit;
4478 // Length too long; communicate this to matchers and assemblers.
4479 // Assemblers are responsible to produce fast hardware clears for it.
4480 if (size > InitArrayShortSize) {
4481 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4482 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4483 return nullptr;
4484 }
4485 if (!IdealizeClearArrayNode) return nullptr;
4486 Node *mem = in(1);
4487 if( phase->type(mem)==Type::TOP ) return nullptr;
4488 Node *adr = in(3);
4489 const Type* at = phase->type(adr);
4490 if( at==Type::TOP ) return nullptr;
4491 const TypePtr* atp = at->isa_ptr();
4492 // adjust atp to be the correct array element address type
4493 if (atp == nullptr) atp = TypePtr::BOTTOM;
4494 else atp = atp->add_offset(Type::OffsetBot);
4495 // Get base for derived pointer purposes
4496 if( adr->Opcode() != Op_AddP ) Unimplemented();
4497 Node *base = adr->in(1);
4498
4499 Node *val = in(4);
4500 Node *off = phase->MakeConX(BytesPerLong);
4501 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4502 count--;
4503 while (count--) {
4504 mem = phase->transform(mem);
4505 adr = phase->transform(AddPNode::make_with_base(base,adr,off));
4506 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4507 }
4508 return mem;
4509 }
4510
4511 //----------------------------step_through----------------------------------
4512 // Return allocation input memory edge if it is different instance
4513 // or itself if it is the one we are looking for.
4514 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4515 Node* n = *np;
4516 assert(n->is_ClearArray(), "sanity");
4517 intptr_t offset;
4518 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4519 // This method is called only before Allocate nodes are expanded
4520 // during macro nodes expansion. Before that ClearArray nodes are
4521 // only generated in PhaseMacroExpand::generate_arraycopy() (before
4522 // Allocate nodes are expanded) which follows allocations.
4523 assert(alloc != nullptr, "should have allocation");
4524 if (alloc->_idx == instance_id) {
4525 // Can not bypass initialization of the instance we are looking for.
4526 return false;
4529 InitializeNode* init = alloc->initialization();
4530 if (init != nullptr)
4531 *np = init->in(TypeFunc::Memory);
4532 else
4533 *np = alloc->in(TypeFunc::Memory);
4534 return true;
4535 }
4536
4537 Node* ClearArrayNode::make_address(Node* dest, Node* offset, bool raw_base, PhaseGVN* phase) {
4538 Node* base = dest;
4539 if (raw_base) {
4540 // May be called as part of the initialization of a just allocated object
4541 base = phase->C->top();
4542 }
4543 return phase->transform(AddPNode::make_with_base(base, dest, offset));
4544 }
4545
4546 //----------------------------clear_memory-------------------------------------
4547 // Generate code to initialize object storage to zero.
4548 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4549 Node* val,
4550 Node* raw_val,
4551 intptr_t start_offset,
4552 Node* end_offset,
4553 bool raw_base,
4554 PhaseGVN* phase) {
4555 intptr_t offset = start_offset;
4556
4557 int unit = BytesPerLong;
4558 if ((offset % unit) != 0) {
4559 Node* adr = make_address(dest, phase->MakeConX(offset), raw_base, phase);
4560 const TypePtr* atp = TypeRawPtr::BOTTOM;
4561 if (val != nullptr) {
4562 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4563 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4564 } else {
4565 assert(raw_val == nullptr, "val may not be null");
4566 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4567 }
4568 mem = phase->transform(mem);
4569 offset += BytesPerInt;
4570 }
4571 assert((offset % unit) == 0, "");
4572
4573 // Initialize the remaining stuff, if any, with a ClearArray.
4574 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, raw_base, phase);
4575 }
4576
4577 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4578 Node* raw_val,
4579 Node* start_offset,
4580 Node* end_offset,
4581 bool raw_base,
4582 PhaseGVN* phase) {
4583 if (start_offset == end_offset) {
4584 // nothing to do
4585 return mem;
4586 }
4587
4588 int unit = BytesPerLong;
4589 Node* zbase = start_offset;
4590 Node* zend = end_offset;
4591
4592 // Scale to the unit required by the CPU:
4593 if (!Matcher::init_array_count_is_in_bytes) {
4594 Node* shift = phase->intcon(exact_log2(unit));
4595 zbase = phase->transform(new URShiftXNode(zbase, shift) );
4596 zend = phase->transform(new URShiftXNode(zend, shift) );
4597 }
4598
4599 // Bulk clear double-words
4600 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4601 Node* adr = make_address(dest, start_offset, raw_base, phase);
4602 if (raw_val == nullptr) {
4603 raw_val = phase->MakeConX(0);
4604 }
4605 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4606 return phase->transform(mem);
4607 }
4608
4609 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4610 Node* val,
4611 Node* raw_val,
4612 intptr_t start_offset,
4613 intptr_t end_offset,
4614 bool raw_base,
4615 PhaseGVN* phase) {
4616 if (start_offset == end_offset) {
4617 // nothing to do
4618 return mem;
4619 }
4620
4621 assert((end_offset % BytesPerInt) == 0, "odd end offset");
4622 intptr_t done_offset = end_offset;
4623 if ((done_offset % BytesPerLong) != 0) {
4624 done_offset -= BytesPerInt;
4625 }
4626 if (done_offset > start_offset) {
4627 mem = clear_memory(ctl, mem, dest, val, raw_val,
4628 start_offset, phase->MakeConX(done_offset), raw_base, phase);
4629 }
4630 if (done_offset < end_offset) { // emit the final 32-bit store
4631 Node* adr = make_address(dest, phase->MakeConX(done_offset), raw_base, phase);
4632 const TypePtr* atp = TypeRawPtr::BOTTOM;
4633 if (val != nullptr) {
4634 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4635 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4636 } else {
4637 assert(raw_val == nullptr, "val may not be null");
4638 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4639 }
4640 mem = phase->transform(mem);
4641 done_offset += BytesPerInt;
4642 }
4643 assert(done_offset == end_offset, "");
4644 return mem;
4645 }
4646
4647 //=============================================================================
4648 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4649 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4650 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4651 #ifdef ASSERT
4652 , _pair_idx(0)
4653 #endif
4654 {
4655 init_class_id(Class_MemBar);
4656 Node* top = C->top();
4657 init_req(TypeFunc::I_O,top);
4658 init_req(TypeFunc::FramePtr,top);
4659 init_req(TypeFunc::ReturnAdr,top);
4768 PhaseIterGVN* igvn = phase->is_IterGVN();
4769 remove(igvn);
4770 // Must return either the original node (now dead) or a new node
4771 // (Do not return a top here, since that would break the uniqueness of top.)
4772 return new ConINode(TypeInt::ZERO);
4773 }
4774 }
4775 return progress ? this : nullptr;
4776 }
4777
4778 //------------------------------Value------------------------------------------
4779 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4780 if( !in(0) ) return Type::TOP;
4781 if( phase->type(in(0)) == Type::TOP )
4782 return Type::TOP;
4783 return TypeTuple::MEMBAR;
4784 }
4785
4786 //------------------------------match------------------------------------------
4787 // Construct projections for memory.
4788 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4789 switch (proj->_con) {
4790 case TypeFunc::Control:
4791 case TypeFunc::Memory:
4792 return new MachProjNode(this, proj->_con, RegMask::EMPTY, MachProjNode::unmatched_proj);
4793 }
4794 ShouldNotReachHere();
4795 return nullptr;
4796 }
4797
4798 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4799 trailing->_kind = TrailingStore;
4800 leading->_kind = LeadingStore;
4801 #ifdef ASSERT
4802 trailing->_pair_idx = leading->_idx;
4803 leading->_pair_idx = leading->_idx;
4804 #endif
4805 }
4806
4807 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4808 trailing->_kind = TrailingLoadStore;
5055 return (req() > RawStores);
5056 }
5057
5058 void InitializeNode::set_complete(PhaseGVN* phase) {
5059 assert(!is_complete(), "caller responsibility");
5060 _is_complete = Complete;
5061
5062 // After this node is complete, it contains a bunch of
5063 // raw-memory initializations. There is no need for
5064 // it to have anything to do with non-raw memory effects.
5065 // Therefore, tell all non-raw users to re-optimize themselves,
5066 // after skipping the memory effects of this initialization.
5067 PhaseIterGVN* igvn = phase->is_IterGVN();
5068 if (igvn) igvn->add_users_to_worklist(this);
5069 }
5070
5071 // convenience function
5072 // return false if the init contains any stores already
5073 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
5074 InitializeNode* init = initialization();
5075 if (init == nullptr || init->is_complete()) {
5076 return false;
5077 }
5078 init->remove_extra_zeroes();
5079 // for now, if this allocation has already collected any inits, bail:
5080 if (init->is_non_zero()) return false;
5081 init->set_complete(phase);
5082 return true;
5083 }
5084
5085 void InitializeNode::remove_extra_zeroes() {
5086 if (req() == RawStores) return;
5087 Node* zmem = zero_memory();
5088 uint fill = RawStores;
5089 for (uint i = fill; i < req(); i++) {
5090 Node* n = in(i);
5091 if (n->is_top() || n == zmem) continue; // skip
5092 if (fill < i) set_req(fill, n); // compact
5093 ++fill;
5094 }
5095 // delete any empty spaces created:
5096 while (fill < req()) {
5097 del_req(fill);
5241 // store node that we'd like to capture. We need to check
5242 // the uses of the MergeMemNode.
5243 mems.push(n);
5244 }
5245 } else if (n->is_Mem()) {
5246 Node* other_adr = n->in(MemNode::Address);
5247 if (other_adr == adr) {
5248 failed = true;
5249 break;
5250 } else {
5251 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5252 if (other_t_adr != nullptr) {
5253 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5254 if (other_alias_idx == alias_idx) {
5255 // A load from the same memory slice as the store right
5256 // after the InitializeNode. We check the control of the
5257 // object/array that is loaded from. If it's the same as
5258 // the store control then we cannot capture the store.
5259 assert(!n->is_Store(), "2 stores to same slice on same control?");
5260 Node* base = other_adr;
5261 if (base->is_Phi()) {
5262 // In rare case, base may be a PhiNode and it may read
5263 // the same memory slice between InitializeNode and store.
5264 failed = true;
5265 break;
5266 }
5267 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5268 base = base->in(AddPNode::Base);
5269 if (base != nullptr) {
5270 base = base->uncast();
5271 if (base->is_Proj() && base->in(0) == alloc) {
5272 failed = true;
5273 break;
5274 }
5275 }
5276 }
5277 }
5278 }
5279 } else {
5280 failed = true;
5281 break;
5282 }
5283 }
5284 }
5285 }
5286 if (failed) {
5832 // z's_done 12 16 16 16 12 16 12
5833 // z's_needed 12 16 16 16 16 16 16
5834 // zsize 0 0 0 0 4 0 4
5835 if (next_full_store < 0) {
5836 // Conservative tack: Zero to end of current word.
5837 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5838 } else {
5839 // Zero to beginning of next fully initialized word.
5840 // Or, don't zero at all, if we are already in that word.
5841 assert(next_full_store >= zeroes_needed, "must go forward");
5842 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5843 zeroes_needed = next_full_store;
5844 }
5845 }
5846
5847 if (zeroes_needed > zeroes_done) {
5848 intptr_t zsize = zeroes_needed - zeroes_done;
5849 // Do some incremental zeroing on rawmem, in parallel with inits.
5850 zeroes_done = align_down(zeroes_done, BytesPerInt);
5851 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5852 allocation()->in(AllocateNode::InitValue),
5853 allocation()->in(AllocateNode::RawInitValue),
5854 zeroes_done, zeroes_needed,
5855 true,
5856 phase);
5857 zeroes_done = zeroes_needed;
5858 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5859 do_zeroing = false; // leave the hole, next time
5860 }
5861 }
5862
5863 // Collect the store and move on:
5864 phase->replace_input_of(st, MemNode::Memory, inits);
5865 inits = st; // put it on the linearized chain
5866 set_req(i, zmem); // unhook from previous position
5867
5868 if (zeroes_done == st_off)
5869 zeroes_done = next_init_off;
5870
5871 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5872
5873 #ifdef ASSERT
5894 remove_extra_zeroes(); // clear out all the zmems left over
5895 add_req(inits);
5896
5897 if (!(UseTLAB && ZeroTLAB)) {
5898 // If anything remains to be zeroed, zero it all now.
5899 zeroes_done = align_down(zeroes_done, BytesPerInt);
5900 // if it is the last unused 4 bytes of an instance, forget about it
5901 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5902 if (zeroes_done + BytesPerLong >= size_limit) {
5903 AllocateNode* alloc = allocation();
5904 assert(alloc != nullptr, "must be present");
5905 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5906 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5907 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5908 if (zeroes_done == k->layout_helper())
5909 zeroes_done = size_limit;
5910 }
5911 }
5912 if (zeroes_done < size_limit) {
5913 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5914 allocation()->in(AllocateNode::InitValue),
5915 allocation()->in(AllocateNode::RawInitValue),
5916 zeroes_done, size_in_bytes, true, phase);
5917 }
5918 }
5919
5920 set_complete(phase);
5921 return rawmem;
5922 }
5923
5924 void InitializeNode::replace_mem_projs_by(Node* mem, Compile* C) {
5925 auto replace_proj = [&](ProjNode* proj) {
5926 C->gvn_replace_by(proj, mem);
5927 return CONTINUE;
5928 };
5929 apply_to_projs(replace_proj, TypeFunc::Memory);
5930 }
5931
5932 void InitializeNode::replace_mem_projs_by(Node* mem, PhaseIterGVN* igvn) {
5933 DUIterator_Fast imax, i = fast_outs(imax);
5934 auto replace_proj = [&](ProjNode* proj) {
5935 igvn->replace_node(proj, mem);
6133 //------------------------------Identity---------------------------------------
6134 Node* MergeMemNode::Identity(PhaseGVN* phase) {
6135 // Identity if this merge point does not record any interesting memory
6136 // disambiguations.
6137 Node* base_mem = base_memory();
6138 Node* empty_mem = empty_memory();
6139 if (base_mem != empty_mem) { // Memory path is not dead?
6140 for (uint i = Compile::AliasIdxRaw; i < req(); i++) {
6141 Node* mem = in(i);
6142 if (mem != empty_mem && mem != base_mem) {
6143 return this; // Many memory splits; no change
6144 }
6145 }
6146 }
6147 return base_mem; // No memory splits; ID on the one true input
6148 }
6149
6150 //------------------------------Ideal------------------------------------------
6151 // This method is invoked recursively on chains of MergeMem nodes
6152 Node *MergeMemNode::Ideal(PhaseGVN *phase, bool can_reshape) {
6153 if (Identity(phase) != this) {
6154 // Let Identity handle this case
6155 return nullptr;
6156 }
6157
6158 // Remove chain'd MergeMems
6159 //
6160 // This is delicate, because the each "in(i)" (i >= Raw) is interpreted
6161 // relative to the "in(Bot)". Since we are patching both at the same time,
6162 // we have to be careful to read each "in(i)" relative to the old "in(Bot)",
6163 // but rewrite each "in(i)" relative to the new "in(Bot)".
6164 Node *progress = nullptr;
6165
6166
6167 Node* old_base = base_memory();
6168 Node* empty_mem = empty_memory();
6169 if (old_base == empty_mem)
6170 return nullptr; // Dead memory path.
6171
6172 MergeMemNode* old_mbase;
6173 if (old_base != nullptr && old_base->is_MergeMem())
6174 old_mbase = old_base->as_MergeMem();
6175 else
6176 old_mbase = nullptr;
6177 Node* new_base = old_base;
|