6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/regalloc.hpp"
38 #include "opto/compile.hpp"
39 #include "opto/connode.hpp"
40 #include "opto/convertnode.hpp"
41 #include "opto/loopnode.hpp"
42 #include "opto/machnode.hpp"
43 #include "opto/matcher.hpp"
44 #include "opto/memnode.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/phaseX.hpp"
48 #include "opto/regmask.hpp"
49 #include "opto/rootnode.hpp"
50 #include "opto/vectornode.hpp"
51 #include "utilities/align.hpp"
52 #include "utilities/copy.hpp"
53 #include "utilities/macros.hpp"
54 #include "utilities/powerOfTwo.hpp"
55 #include "utilities/vmError.hpp"
56
57 // Portions of code courtesy of Clifford Click
58
59 // Optimization - Graph Style
60
214 bool is_instance = t_oop->is_known_instance_field();
215 PhaseIterGVN *igvn = phase->is_IterGVN();
216 if (is_instance && igvn != nullptr && result->is_Phi()) {
217 PhiNode *mphi = result->as_Phi();
218 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
219 const TypePtr *t = mphi->adr_type();
220 bool do_split = false;
221 // In the following cases, Load memory input can be further optimized based on
222 // its precise address type
223 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
224 do_split = true;
225 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
226 const TypeOopPtr* mem_t =
227 t->is_oopptr()->cast_to_exactness(true)
228 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
229 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
230 if (t_oop->is_aryptr()) {
231 mem_t = mem_t->is_aryptr()
232 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
233 ->cast_to_size(t_oop->is_aryptr()->size())
234 ->with_offset(t_oop->is_aryptr()->offset())
235 ->is_aryptr();
236 }
237 do_split = mem_t == t_oop;
238 }
239 if (do_split) {
240 // clone the Phi with our address type
241 result = mphi->split_out_instance(t_adr, igvn);
242 } else {
243 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
244 }
245 }
246 return result;
247 }
248
249 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
250 uint alias_idx = phase->C->get_alias_index(tp);
251 Node *mem = mmem;
252 #ifdef ASSERT
253 {
254 // Check that current type is consistent with the alias index used during graph construction
255 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
256 bool consistent = adr_check == nullptr || adr_check->empty() ||
257 phase->C->must_alias(adr_check, alias_idx );
258 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
259 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
260 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
261 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
262 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
263 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
264 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
265 // don't assert if it is dead code.
266 consistent = true;
267 }
268 if( !consistent ) {
269 st->print("alias_idx==%d, adr_check==", alias_idx);
270 if( adr_check == nullptr ) {
271 st->print("null");
272 } else {
273 adr_check->dump();
274 }
275 st->cr();
276 print_alias_types();
277 assert(consistent, "adr_check must match alias idx");
278 }
279 }
280 #endif
908 "use LoadKlassNode instead");
909 assert(!(adr_type->isa_aryptr() &&
910 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
911 "use LoadRangeNode instead");
912 // Check control edge of raw loads
913 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
914 // oop will be recorded in oop map if load crosses safepoint
915 rt->isa_oopptr() || is_immutable_value(adr),
916 "raw memory operations should have control edge");
917 LoadNode* load = nullptr;
918 switch (bt) {
919 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
920 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
921 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
922 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
923 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
924 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
925 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
926 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
927 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
928 case T_OBJECT:
929 #ifdef _LP64
930 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
931 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
932 } else
933 #endif
934 {
935 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
936 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
937 }
938 break;
939 default:
940 ShouldNotReachHere();
941 break;
942 }
943 assert(load != nullptr, "LoadNode should have been created");
944 if (unaligned) {
945 load->set_unaligned_access();
946 }
947 if (mismatched) {
1003 assert(ld_alloc != nullptr, "need an alloc");
1004 assert(addp->is_AddP(), "address must be addp");
1005 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1006 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1007 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1008 addp->set_req(AddPNode::Base, src);
1009 addp->set_req(AddPNode::Address, src);
1010 } else {
1011 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1012 ac->as_ArrayCopy()->is_copyof_validated() ||
1013 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1014 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1015 addp->set_req(AddPNode::Base, src);
1016 addp->set_req(AddPNode::Address, src);
1017
1018 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1019 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1020 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1021
1022 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1023 uint shift = exact_log2(type2aelembytes(ary_elem));
1024
1025 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1026 #ifdef _LP64
1027 diff = phase->transform(new ConvI2LNode(diff));
1028 #endif
1029 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1030
1031 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1032 addp->set_req(AddPNode::Offset, offset);
1033 }
1034 addp = phase->transform(addp);
1035 #ifdef ASSERT
1036 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1037 ld->_adr_type = adr_type;
1038 #endif
1039 ld->set_req(MemNode::Address, addp);
1040 ld->set_req(0, ctl);
1041 ld->set_req(MemNode::Memory, mem);
1042 // load depends on the tests that validate the arraycopy
1043 ld->_control_dependency = UnknownControl;
1124 // Same base, same offset.
1125 // Possible improvement for arrays: check index value instead of absolute offset.
1126
1127 // At this point we have proven something like this setup:
1128 // B = << base >>
1129 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1130 // S = StoreQ(AddP( B , #Off), V)
1131 // (Actually, we haven't yet proven the Q's are the same.)
1132 // In other words, we are loading from a casted version of
1133 // the same pointer-and-offset that we stored to.
1134 // Casted version may carry a dependency and it is respected.
1135 // Thus, we are able to replace L by V.
1136 }
1137 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1138 if (store_Opcode() != st->Opcode()) {
1139 return nullptr;
1140 }
1141 // LoadVector/StoreVector needs additional check to ensure the types match.
1142 if (st->is_StoreVector()) {
1143 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1144 const TypeVect* out_vt = as_LoadVector()->vect_type();
1145 if (in_vt != out_vt) {
1146 return nullptr;
1147 }
1148 }
1149 return st->in(MemNode::ValueIn);
1150 }
1151
1152 // A load from a freshly-created object always returns zero.
1153 // (This can happen after LoadNode::Ideal resets the load's memory input
1154 // to find_captured_store, which returned InitializeNode::zero_memory.)
1155 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1156 (st->in(0) == ld_alloc) &&
1157 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1158 // return a zero value for the load's basic type
1159 // (This is one of the few places where a generic PhaseTransform
1160 // can create new nodes. Think of it as lazily manifesting
1161 // virtually pre-existing constants.)
1162 if (memory_type() != T_VOID) {
1163 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1164 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1165 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1166 // by the ArrayCopyNode.
1167 return phase->zerocon(memory_type());
1168 }
1169 } else {
1170 // TODO: materialize all-zero vector constant
1171 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1172 }
1173 }
1174
1175 // A load from an initialization barrier can match a captured store.
1176 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1177 InitializeNode* init = st->in(0)->as_Initialize();
1178 AllocateNode* alloc = init->allocation();
1179 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1180 // examine a captured store value
1181 st = init->find_captured_store(ld_off, memory_size(), phase);
1209 //----------------------is_instance_field_load_with_local_phi------------------
1210 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1211 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1212 in(Address)->is_AddP() ) {
1213 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1214 // Only instances and boxed values.
1215 if( t_oop != nullptr &&
1216 (t_oop->is_ptr_to_boxed_value() ||
1217 t_oop->is_known_instance_field()) &&
1218 t_oop->offset() != Type::OffsetBot &&
1219 t_oop->offset() != Type::OffsetTop) {
1220 return true;
1221 }
1222 }
1223 return false;
1224 }
1225
1226 //------------------------------Identity---------------------------------------
1227 // Loads are identity if previous store is to same address
1228 Node* LoadNode::Identity(PhaseGVN* phase) {
1229 // If the previous store-maker is the right kind of Store, and the store is
1230 // to the same address, then we are equal to the value stored.
1231 Node* mem = in(Memory);
1232 Node* value = can_see_stored_value(mem, phase);
1233 if( value ) {
1234 // byte, short & char stores truncate naturally.
1235 // A load has to load the truncated value which requires
1236 // some sort of masking operation and that requires an
1237 // Ideal call instead of an Identity call.
1238 if (memory_size() < BytesPerInt) {
1239 // If the input to the store does not fit with the load's result type,
1240 // it must be truncated via an Ideal call.
1241 if (!phase->type(value)->higher_equal(phase->type(this)))
1242 return this;
1243 }
1244 // (This works even when value is a Con, but LoadNode::Value
1245 // usually runs first, producing the singleton type of the Con.)
1246 if (!has_pinned_control_dependency() || value->is_Con()) {
1247 return value;
1248 } else {
1967 }
1968 }
1969
1970 // Don't do this for integer types. There is only potential profit if
1971 // the element type t is lower than _type; that is, for int types, if _type is
1972 // more restrictive than t. This only happens here if one is short and the other
1973 // char (both 16 bits), and in those cases we've made an intentional decision
1974 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1975 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1976 //
1977 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1978 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
1979 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1980 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
1981 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1982 // In fact, that could have been the original type of p1, and p1 could have
1983 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1984 // expression (LShiftL quux 3) independently optimized to the constant 8.
1985 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
1986 && (_type->isa_vect() == nullptr)
1987 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1988 // t might actually be lower than _type, if _type is a unique
1989 // concrete subclass of abstract class t.
1990 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
1991 const Type* jt = t->join_speculative(_type);
1992 // In any case, do not allow the join, per se, to empty out the type.
1993 if (jt->empty() && !t->empty()) {
1994 // This can happen if a interface-typed array narrows to a class type.
1995 jt = _type;
1996 }
1997 #ifdef ASSERT
1998 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1999 // The pointers in the autobox arrays are always non-null
2000 Node* base = adr->in(AddPNode::Base);
2001 if ((base != nullptr) && base->is_DecodeN()) {
2002 // Get LoadN node which loads IntegerCache.cache field
2003 base = base->in(1);
2004 }
2005 if ((base != nullptr) && base->is_Con()) {
2006 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2007 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2008 // It could be narrow oop
2009 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2010 }
2011 }
2012 }
2013 #endif
2014 return jt;
2015 }
2016 }
2017 } else if (tp->base() == Type::InstPtr) {
2018 assert( off != Type::OffsetBot ||
2019 // arrays can be cast to Objects
2020 !tp->isa_instptr() ||
2021 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2022 // unsafe field access may not have a constant offset
2023 C->has_unsafe_access(),
2024 "Field accesses must be precise" );
2025 // For oop loads, we expect the _type to be precise.
2026
2027 // Optimize loads from constant fields.
2028 const TypeInstPtr* tinst = tp->is_instptr();
2029 ciObject* const_oop = tinst->const_oop();
2030 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2031 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
2032 if (con_type != nullptr) {
2033 return con_type;
2034 }
2035 }
2036 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2037 assert(off != Type::OffsetBot ||
2038 !tp->isa_instklassptr() ||
2039 // arrays can be cast to Objects
2040 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2041 // also allow array-loading from the primary supertype
2042 // array during subtype checks
2043 Opcode() == Op_LoadKlass,
2044 "Field accesses must be precise");
2045 // For klass/static loads, we expect the _type to be precise
2046 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2047 /* With mirrors being an indirect in the Klass*
2048 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2049 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2050 *
2051 * So check the type and klass of the node before the LoadP.
2052 */
2053 Node* adr2 = adr->in(MemNode::Address);
2054 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2055 if (tkls != nullptr && !StressReflectiveCode) {
2056 if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2057 ciKlass* klass = tkls->exact_klass();
2058 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2059 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2060 return TypeInstPtr::make(klass->java_mirror());
2061 }
2062 }
2063 }
2064
2065 const TypeKlassPtr *tkls = tp->isa_klassptr();
2066 if (tkls != nullptr) {
2067 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2068 ciKlass* klass = tkls->exact_klass();
2069 // We are loading a field from a Klass metaobject whose identity
2070 // is known at compile time (the type is "exact" or "precise").
2071 // Check for fields we know are maintained as constants by the VM.
2072 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2073 // The field is Klass::_super_check_offset. Return its (constant) value.
2074 // (Folds up type checking code.)
2075 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2076 return TypeInt::make(klass->super_check_offset());
2077 }
2078 // Compute index into primary_supers array
2079 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2080 // Check for overflowing; use unsigned compare to handle the negative case.
2146 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2147 Node* value = can_see_stored_value(mem,phase);
2148 if (value != nullptr && value->is_Con()) {
2149 assert(value->bottom_type()->higher_equal(_type),"sanity");
2150 return value->bottom_type();
2151 }
2152 }
2153
2154 bool is_vect = (_type->isa_vect() != nullptr);
2155 if (is_instance && !is_vect) {
2156 // If we have an instance type and our memory input is the
2157 // programs's initial memory state, there is no matching store,
2158 // so just return a zero of the appropriate type -
2159 // except if it is vectorized - then we have no zero constant.
2160 Node *mem = in(MemNode::Memory);
2161 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2162 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2163 return Type::get_zero_type(_type->basic_type());
2164 }
2165 }
2166
2167 Node* alloc = is_new_object_mark_load();
2168 if (alloc != nullptr) {
2169 return TypeX::make(markWord::prototype().value());
2170 }
2171
2172 return _type;
2173 }
2174
2175 //------------------------------match_edge-------------------------------------
2176 // Do we Match on this edge index or not? Match only the address.
2177 uint LoadNode::match_edge(uint idx) const {
2178 return idx == MemNode::Address;
2179 }
2180
2181 //--------------------------LoadBNode::Ideal--------------------------------------
2182 //
2183 // If the previous store is to the same address as this load,
2184 // and the value stored was larger than a byte, replace this load
2185 // with the value stored truncated to a byte. If no truncation is
2186 // needed, the replacement is done in LoadNode::Identity().
2187 //
2188 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2189 Node* mem = in(MemNode::Memory);
2300 return LoadNode::Ideal(phase, can_reshape);
2301 }
2302
2303 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2304 Node* mem = in(MemNode::Memory);
2305 Node* value = can_see_stored_value(mem,phase);
2306 if (value != nullptr && value->is_Con() &&
2307 !value->bottom_type()->higher_equal(_type)) {
2308 // If the input to the store does not fit with the load's result type,
2309 // it must be truncated. We can't delay until Ideal call since
2310 // a singleton Value is needed for split_thru_phi optimization.
2311 int con = value->get_int();
2312 return TypeInt::make((con << 16) >> 16);
2313 }
2314 return LoadNode::Value(phase);
2315 }
2316
2317 //=============================================================================
2318 //----------------------------LoadKlassNode::make------------------------------
2319 // Polymorphic factory method:
2320 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2321 // sanity check the alias category against the created node type
2322 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2323 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2324 #ifdef _LP64
2325 if (adr_type->is_ptr_to_narrowklass()) {
2326 assert(UseCompressedClassPointers, "no compressed klasses");
2327 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2328 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2329 }
2330 #endif
2331 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2332 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2333 }
2334
2335 //------------------------------Value------------------------------------------
2336 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2337 return klass_value_common(phase);
2338 }
2339
2340 // In most cases, LoadKlassNode does not have the control input set. If the control
2347 // Either input is TOP ==> the result is TOP
2348 const Type *t1 = phase->type( in(MemNode::Memory) );
2349 if (t1 == Type::TOP) return Type::TOP;
2350 Node *adr = in(MemNode::Address);
2351 const Type *t2 = phase->type( adr );
2352 if (t2 == Type::TOP) return Type::TOP;
2353 const TypePtr *tp = t2->is_ptr();
2354 if (TypePtr::above_centerline(tp->ptr()) ||
2355 tp->ptr() == TypePtr::Null) return Type::TOP;
2356
2357 // Return a more precise klass, if possible
2358 const TypeInstPtr *tinst = tp->isa_instptr();
2359 if (tinst != nullptr) {
2360 ciInstanceKlass* ik = tinst->instance_klass();
2361 int offset = tinst->offset();
2362 if (ik == phase->C->env()->Class_klass()
2363 && (offset == java_lang_Class::klass_offset() ||
2364 offset == java_lang_Class::array_klass_offset())) {
2365 // We are loading a special hidden field from a Class mirror object,
2366 // the field which points to the VM's Klass metaobject.
2367 ciType* t = tinst->java_mirror_type();
2368 // java_mirror_type returns non-null for compile-time Class constants.
2369 if (t != nullptr) {
2370 // constant oop => constant klass
2371 if (offset == java_lang_Class::array_klass_offset()) {
2372 if (t->is_void()) {
2373 // We cannot create a void array. Since void is a primitive type return null
2374 // klass. Users of this result need to do a null check on the returned klass.
2375 return TypePtr::NULL_PTR;
2376 }
2377 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2378 }
2379 if (!t->is_klass()) {
2380 // a primitive Class (e.g., int.class) has null for a klass field
2381 return TypePtr::NULL_PTR;
2382 }
2383 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2384 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2385 }
2386 // non-constant mirror, so we can't tell what's going on
2387 }
2388 if (!tinst->is_loaded())
2389 return _type; // Bail out if not loaded
2390 if (offset == oopDesc::klass_offset_in_bytes()) {
2391 return tinst->as_klass_type(true);
2392 }
2393 }
2394
2395 // Check for loading klass from an array
2396 const TypeAryPtr *tary = tp->isa_aryptr();
2397 if (tary != nullptr &&
2398 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2399 return tary->as_klass_type(true);
2400 }
2401
2402 // Check for loading klass from an array klass
2403 const TypeKlassPtr *tkls = tp->isa_klassptr();
2404 if (tkls != nullptr && !StressReflectiveCode) {
2405 if (!tkls->is_loaded())
2406 return _type; // Bail out if not loaded
2407 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2408 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2409 // // Always returning precise element type is incorrect,
2410 // // e.g., element type could be object and array may contain strings
2411 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2412
2413 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2414 // according to the element type's subclassing.
2415 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2416 }
2602 //=============================================================================
2603 //---------------------------StoreNode::make-----------------------------------
2604 // Polymorphic factory method:
2605 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2606 assert((mo == unordered || mo == release), "unexpected");
2607 Compile* C = gvn.C;
2608 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2609 ctl != nullptr, "raw memory operations should have control edge");
2610
2611 switch (bt) {
2612 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2613 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2614 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2615 case T_CHAR:
2616 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2617 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2618 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2619 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2620 case T_METADATA:
2621 case T_ADDRESS:
2622 case T_OBJECT:
2623 #ifdef _LP64
2624 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2625 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2626 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2627 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2628 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2629 adr->bottom_type()->isa_rawptr())) {
2630 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2631 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2632 }
2633 #endif
2634 {
2635 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2636 }
2637 default:
2638 ShouldNotReachHere();
2639 return (StoreNode*)nullptr;
2640 }
2641 }
2652
2653 // Since they are not commoned, do not hash them:
2654 return NO_HASH;
2655 }
2656
2657 //------------------------------Ideal------------------------------------------
2658 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2659 // When a store immediately follows a relevant allocation/initialization,
2660 // try to capture it into the initialization, or hoist it above.
2661 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2662 Node* p = MemNode::Ideal_common(phase, can_reshape);
2663 if (p) return (p == NodeSentinel) ? nullptr : p;
2664
2665 Node* mem = in(MemNode::Memory);
2666 Node* address = in(MemNode::Address);
2667 Node* value = in(MemNode::ValueIn);
2668 // Back-to-back stores to same address? Fold em up. Generally
2669 // unsafe if I have intervening uses... Also disallowed for StoreCM
2670 // since they must follow each StoreP operation. Redundant StoreCMs
2671 // are eliminated just before matching in final_graph_reshape.
2672 {
2673 Node* st = mem;
2674 // If Store 'st' has more than one use, we cannot fold 'st' away.
2675 // For example, 'st' might be the final state at a conditional
2676 // return. Or, 'st' might be used by some node which is live at
2677 // the same time 'st' is live, which might be unschedulable. So,
2678 // require exactly ONE user until such time as we clone 'mem' for
2679 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2680 // true).
2681 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2682 // Looking at a dead closed cycle of memory?
2683 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2684 assert(Opcode() == st->Opcode() ||
2685 st->Opcode() == Op_StoreVector ||
2686 Opcode() == Op_StoreVector ||
2687 st->Opcode() == Op_StoreVectorScatter ||
2688 Opcode() == Op_StoreVectorScatter ||
2689 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2690 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2691 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2692 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2693 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2694
2695 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2696 st->as_Store()->memory_size() <= this->memory_size()) {
2697 Node* use = st->raw_out(0);
2698 if (phase->is_IterGVN()) {
2699 phase->is_IterGVN()->rehash_node_delayed(use);
2700 }
2701 // It's OK to do this in the parser, since DU info is always accurate,
2702 // and the parser always refers to nodes via SafePointNode maps.
2703 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2704 return this;
2705 }
2706 st = st->in(MemNode::Memory);
2707 }
2708 }
2709
2710
2711 // Capture an unaliased, unconditional, simple store into an initializer.
2768 // Load then Store? Then the Store is useless
2769 if (val->is_Load() &&
2770 val->in(MemNode::Address)->eqv_uncast(adr) &&
2771 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2772 val->as_Load()->store_Opcode() == Opcode()) {
2773 result = mem;
2774 }
2775
2776 // Two stores in a row of the same value?
2777 if (result == this &&
2778 mem->is_Store() &&
2779 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2780 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2781 mem->Opcode() == Opcode()) {
2782 result = mem;
2783 }
2784
2785 // Store of zero anywhere into a freshly-allocated object?
2786 // Then the store is useless.
2787 // (It must already have been captured by the InitializeNode.)
2788 if (result == this &&
2789 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2790 // a newly allocated object is already all-zeroes everywhere
2791 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
2792 result = mem;
2793 }
2794
2795 if (result == this) {
2796 // the store may also apply to zero-bits in an earlier object
2797 Node* prev_mem = find_previous_store(phase);
2798 // Steps (a), (b): Walk past independent stores to find an exact match.
2799 if (prev_mem != nullptr) {
2800 Node* prev_val = can_see_stored_value(prev_mem, phase);
2801 if (prev_val != nullptr && prev_val == val) {
2802 // prev_val and val might differ by a cast; it would be good
2803 // to keep the more informative of the two.
2804 result = mem;
2805 }
2806 }
2807 }
2808 }
2809
2810 PhaseIterGVN* igvn = phase->is_IterGVN();
2811 if (result != this && igvn != nullptr) {
2812 MemBarNode* trailing = trailing_membar();
2813 if (trailing != nullptr) {
2814 #ifdef ASSERT
2815 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2960 Node* StoreCMNode::Identity(PhaseGVN* phase) {
2961 // No need to card mark when storing a null ptr
2962 Node* my_store = in(MemNode::OopStore);
2963 if (my_store->is_Store()) {
2964 const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
2965 if( t1 == TypePtr::NULL_PTR ) {
2966 return in(MemNode::Memory);
2967 }
2968 }
2969 return this;
2970 }
2971
2972 //=============================================================================
2973 //------------------------------Ideal---------------------------------------
2974 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
2975 Node* progress = StoreNode::Ideal(phase, can_reshape);
2976 if (progress != nullptr) return progress;
2977
2978 Node* my_store = in(MemNode::OopStore);
2979 if (my_store->is_MergeMem()) {
2980 Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
2981 set_req_X(MemNode::OopStore, mem, phase);
2982 return this;
2983 }
2984
2985 return nullptr;
2986 }
2987
2988 //------------------------------Value-----------------------------------------
2989 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
2990 // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
2991 // If extra input is TOP ==> the result is TOP
2992 const Type* t = phase->type(in(MemNode::OopStore));
2993 if (t == Type::TOP) {
2994 return Type::TOP;
2995 }
2996 return StoreNode::Value(phase);
2997 }
2998
2999
3000 //=============================================================================
3001 //----------------------------------SCMemProjNode------------------------------
3002 const Type* SCMemProjNode::Value(PhaseGVN* phase) const
3133 // Clearing a short array is faster with stores
3134 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3135 // Already know this is a large node, do not try to ideal it
3136 if (_is_large) return nullptr;
3137
3138 const int unit = BytesPerLong;
3139 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3140 if (!t) return nullptr;
3141 if (!t->is_con()) return nullptr;
3142 intptr_t raw_count = t->get_con();
3143 intptr_t size = raw_count;
3144 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3145 // Clearing nothing uses the Identity call.
3146 // Negative clears are possible on dead ClearArrays
3147 // (see jck test stmt114.stmt11402.val).
3148 if (size <= 0 || size % unit != 0) return nullptr;
3149 intptr_t count = size / unit;
3150 // Length too long; communicate this to matchers and assemblers.
3151 // Assemblers are responsible to produce fast hardware clears for it.
3152 if (size > InitArrayShortSize) {
3153 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3154 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3155 return nullptr;
3156 }
3157 if (!IdealizeClearArrayNode) return nullptr;
3158 Node *mem = in(1);
3159 if( phase->type(mem)==Type::TOP ) return nullptr;
3160 Node *adr = in(3);
3161 const Type* at = phase->type(adr);
3162 if( at==Type::TOP ) return nullptr;
3163 const TypePtr* atp = at->isa_ptr();
3164 // adjust atp to be the correct array element address type
3165 if (atp == nullptr) atp = TypePtr::BOTTOM;
3166 else atp = atp->add_offset(Type::OffsetBot);
3167 // Get base for derived pointer purposes
3168 if( adr->Opcode() != Op_AddP ) Unimplemented();
3169 Node *base = adr->in(1);
3170
3171 Node *zero = phase->makecon(TypeLong::ZERO);
3172 Node *off = phase->MakeConX(BytesPerLong);
3173 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3174 count--;
3175 while( count-- ) {
3176 mem = phase->transform(mem);
3177 adr = phase->transform(new AddPNode(base,adr,off));
3178 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3179 }
3180 return mem;
3181 }
3182
3183 //----------------------------step_through----------------------------------
3184 // Return allocation input memory edge if it is different instance
3185 // or itself if it is the one we are looking for.
3186 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3187 Node* n = *np;
3188 assert(n->is_ClearArray(), "sanity");
3189 intptr_t offset;
3190 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3191 // This method is called only before Allocate nodes are expanded
3192 // during macro nodes expansion. Before that ClearArray nodes are
3193 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3194 // Allocate nodes are expanded) which follows allocations.
3195 assert(alloc != nullptr, "should have allocation");
3196 if (alloc->_idx == instance_id) {
3197 // Can not bypass initialization of the instance we are looking for.
3198 return false;
3199 }
3200 // Otherwise skip it.
3201 InitializeNode* init = alloc->initialization();
3202 if (init != nullptr)
3203 *np = init->in(TypeFunc::Memory);
3204 else
3205 *np = alloc->in(TypeFunc::Memory);
3206 return true;
3207 }
3208
3209 //----------------------------clear_memory-------------------------------------
3210 // Generate code to initialize object storage to zero.
3211 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3212 intptr_t start_offset,
3213 Node* end_offset,
3214 PhaseGVN* phase) {
3215 intptr_t offset = start_offset;
3216
3217 int unit = BytesPerLong;
3218 if ((offset % unit) != 0) {
3219 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3220 adr = phase->transform(adr);
3221 const TypePtr* atp = TypeRawPtr::BOTTOM;
3222 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3223 mem = phase->transform(mem);
3224 offset += BytesPerInt;
3225 }
3226 assert((offset % unit) == 0, "");
3227
3228 // Initialize the remaining stuff, if any, with a ClearArray.
3229 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
3230 }
3231
3232 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3233 Node* start_offset,
3234 Node* end_offset,
3235 PhaseGVN* phase) {
3236 if (start_offset == end_offset) {
3237 // nothing to do
3238 return mem;
3239 }
3240
3241 int unit = BytesPerLong;
3242 Node* zbase = start_offset;
3243 Node* zend = end_offset;
3244
3245 // Scale to the unit required by the CPU:
3246 if (!Matcher::init_array_count_is_in_bytes) {
3247 Node* shift = phase->intcon(exact_log2(unit));
3248 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3249 zend = phase->transform(new URShiftXNode(zend, shift) );
3250 }
3251
3252 // Bulk clear double-words
3253 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3254 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3255 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
3256 return phase->transform(mem);
3257 }
3258
3259 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3260 intptr_t start_offset,
3261 intptr_t end_offset,
3262 PhaseGVN* phase) {
3263 if (start_offset == end_offset) {
3264 // nothing to do
3265 return mem;
3266 }
3267
3268 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3269 intptr_t done_offset = end_offset;
3270 if ((done_offset % BytesPerLong) != 0) {
3271 done_offset -= BytesPerInt;
3272 }
3273 if (done_offset > start_offset) {
3274 mem = clear_memory(ctl, mem, dest,
3275 start_offset, phase->MakeConX(done_offset), phase);
3276 }
3277 if (done_offset < end_offset) { // emit the final 32-bit store
3278 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3279 adr = phase->transform(adr);
3280 const TypePtr* atp = TypeRawPtr::BOTTOM;
3281 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3282 mem = phase->transform(mem);
3283 done_offset += BytesPerInt;
3284 }
3285 assert(done_offset == end_offset, "");
3286 return mem;
3287 }
3288
3289 //=============================================================================
3290 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3291 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3292 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3293 #ifdef ASSERT
3294 , _pair_idx(0)
3295 #endif
3296 {
3297 init_class_id(Class_MemBar);
3298 Node* top = C->top();
3299 init_req(TypeFunc::I_O,top);
3300 init_req(TypeFunc::FramePtr,top);
3301 init_req(TypeFunc::ReturnAdr,top);
3406 PhaseIterGVN* igvn = phase->is_IterGVN();
3407 remove(igvn);
3408 // Must return either the original node (now dead) or a new node
3409 // (Do not return a top here, since that would break the uniqueness of top.)
3410 return new ConINode(TypeInt::ZERO);
3411 }
3412 }
3413 return progress ? this : nullptr;
3414 }
3415
3416 //------------------------------Value------------------------------------------
3417 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3418 if( !in(0) ) return Type::TOP;
3419 if( phase->type(in(0)) == Type::TOP )
3420 return Type::TOP;
3421 return TypeTuple::MEMBAR;
3422 }
3423
3424 //------------------------------match------------------------------------------
3425 // Construct projections for memory.
3426 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3427 switch (proj->_con) {
3428 case TypeFunc::Control:
3429 case TypeFunc::Memory:
3430 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3431 }
3432 ShouldNotReachHere();
3433 return nullptr;
3434 }
3435
3436 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3437 trailing->_kind = TrailingStore;
3438 leading->_kind = LeadingStore;
3439 #ifdef ASSERT
3440 trailing->_pair_idx = leading->_idx;
3441 leading->_pair_idx = leading->_idx;
3442 #endif
3443 }
3444
3445 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3446 trailing->_kind = TrailingLoadStore;
3693 return (req() > RawStores);
3694 }
3695
3696 void InitializeNode::set_complete(PhaseGVN* phase) {
3697 assert(!is_complete(), "caller responsibility");
3698 _is_complete = Complete;
3699
3700 // After this node is complete, it contains a bunch of
3701 // raw-memory initializations. There is no need for
3702 // it to have anything to do with non-raw memory effects.
3703 // Therefore, tell all non-raw users to re-optimize themselves,
3704 // after skipping the memory effects of this initialization.
3705 PhaseIterGVN* igvn = phase->is_IterGVN();
3706 if (igvn) igvn->add_users_to_worklist(this);
3707 }
3708
3709 // convenience function
3710 // return false if the init contains any stores already
3711 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3712 InitializeNode* init = initialization();
3713 if (init == nullptr || init->is_complete()) return false;
3714 init->remove_extra_zeroes();
3715 // for now, if this allocation has already collected any inits, bail:
3716 if (init->is_non_zero()) return false;
3717 init->set_complete(phase);
3718 return true;
3719 }
3720
3721 void InitializeNode::remove_extra_zeroes() {
3722 if (req() == RawStores) return;
3723 Node* zmem = zero_memory();
3724 uint fill = RawStores;
3725 for (uint i = fill; i < req(); i++) {
3726 Node* n = in(i);
3727 if (n->is_top() || n == zmem) continue; // skip
3728 if (fill < i) set_req(fill, n); // compact
3729 ++fill;
3730 }
3731 // delete any empty spaces created:
3732 while (fill < req()) {
3733 del_req(fill);
3871 // store node that we'd like to capture. We need to check
3872 // the uses of the MergeMemNode.
3873 mems.push(n);
3874 }
3875 } else if (n->is_Mem()) {
3876 Node* other_adr = n->in(MemNode::Address);
3877 if (other_adr == adr) {
3878 failed = true;
3879 break;
3880 } else {
3881 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3882 if (other_t_adr != nullptr) {
3883 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3884 if (other_alias_idx == alias_idx) {
3885 // A load from the same memory slice as the store right
3886 // after the InitializeNode. We check the control of the
3887 // object/array that is loaded from. If it's the same as
3888 // the store control then we cannot capture the store.
3889 assert(!n->is_Store(), "2 stores to same slice on same control?");
3890 Node* base = other_adr;
3891 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3892 base = base->in(AddPNode::Base);
3893 if (base != nullptr) {
3894 base = base->uncast();
3895 if (base->is_Proj() && base->in(0) == alloc) {
3896 failed = true;
3897 break;
3898 }
3899 }
3900 }
3901 }
3902 }
3903 } else {
3904 failed = true;
3905 break;
3906 }
3907 }
3908 }
3909 }
3910 if (failed) {
4457 // z's_done 12 16 16 16 12 16 12
4458 // z's_needed 12 16 16 16 16 16 16
4459 // zsize 0 0 0 0 4 0 4
4460 if (next_full_store < 0) {
4461 // Conservative tack: Zero to end of current word.
4462 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4463 } else {
4464 // Zero to beginning of next fully initialized word.
4465 // Or, don't zero at all, if we are already in that word.
4466 assert(next_full_store >= zeroes_needed, "must go forward");
4467 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4468 zeroes_needed = next_full_store;
4469 }
4470 }
4471
4472 if (zeroes_needed > zeroes_done) {
4473 intptr_t zsize = zeroes_needed - zeroes_done;
4474 // Do some incremental zeroing on rawmem, in parallel with inits.
4475 zeroes_done = align_down(zeroes_done, BytesPerInt);
4476 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4477 zeroes_done, zeroes_needed,
4478 phase);
4479 zeroes_done = zeroes_needed;
4480 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4481 do_zeroing = false; // leave the hole, next time
4482 }
4483 }
4484
4485 // Collect the store and move on:
4486 phase->replace_input_of(st, MemNode::Memory, inits);
4487 inits = st; // put it on the linearized chain
4488 set_req(i, zmem); // unhook from previous position
4489
4490 if (zeroes_done == st_off)
4491 zeroes_done = next_init_off;
4492
4493 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4494
4495 #ifdef ASSERT
4496 // Various order invariants. Weaker than stores_are_sane because
4516 remove_extra_zeroes(); // clear out all the zmems left over
4517 add_req(inits);
4518
4519 if (!(UseTLAB && ZeroTLAB)) {
4520 // If anything remains to be zeroed, zero it all now.
4521 zeroes_done = align_down(zeroes_done, BytesPerInt);
4522 // if it is the last unused 4 bytes of an instance, forget about it
4523 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4524 if (zeroes_done + BytesPerLong >= size_limit) {
4525 AllocateNode* alloc = allocation();
4526 assert(alloc != nullptr, "must be present");
4527 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
4528 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4529 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4530 if (zeroes_done == k->layout_helper())
4531 zeroes_done = size_limit;
4532 }
4533 }
4534 if (zeroes_done < size_limit) {
4535 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4536 zeroes_done, size_in_bytes, phase);
4537 }
4538 }
4539
4540 set_complete(phase);
4541 return rawmem;
4542 }
4543
4544
4545 #ifdef ASSERT
4546 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
4547 if (is_complete())
4548 return true; // stores could be anything at this point
4549 assert(allocation() != nullptr, "must be present");
4550 intptr_t last_off = allocation()->minimum_header_size();
4551 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4552 Node* st = in(i);
4553 intptr_t st_off = get_store_offset(st, phase);
4554 if (st_off < 0) continue; // ignore dead garbage
4555 if (last_off > st_off) {
|
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "compiler/compileLog.hpp"
30 #include "gc/shared/barrierSet.hpp"
31 #include "gc/shared/c2/barrierSetC2.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "memory/allocation.inline.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "oops/objArrayKlass.hpp"
36 #include "opto/addnode.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/cfgnode.hpp"
39 #include "opto/regalloc.hpp"
40 #include "opto/compile.hpp"
41 #include "opto/connode.hpp"
42 #include "opto/convertnode.hpp"
43 #include "opto/inlinetypenode.hpp"
44 #include "opto/loopnode.hpp"
45 #include "opto/machnode.hpp"
46 #include "opto/matcher.hpp"
47 #include "opto/memnode.hpp"
48 #include "opto/mulnode.hpp"
49 #include "opto/narrowptrnode.hpp"
50 #include "opto/phaseX.hpp"
51 #include "opto/regmask.hpp"
52 #include "opto/rootnode.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/macros.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #include "utilities/vmError.hpp"
59
60 // Portions of code courtesy of Clifford Click
61
62 // Optimization - Graph Style
63
217 bool is_instance = t_oop->is_known_instance_field();
218 PhaseIterGVN *igvn = phase->is_IterGVN();
219 if (is_instance && igvn != nullptr && result->is_Phi()) {
220 PhiNode *mphi = result->as_Phi();
221 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
222 const TypePtr *t = mphi->adr_type();
223 bool do_split = false;
224 // In the following cases, Load memory input can be further optimized based on
225 // its precise address type
226 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
227 do_split = true;
228 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
229 const TypeOopPtr* mem_t =
230 t->is_oopptr()->cast_to_exactness(true)
231 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
232 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
233 if (t_oop->is_aryptr()) {
234 mem_t = mem_t->is_aryptr()
235 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
236 ->cast_to_size(t_oop->is_aryptr()->size())
237 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
238 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
239 ->with_offset(t_oop->is_aryptr()->offset())
240 ->is_aryptr();
241 }
242 do_split = mem_t == t_oop;
243 }
244 if (do_split) {
245 // clone the Phi with our address type
246 result = mphi->split_out_instance(t_adr, igvn);
247 } else {
248 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
249 }
250 }
251 return result;
252 }
253
254 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
255 uint alias_idx = phase->C->get_alias_index(tp);
256 Node *mem = mmem;
257 #ifdef ASSERT
258 {
259 // Check that current type is consistent with the alias index used during graph construction
260 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
261 bool consistent = adr_check == nullptr || adr_check->empty() ||
262 phase->C->must_alias(adr_check, alias_idx );
263 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
264 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
265 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
266 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
267 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
268 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
269 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
270 // don't assert if it is dead code.
271 consistent = true;
272 }
273 if( !consistent ) {
274 st->print("alias_idx==%d, adr_check==", alias_idx);
275 if( adr_check == nullptr ) {
276 st->print("null");
277 } else {
278 adr_check->dump();
279 }
280 st->cr();
281 print_alias_types();
282 assert(consistent, "adr_check must match alias idx");
283 }
284 }
285 #endif
913 "use LoadKlassNode instead");
914 assert(!(adr_type->isa_aryptr() &&
915 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
916 "use LoadRangeNode instead");
917 // Check control edge of raw loads
918 assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
919 // oop will be recorded in oop map if load crosses safepoint
920 rt->isa_oopptr() || is_immutable_value(adr),
921 "raw memory operations should have control edge");
922 LoadNode* load = nullptr;
923 switch (bt) {
924 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
925 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
926 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
927 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
928 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
929 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
930 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
931 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
932 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
933 case T_PRIMITIVE_OBJECT:
934 case T_OBJECT:
935 #ifdef _LP64
936 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
937 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
938 } else
939 #endif
940 {
941 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
942 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
943 }
944 break;
945 default:
946 ShouldNotReachHere();
947 break;
948 }
949 assert(load != nullptr, "LoadNode should have been created");
950 if (unaligned) {
951 load->set_unaligned_access();
952 }
953 if (mismatched) {
1009 assert(ld_alloc != nullptr, "need an alloc");
1010 assert(addp->is_AddP(), "address must be addp");
1011 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1012 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1013 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1014 addp->set_req(AddPNode::Base, src);
1015 addp->set_req(AddPNode::Address, src);
1016 } else {
1017 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1018 ac->as_ArrayCopy()->is_copyof_validated() ||
1019 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1020 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1021 addp->set_req(AddPNode::Base, src);
1022 addp->set_req(AddPNode::Address, src);
1023
1024 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1025 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1026 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1027
1028 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1029 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1030
1031 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1032 #ifdef _LP64
1033 diff = phase->transform(new ConvI2LNode(diff));
1034 #endif
1035 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1036
1037 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1038 addp->set_req(AddPNode::Offset, offset);
1039 }
1040 addp = phase->transform(addp);
1041 #ifdef ASSERT
1042 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1043 ld->_adr_type = adr_type;
1044 #endif
1045 ld->set_req(MemNode::Address, addp);
1046 ld->set_req(0, ctl);
1047 ld->set_req(MemNode::Memory, mem);
1048 // load depends on the tests that validate the arraycopy
1049 ld->_control_dependency = UnknownControl;
1130 // Same base, same offset.
1131 // Possible improvement for arrays: check index value instead of absolute offset.
1132
1133 // At this point we have proven something like this setup:
1134 // B = << base >>
1135 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1136 // S = StoreQ(AddP( B , #Off), V)
1137 // (Actually, we haven't yet proven the Q's are the same.)
1138 // In other words, we are loading from a casted version of
1139 // the same pointer-and-offset that we stored to.
1140 // Casted version may carry a dependency and it is respected.
1141 // Thus, we are able to replace L by V.
1142 }
1143 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1144 if (store_Opcode() != st->Opcode()) {
1145 return nullptr;
1146 }
1147 // LoadVector/StoreVector needs additional check to ensure the types match.
1148 if (st->is_StoreVector()) {
1149 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1150 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1151 if (in_vt != out_vt) {
1152 return nullptr;
1153 }
1154 }
1155 return st->in(MemNode::ValueIn);
1156 }
1157
1158 // A load from a freshly-created object always returns zero.
1159 // (This can happen after LoadNode::Ideal resets the load's memory input
1160 // to find_captured_store, which returned InitializeNode::zero_memory.)
1161 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1162 (st->in(0) == ld_alloc) &&
1163 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1164 // return a zero value for the load's basic type
1165 // (This is one of the few places where a generic PhaseTransform
1166 // can create new nodes. Think of it as lazily manifesting
1167 // virtually pre-existing constants.)
1168 assert(memory_type() != T_PRIMITIVE_OBJECT, "should not be used for inline types");
1169 Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1170 if (default_value != nullptr) {
1171 return default_value;
1172 }
1173 assert(ld_alloc->in(AllocateNode::RawDefaultValue) == nullptr, "default value may not be null");
1174 if (memory_type() != T_VOID) {
1175 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1176 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1177 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1178 // by the ArrayCopyNode.
1179 return phase->zerocon(memory_type());
1180 }
1181 } else {
1182 // TODO: materialize all-zero vector constant
1183 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1184 }
1185 }
1186
1187 // A load from an initialization barrier can match a captured store.
1188 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1189 InitializeNode* init = st->in(0)->as_Initialize();
1190 AllocateNode* alloc = init->allocation();
1191 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1192 // examine a captured store value
1193 st = init->find_captured_store(ld_off, memory_size(), phase);
1221 //----------------------is_instance_field_load_with_local_phi------------------
1222 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1223 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1224 in(Address)->is_AddP() ) {
1225 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1226 // Only instances and boxed values.
1227 if( t_oop != nullptr &&
1228 (t_oop->is_ptr_to_boxed_value() ||
1229 t_oop->is_known_instance_field()) &&
1230 t_oop->offset() != Type::OffsetBot &&
1231 t_oop->offset() != Type::OffsetTop) {
1232 return true;
1233 }
1234 }
1235 return false;
1236 }
1237
1238 //------------------------------Identity---------------------------------------
1239 // Loads are identity if previous store is to same address
1240 Node* LoadNode::Identity(PhaseGVN* phase) {
1241 // Loading from an InlineType? The InlineType has the values of
1242 // all fields as input. Look for the field with matching offset.
1243 Node* addr = in(Address);
1244 intptr_t offset;
1245 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1246 if (base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1247 Node* value = base->as_InlineType()->field_value_by_offset((int)offset, true);
1248 if (value != nullptr) {
1249 if (Opcode() == Op_LoadN) {
1250 // Encode oop value if we are loading a narrow oop
1251 assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1252 value = phase->transform(new EncodePNode(value, bottom_type()));
1253 }
1254 return value;
1255 }
1256 }
1257
1258 // If the previous store-maker is the right kind of Store, and the store is
1259 // to the same address, then we are equal to the value stored.
1260 Node* mem = in(Memory);
1261 Node* value = can_see_stored_value(mem, phase);
1262 if( value ) {
1263 // byte, short & char stores truncate naturally.
1264 // A load has to load the truncated value which requires
1265 // some sort of masking operation and that requires an
1266 // Ideal call instead of an Identity call.
1267 if (memory_size() < BytesPerInt) {
1268 // If the input to the store does not fit with the load's result type,
1269 // it must be truncated via an Ideal call.
1270 if (!phase->type(value)->higher_equal(phase->type(this)))
1271 return this;
1272 }
1273 // (This works even when value is a Con, but LoadNode::Value
1274 // usually runs first, producing the singleton type of the Con.)
1275 if (!has_pinned_control_dependency() || value->is_Con()) {
1276 return value;
1277 } else {
1996 }
1997 }
1998
1999 // Don't do this for integer types. There is only potential profit if
2000 // the element type t is lower than _type; that is, for int types, if _type is
2001 // more restrictive than t. This only happens here if one is short and the other
2002 // char (both 16 bits), and in those cases we've made an intentional decision
2003 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2004 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2005 //
2006 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2007 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2008 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2009 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2010 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2011 // In fact, that could have been the original type of p1, and p1 could have
2012 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2013 // expression (LShiftL quux 3) independently optimized to the constant 8.
2014 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2015 && (_type->isa_vect() == nullptr)
2016 && !ary->is_flat()
2017 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2018 // t might actually be lower than _type, if _type is a unique
2019 // concrete subclass of abstract class t.
2020 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2021 const Type* jt = t->join_speculative(_type);
2022 // In any case, do not allow the join, per se, to empty out the type.
2023 if (jt->empty() && !t->empty()) {
2024 // This can happen if a interface-typed array narrows to a class type.
2025 jt = _type;
2026 }
2027 #ifdef ASSERT
2028 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2029 // The pointers in the autobox arrays are always non-null
2030 Node* base = adr->in(AddPNode::Base);
2031 if ((base != nullptr) && base->is_DecodeN()) {
2032 // Get LoadN node which loads IntegerCache.cache field
2033 base = base->in(1);
2034 }
2035 if ((base != nullptr) && base->is_Con()) {
2036 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2037 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2038 // It could be narrow oop
2039 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2040 }
2041 }
2042 }
2043 #endif
2044 return jt;
2045 }
2046 }
2047 } else if (tp->base() == Type::InstPtr) {
2048 assert( off != Type::OffsetBot ||
2049 // arrays can be cast to Objects
2050 !tp->isa_instptr() ||
2051 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2052 // Default value load
2053 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2054 // unsafe field access may not have a constant offset
2055 C->has_unsafe_access(),
2056 "Field accesses must be precise" );
2057 // For oop loads, we expect the _type to be precise.
2058
2059 const TypeInstPtr* tinst = tp->is_instptr();
2060 BasicType bt = memory_type();
2061
2062 // Optimize loads from constant fields.
2063 ciObject* const_oop = tinst->const_oop();
2064 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2065 ciType* mirror_type = const_oop->as_instance()->java_mirror_type();
2066 if (mirror_type != nullptr) {
2067 const Type* const_oop = nullptr;
2068 ciInlineKlass* vk = mirror_type->is_inlinetype() ? mirror_type->as_inline_klass() : nullptr;
2069 // Fold default value loads
2070 if (vk != nullptr && off == vk->default_value_offset()) {
2071 const_oop = TypeInstPtr::make(vk->default_instance());
2072 }
2073 // Fold class mirror loads
2074 if (off == java_lang_Class::primary_mirror_offset()) {
2075 const_oop = (vk == nullptr) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->ref_instance());
2076 } else if (off == java_lang_Class::secondary_mirror_offset()) {
2077 const_oop = (vk == nullptr) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->val_instance());
2078 }
2079 if (const_oop != nullptr) {
2080 return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
2081 }
2082 }
2083 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2084 if (con_type != nullptr) {
2085 return con_type;
2086 }
2087 }
2088 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2089 assert(off != Type::OffsetBot ||
2090 !tp->isa_instklassptr() ||
2091 // arrays can be cast to Objects
2092 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2093 // also allow array-loading from the primary supertype
2094 // array during subtype checks
2095 Opcode() == Op_LoadKlass,
2096 "Field accesses must be precise");
2097 // For klass/static loads, we expect the _type to be precise
2098 } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
2099 if (adr->is_Load() && off == 0) {
2100 /* With mirrors being an indirect in the Klass*
2101 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2102 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2103 *
2104 * So check the type and klass of the node before the LoadP.
2105 */
2106 Node* adr2 = adr->in(MemNode::Address);
2107 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2108 if (tkls != nullptr) {
2109 if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2110 ciKlass* klass = tkls->exact_klass();
2111 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2112 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2113 return TypeInstPtr::make(klass->java_mirror());
2114 }
2115 }
2116 } else {
2117 // Check for a load of the default value offset from the InlineKlassFixedBlock:
2118 // LoadI(LoadP(inline_klass, adr_inlineklass_fixed_block_offset), default_value_offset_offset)
2119 intptr_t offset = 0;
2120 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2121 if (base != nullptr && base->is_Load() && offset == in_bytes(InlineKlass::default_value_offset_offset())) {
2122 const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
2123 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact() && tkls->exact_klass()->is_inlinetype() &&
2124 tkls->offset() == in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())) {
2125 assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
2126 assert(Opcode() == Op_LoadI, "must load an int from fixed block");
2127 return TypeInt::make(tkls->exact_klass()->as_inline_klass()->default_value_offset());
2128 }
2129 }
2130 }
2131 }
2132
2133 const TypeKlassPtr *tkls = tp->isa_klassptr();
2134 if (tkls != nullptr) {
2135 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2136 ciKlass* klass = tkls->exact_klass();
2137 // We are loading a field from a Klass metaobject whose identity
2138 // is known at compile time (the type is "exact" or "precise").
2139 // Check for fields we know are maintained as constants by the VM.
2140 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2141 // The field is Klass::_super_check_offset. Return its (constant) value.
2142 // (Folds up type checking code.)
2143 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2144 return TypeInt::make(klass->super_check_offset());
2145 }
2146 // Compute index into primary_supers array
2147 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2148 // Check for overflowing; use unsigned compare to handle the negative case.
2214 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2215 Node* value = can_see_stored_value(mem,phase);
2216 if (value != nullptr && value->is_Con()) {
2217 assert(value->bottom_type()->higher_equal(_type),"sanity");
2218 return value->bottom_type();
2219 }
2220 }
2221
2222 bool is_vect = (_type->isa_vect() != nullptr);
2223 if (is_instance && !is_vect) {
2224 // If we have an instance type and our memory input is the
2225 // programs's initial memory state, there is no matching store,
2226 // so just return a zero of the appropriate type -
2227 // except if it is vectorized - then we have no zero constant.
2228 Node *mem = in(MemNode::Memory);
2229 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2230 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2231 return Type::get_zero_type(_type->basic_type());
2232 }
2233 }
2234 Node* alloc = is_new_object_mark_load();
2235 if (alloc != nullptr) {
2236 if (EnableValhalla) {
2237 // The mark word may contain property bits (inline, flat, null-free)
2238 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2239 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2240 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2241 return TypeX::make(tkls->exact_klass()->prototype_header().value());
2242 }
2243 } else {
2244 return TypeX::make(markWord::prototype().value());
2245 }
2246 }
2247
2248 return _type;
2249 }
2250
2251 //------------------------------match_edge-------------------------------------
2252 // Do we Match on this edge index or not? Match only the address.
2253 uint LoadNode::match_edge(uint idx) const {
2254 return idx == MemNode::Address;
2255 }
2256
2257 //--------------------------LoadBNode::Ideal--------------------------------------
2258 //
2259 // If the previous store is to the same address as this load,
2260 // and the value stored was larger than a byte, replace this load
2261 // with the value stored truncated to a byte. If no truncation is
2262 // needed, the replacement is done in LoadNode::Identity().
2263 //
2264 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2265 Node* mem = in(MemNode::Memory);
2376 return LoadNode::Ideal(phase, can_reshape);
2377 }
2378
2379 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2380 Node* mem = in(MemNode::Memory);
2381 Node* value = can_see_stored_value(mem,phase);
2382 if (value != nullptr && value->is_Con() &&
2383 !value->bottom_type()->higher_equal(_type)) {
2384 // If the input to the store does not fit with the load's result type,
2385 // it must be truncated. We can't delay until Ideal call since
2386 // a singleton Value is needed for split_thru_phi optimization.
2387 int con = value->get_int();
2388 return TypeInt::make((con << 16) >> 16);
2389 }
2390 return LoadNode::Value(phase);
2391 }
2392
2393 //=============================================================================
2394 //----------------------------LoadKlassNode::make------------------------------
2395 // Polymorphic factory method:
2396 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2397 const TypeKlassPtr* tk) {
2398 // sanity check the alias category against the created node type
2399 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2400 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2401 #ifdef _LP64
2402 if (adr_type->is_ptr_to_narrowklass()) {
2403 assert(UseCompressedClassPointers, "no compressed klasses");
2404 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2405 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2406 }
2407 #endif
2408 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2409 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2410 }
2411
2412 //------------------------------Value------------------------------------------
2413 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2414 return klass_value_common(phase);
2415 }
2416
2417 // In most cases, LoadKlassNode does not have the control input set. If the control
2424 // Either input is TOP ==> the result is TOP
2425 const Type *t1 = phase->type( in(MemNode::Memory) );
2426 if (t1 == Type::TOP) return Type::TOP;
2427 Node *adr = in(MemNode::Address);
2428 const Type *t2 = phase->type( adr );
2429 if (t2 == Type::TOP) return Type::TOP;
2430 const TypePtr *tp = t2->is_ptr();
2431 if (TypePtr::above_centerline(tp->ptr()) ||
2432 tp->ptr() == TypePtr::Null) return Type::TOP;
2433
2434 // Return a more precise klass, if possible
2435 const TypeInstPtr *tinst = tp->isa_instptr();
2436 if (tinst != nullptr) {
2437 ciInstanceKlass* ik = tinst->instance_klass();
2438 int offset = tinst->offset();
2439 if (ik == phase->C->env()->Class_klass()
2440 && (offset == java_lang_Class::klass_offset() ||
2441 offset == java_lang_Class::array_klass_offset())) {
2442 // We are loading a special hidden field from a Class mirror object,
2443 // the field which points to the VM's Klass metaobject.
2444 bool null_free = false;
2445 ciType* t = tinst->java_mirror_type(&null_free);
2446 // java_mirror_type returns non-null for compile-time Class constants.
2447 if (t != nullptr) {
2448 // constant oop => constant klass
2449 if (offset == java_lang_Class::array_klass_offset()) {
2450 if (t->is_void()) {
2451 // We cannot create a void array. Since void is a primitive type return null
2452 // klass. Users of this result need to do a null check on the returned klass.
2453 return TypePtr::NULL_PTR;
2454 }
2455 return TypeKlassPtr::make(ciArrayKlass::make(t, null_free), Type::trust_interfaces);
2456 }
2457 if (!t->is_klass()) {
2458 // a primitive Class (e.g., int.class) has null for a klass field
2459 return TypePtr::NULL_PTR;
2460 }
2461 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2462 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2463 }
2464 // non-constant mirror, so we can't tell what's going on
2465 }
2466 if (!tinst->is_loaded())
2467 return _type; // Bail out if not loaded
2468 if (offset == oopDesc::klass_offset_in_bytes()) {
2469 return tinst->as_klass_type(true);
2470 }
2471 }
2472
2473 // Check for loading klass from an array
2474 const TypeAryPtr* tary = tp->isa_aryptr();
2475 if (tary != nullptr &&
2476 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2477 return tary->as_klass_type(true);
2478 }
2479
2480 // Check for loading klass from an array klass
2481 const TypeKlassPtr *tkls = tp->isa_klassptr();
2482 if (tkls != nullptr && !StressReflectiveCode) {
2483 if (!tkls->is_loaded())
2484 return _type; // Bail out if not loaded
2485 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2486 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2487 // // Always returning precise element type is incorrect,
2488 // // e.g., element type could be object and array may contain strings
2489 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2490
2491 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2492 // according to the element type's subclassing.
2493 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2494 }
2680 //=============================================================================
2681 //---------------------------StoreNode::make-----------------------------------
2682 // Polymorphic factory method:
2683 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2684 assert((mo == unordered || mo == release), "unexpected");
2685 Compile* C = gvn.C;
2686 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2687 ctl != nullptr, "raw memory operations should have control edge");
2688
2689 switch (bt) {
2690 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2691 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2692 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2693 case T_CHAR:
2694 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2695 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2696 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2697 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2698 case T_METADATA:
2699 case T_ADDRESS:
2700 case T_PRIMITIVE_OBJECT:
2701 case T_OBJECT:
2702 #ifdef _LP64
2703 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2704 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2705 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2706 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2707 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2708 adr->bottom_type()->isa_rawptr())) {
2709 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2710 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2711 }
2712 #endif
2713 {
2714 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2715 }
2716 default:
2717 ShouldNotReachHere();
2718 return (StoreNode*)nullptr;
2719 }
2720 }
2731
2732 // Since they are not commoned, do not hash them:
2733 return NO_HASH;
2734 }
2735
2736 //------------------------------Ideal------------------------------------------
2737 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2738 // When a store immediately follows a relevant allocation/initialization,
2739 // try to capture it into the initialization, or hoist it above.
2740 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2741 Node* p = MemNode::Ideal_common(phase, can_reshape);
2742 if (p) return (p == NodeSentinel) ? nullptr : p;
2743
2744 Node* mem = in(MemNode::Memory);
2745 Node* address = in(MemNode::Address);
2746 Node* value = in(MemNode::ValueIn);
2747 // Back-to-back stores to same address? Fold em up. Generally
2748 // unsafe if I have intervening uses... Also disallowed for StoreCM
2749 // since they must follow each StoreP operation. Redundant StoreCMs
2750 // are eliminated just before matching in final_graph_reshape.
2751 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
2752 Node* st = mem;
2753 // If Store 'st' has more than one use, we cannot fold 'st' away.
2754 // For example, 'st' might be the final state at a conditional
2755 // return. Or, 'st' might be used by some node which is live at
2756 // the same time 'st' is live, which might be unschedulable. So,
2757 // require exactly ONE user until such time as we clone 'mem' for
2758 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2759 // true).
2760 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2761 // Looking at a dead closed cycle of memory?
2762 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2763 assert(Opcode() == st->Opcode() ||
2764 st->Opcode() == Op_StoreVector ||
2765 Opcode() == Op_StoreVector ||
2766 st->Opcode() == Op_StoreVectorScatter ||
2767 Opcode() == Op_StoreVectorScatter ||
2768 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2769 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2770 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2771 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
2772 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2773 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2774
2775 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2776 st->as_Store()->memory_size() <= this->memory_size()) {
2777 Node* use = st->raw_out(0);
2778 if (phase->is_IterGVN()) {
2779 phase->is_IterGVN()->rehash_node_delayed(use);
2780 }
2781 // It's OK to do this in the parser, since DU info is always accurate,
2782 // and the parser always refers to nodes via SafePointNode maps.
2783 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2784 return this;
2785 }
2786 st = st->in(MemNode::Memory);
2787 }
2788 }
2789
2790
2791 // Capture an unaliased, unconditional, simple store into an initializer.
2848 // Load then Store? Then the Store is useless
2849 if (val->is_Load() &&
2850 val->in(MemNode::Address)->eqv_uncast(adr) &&
2851 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2852 val->as_Load()->store_Opcode() == Opcode()) {
2853 result = mem;
2854 }
2855
2856 // Two stores in a row of the same value?
2857 if (result == this &&
2858 mem->is_Store() &&
2859 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2860 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2861 mem->Opcode() == Opcode()) {
2862 result = mem;
2863 }
2864
2865 // Store of zero anywhere into a freshly-allocated object?
2866 // Then the store is useless.
2867 // (It must already have been captured by the InitializeNode.)
2868 if (result == this && ReduceFieldZeroing) {
2869 // a newly allocated object is already all-zeroes everywhere
2870 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
2871 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
2872 result = mem;
2873 }
2874
2875 if (result == this && phase->type(val)->is_zero_type()) {
2876 // the store may also apply to zero-bits in an earlier object
2877 Node* prev_mem = find_previous_store(phase);
2878 // Steps (a), (b): Walk past independent stores to find an exact match.
2879 if (prev_mem != nullptr) {
2880 Node* prev_val = can_see_stored_value(prev_mem, phase);
2881 if (prev_val != nullptr && prev_val == val) {
2882 // prev_val and val might differ by a cast; it would be good
2883 // to keep the more informative of the two.
2884 result = mem;
2885 }
2886 }
2887 }
2888 }
2889
2890 PhaseIterGVN* igvn = phase->is_IterGVN();
2891 if (result != this && igvn != nullptr) {
2892 MemBarNode* trailing = trailing_membar();
2893 if (trailing != nullptr) {
2894 #ifdef ASSERT
2895 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
3040 Node* StoreCMNode::Identity(PhaseGVN* phase) {
3041 // No need to card mark when storing a null ptr
3042 Node* my_store = in(MemNode::OopStore);
3043 if (my_store->is_Store()) {
3044 const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
3045 if( t1 == TypePtr::NULL_PTR ) {
3046 return in(MemNode::Memory);
3047 }
3048 }
3049 return this;
3050 }
3051
3052 //=============================================================================
3053 //------------------------------Ideal---------------------------------------
3054 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
3055 Node* progress = StoreNode::Ideal(phase, can_reshape);
3056 if (progress != nullptr) return progress;
3057
3058 Node* my_store = in(MemNode::OopStore);
3059 if (my_store->is_MergeMem()) {
3060 if (oop_alias_idx() != phase->C->get_alias_index(TypeAryPtr::INLINES) ||
3061 phase->C->flat_accesses_share_alias()) {
3062 // The alias that was recorded is no longer accurate enough.
3063 Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
3064 set_req_X(MemNode::OopStore, mem, phase);
3065 return this;
3066 }
3067 }
3068
3069 return nullptr;
3070 }
3071
3072 //------------------------------Value-----------------------------------------
3073 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
3074 // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
3075 // If extra input is TOP ==> the result is TOP
3076 const Type* t = phase->type(in(MemNode::OopStore));
3077 if (t == Type::TOP) {
3078 return Type::TOP;
3079 }
3080 return StoreNode::Value(phase);
3081 }
3082
3083
3084 //=============================================================================
3085 //----------------------------------SCMemProjNode------------------------------
3086 const Type* SCMemProjNode::Value(PhaseGVN* phase) const
3217 // Clearing a short array is faster with stores
3218 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3219 // Already know this is a large node, do not try to ideal it
3220 if (_is_large) return nullptr;
3221
3222 const int unit = BytesPerLong;
3223 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3224 if (!t) return nullptr;
3225 if (!t->is_con()) return nullptr;
3226 intptr_t raw_count = t->get_con();
3227 intptr_t size = raw_count;
3228 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3229 // Clearing nothing uses the Identity call.
3230 // Negative clears are possible on dead ClearArrays
3231 // (see jck test stmt114.stmt11402.val).
3232 if (size <= 0 || size % unit != 0) return nullptr;
3233 intptr_t count = size / unit;
3234 // Length too long; communicate this to matchers and assemblers.
3235 // Assemblers are responsible to produce fast hardware clears for it.
3236 if (size > InitArrayShortSize) {
3237 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3238 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3239 return nullptr;
3240 }
3241 if (!IdealizeClearArrayNode) return nullptr;
3242 Node *mem = in(1);
3243 if( phase->type(mem)==Type::TOP ) return nullptr;
3244 Node *adr = in(3);
3245 const Type* at = phase->type(adr);
3246 if( at==Type::TOP ) return nullptr;
3247 const TypePtr* atp = at->isa_ptr();
3248 // adjust atp to be the correct array element address type
3249 if (atp == nullptr) atp = TypePtr::BOTTOM;
3250 else atp = atp->add_offset(Type::OffsetBot);
3251 // Get base for derived pointer purposes
3252 if( adr->Opcode() != Op_AddP ) Unimplemented();
3253 Node *base = adr->in(1);
3254
3255 Node *val = in(4);
3256 Node *off = phase->MakeConX(BytesPerLong);
3257 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3258 count--;
3259 while( count-- ) {
3260 mem = phase->transform(mem);
3261 adr = phase->transform(new AddPNode(base,adr,off));
3262 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3263 }
3264 return mem;
3265 }
3266
3267 //----------------------------step_through----------------------------------
3268 // Return allocation input memory edge if it is different instance
3269 // or itself if it is the one we are looking for.
3270 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3271 Node* n = *np;
3272 assert(n->is_ClearArray(), "sanity");
3273 intptr_t offset;
3274 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3275 // This method is called only before Allocate nodes are expanded
3276 // during macro nodes expansion. Before that ClearArray nodes are
3277 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3278 // Allocate nodes are expanded) which follows allocations.
3279 assert(alloc != nullptr, "should have allocation");
3280 if (alloc->_idx == instance_id) {
3281 // Can not bypass initialization of the instance we are looking for.
3282 return false;
3283 }
3284 // Otherwise skip it.
3285 InitializeNode* init = alloc->initialization();
3286 if (init != nullptr)
3287 *np = init->in(TypeFunc::Memory);
3288 else
3289 *np = alloc->in(TypeFunc::Memory);
3290 return true;
3291 }
3292
3293 //----------------------------clear_memory-------------------------------------
3294 // Generate code to initialize object storage to zero.
3295 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3296 Node* val,
3297 Node* raw_val,
3298 intptr_t start_offset,
3299 Node* end_offset,
3300 PhaseGVN* phase) {
3301 intptr_t offset = start_offset;
3302
3303 int unit = BytesPerLong;
3304 if ((offset % unit) != 0) {
3305 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3306 adr = phase->transform(adr);
3307 const TypePtr* atp = TypeRawPtr::BOTTOM;
3308 if (val != nullptr) {
3309 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3310 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3311 } else {
3312 assert(raw_val == nullptr, "val may not be null");
3313 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3314 }
3315 mem = phase->transform(mem);
3316 offset += BytesPerInt;
3317 }
3318 assert((offset % unit) == 0, "");
3319
3320 // Initialize the remaining stuff, if any, with a ClearArray.
3321 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3322 }
3323
3324 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3325 Node* raw_val,
3326 Node* start_offset,
3327 Node* end_offset,
3328 PhaseGVN* phase) {
3329 if (start_offset == end_offset) {
3330 // nothing to do
3331 return mem;
3332 }
3333
3334 int unit = BytesPerLong;
3335 Node* zbase = start_offset;
3336 Node* zend = end_offset;
3337
3338 // Scale to the unit required by the CPU:
3339 if (!Matcher::init_array_count_is_in_bytes) {
3340 Node* shift = phase->intcon(exact_log2(unit));
3341 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3342 zend = phase->transform(new URShiftXNode(zend, shift) );
3343 }
3344
3345 // Bulk clear double-words
3346 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3347 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3348 if (raw_val == nullptr) {
3349 raw_val = phase->MakeConX(0);
3350 }
3351 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3352 return phase->transform(mem);
3353 }
3354
3355 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3356 Node* val,
3357 Node* raw_val,
3358 intptr_t start_offset,
3359 intptr_t end_offset,
3360 PhaseGVN* phase) {
3361 if (start_offset == end_offset) {
3362 // nothing to do
3363 return mem;
3364 }
3365
3366 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3367 intptr_t done_offset = end_offset;
3368 if ((done_offset % BytesPerLong) != 0) {
3369 done_offset -= BytesPerInt;
3370 }
3371 if (done_offset > start_offset) {
3372 mem = clear_memory(ctl, mem, dest, val, raw_val,
3373 start_offset, phase->MakeConX(done_offset), phase);
3374 }
3375 if (done_offset < end_offset) { // emit the final 32-bit store
3376 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3377 adr = phase->transform(adr);
3378 const TypePtr* atp = TypeRawPtr::BOTTOM;
3379 if (val != nullptr) {
3380 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3381 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3382 } else {
3383 assert(raw_val == nullptr, "val may not be null");
3384 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3385 }
3386 mem = phase->transform(mem);
3387 done_offset += BytesPerInt;
3388 }
3389 assert(done_offset == end_offset, "");
3390 return mem;
3391 }
3392
3393 //=============================================================================
3394 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3395 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3396 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3397 #ifdef ASSERT
3398 , _pair_idx(0)
3399 #endif
3400 {
3401 init_class_id(Class_MemBar);
3402 Node* top = C->top();
3403 init_req(TypeFunc::I_O,top);
3404 init_req(TypeFunc::FramePtr,top);
3405 init_req(TypeFunc::ReturnAdr,top);
3510 PhaseIterGVN* igvn = phase->is_IterGVN();
3511 remove(igvn);
3512 // Must return either the original node (now dead) or a new node
3513 // (Do not return a top here, since that would break the uniqueness of top.)
3514 return new ConINode(TypeInt::ZERO);
3515 }
3516 }
3517 return progress ? this : nullptr;
3518 }
3519
3520 //------------------------------Value------------------------------------------
3521 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3522 if( !in(0) ) return Type::TOP;
3523 if( phase->type(in(0)) == Type::TOP )
3524 return Type::TOP;
3525 return TypeTuple::MEMBAR;
3526 }
3527
3528 //------------------------------match------------------------------------------
3529 // Construct projections for memory.
3530 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
3531 switch (proj->_con) {
3532 case TypeFunc::Control:
3533 case TypeFunc::Memory:
3534 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3535 }
3536 ShouldNotReachHere();
3537 return nullptr;
3538 }
3539
3540 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3541 trailing->_kind = TrailingStore;
3542 leading->_kind = LeadingStore;
3543 #ifdef ASSERT
3544 trailing->_pair_idx = leading->_idx;
3545 leading->_pair_idx = leading->_idx;
3546 #endif
3547 }
3548
3549 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3550 trailing->_kind = TrailingLoadStore;
3797 return (req() > RawStores);
3798 }
3799
3800 void InitializeNode::set_complete(PhaseGVN* phase) {
3801 assert(!is_complete(), "caller responsibility");
3802 _is_complete = Complete;
3803
3804 // After this node is complete, it contains a bunch of
3805 // raw-memory initializations. There is no need for
3806 // it to have anything to do with non-raw memory effects.
3807 // Therefore, tell all non-raw users to re-optimize themselves,
3808 // after skipping the memory effects of this initialization.
3809 PhaseIterGVN* igvn = phase->is_IterGVN();
3810 if (igvn) igvn->add_users_to_worklist(this);
3811 }
3812
3813 // convenience function
3814 // return false if the init contains any stores already
3815 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3816 InitializeNode* init = initialization();
3817 if (init == nullptr || init->is_complete()) {
3818 return false;
3819 }
3820 init->remove_extra_zeroes();
3821 // for now, if this allocation has already collected any inits, bail:
3822 if (init->is_non_zero()) return false;
3823 init->set_complete(phase);
3824 return true;
3825 }
3826
3827 void InitializeNode::remove_extra_zeroes() {
3828 if (req() == RawStores) return;
3829 Node* zmem = zero_memory();
3830 uint fill = RawStores;
3831 for (uint i = fill; i < req(); i++) {
3832 Node* n = in(i);
3833 if (n->is_top() || n == zmem) continue; // skip
3834 if (fill < i) set_req(fill, n); // compact
3835 ++fill;
3836 }
3837 // delete any empty spaces created:
3838 while (fill < req()) {
3839 del_req(fill);
3977 // store node that we'd like to capture. We need to check
3978 // the uses of the MergeMemNode.
3979 mems.push(n);
3980 }
3981 } else if (n->is_Mem()) {
3982 Node* other_adr = n->in(MemNode::Address);
3983 if (other_adr == adr) {
3984 failed = true;
3985 break;
3986 } else {
3987 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3988 if (other_t_adr != nullptr) {
3989 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3990 if (other_alias_idx == alias_idx) {
3991 // A load from the same memory slice as the store right
3992 // after the InitializeNode. We check the control of the
3993 // object/array that is loaded from. If it's the same as
3994 // the store control then we cannot capture the store.
3995 assert(!n->is_Store(), "2 stores to same slice on same control?");
3996 Node* base = other_adr;
3997 if (base->is_Phi()) {
3998 // In rare case, base may be a PhiNode and it may read
3999 // the same memory slice between InitializeNode and store.
4000 failed = true;
4001 break;
4002 }
4003 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4004 base = base->in(AddPNode::Base);
4005 if (base != nullptr) {
4006 base = base->uncast();
4007 if (base->is_Proj() && base->in(0) == alloc) {
4008 failed = true;
4009 break;
4010 }
4011 }
4012 }
4013 }
4014 }
4015 } else {
4016 failed = true;
4017 break;
4018 }
4019 }
4020 }
4021 }
4022 if (failed) {
4569 // z's_done 12 16 16 16 12 16 12
4570 // z's_needed 12 16 16 16 16 16 16
4571 // zsize 0 0 0 0 4 0 4
4572 if (next_full_store < 0) {
4573 // Conservative tack: Zero to end of current word.
4574 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4575 } else {
4576 // Zero to beginning of next fully initialized word.
4577 // Or, don't zero at all, if we are already in that word.
4578 assert(next_full_store >= zeroes_needed, "must go forward");
4579 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4580 zeroes_needed = next_full_store;
4581 }
4582 }
4583
4584 if (zeroes_needed > zeroes_done) {
4585 intptr_t zsize = zeroes_needed - zeroes_done;
4586 // Do some incremental zeroing on rawmem, in parallel with inits.
4587 zeroes_done = align_down(zeroes_done, BytesPerInt);
4588 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4589 allocation()->in(AllocateNode::DefaultValue),
4590 allocation()->in(AllocateNode::RawDefaultValue),
4591 zeroes_done, zeroes_needed,
4592 phase);
4593 zeroes_done = zeroes_needed;
4594 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4595 do_zeroing = false; // leave the hole, next time
4596 }
4597 }
4598
4599 // Collect the store and move on:
4600 phase->replace_input_of(st, MemNode::Memory, inits);
4601 inits = st; // put it on the linearized chain
4602 set_req(i, zmem); // unhook from previous position
4603
4604 if (zeroes_done == st_off)
4605 zeroes_done = next_init_off;
4606
4607 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4608
4609 #ifdef ASSERT
4610 // Various order invariants. Weaker than stores_are_sane because
4630 remove_extra_zeroes(); // clear out all the zmems left over
4631 add_req(inits);
4632
4633 if (!(UseTLAB && ZeroTLAB)) {
4634 // If anything remains to be zeroed, zero it all now.
4635 zeroes_done = align_down(zeroes_done, BytesPerInt);
4636 // if it is the last unused 4 bytes of an instance, forget about it
4637 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4638 if (zeroes_done + BytesPerLong >= size_limit) {
4639 AllocateNode* alloc = allocation();
4640 assert(alloc != nullptr, "must be present");
4641 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
4642 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4643 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4644 if (zeroes_done == k->layout_helper())
4645 zeroes_done = size_limit;
4646 }
4647 }
4648 if (zeroes_done < size_limit) {
4649 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4650 allocation()->in(AllocateNode::DefaultValue),
4651 allocation()->in(AllocateNode::RawDefaultValue),
4652 zeroes_done, size_in_bytes, phase);
4653 }
4654 }
4655
4656 set_complete(phase);
4657 return rawmem;
4658 }
4659
4660
4661 #ifdef ASSERT
4662 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
4663 if (is_complete())
4664 return true; // stores could be anything at this point
4665 assert(allocation() != nullptr, "must be present");
4666 intptr_t last_off = allocation()->minimum_header_size();
4667 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4668 Node* st = in(i);
4669 intptr_t st_off = get_store_offset(st, phase);
4670 if (st_off < 0) continue; // ignore dead garbage
4671 if (last_off > st_off) {
|