6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/regalloc.hpp"
38 #include "opto/compile.hpp"
39 #include "opto/connode.hpp"
40 #include "opto/convertnode.hpp"
41 #include "opto/loopnode.hpp"
42 #include "opto/machnode.hpp"
43 #include "opto/matcher.hpp"
44 #include "opto/memnode.hpp"
45 #include "opto/mempointer.hpp"
46 #include "opto/mulnode.hpp"
47 #include "opto/narrowptrnode.hpp"
48 #include "opto/phaseX.hpp"
49 #include "opto/regmask.hpp"
50 #include "opto/rootnode.hpp"
51 #include "opto/traceMergeStoresTag.hpp"
52 #include "opto/vectornode.hpp"
53 #include "utilities/align.hpp"
54 #include "utilities/copy.hpp"
55 #include "utilities/macros.hpp"
56 #include "utilities/powerOfTwo.hpp"
57 #include "utilities/vmError.hpp"
58
59 // Portions of code courtesy of Clifford Click
60
216 bool is_instance = t_oop->is_known_instance_field();
217 PhaseIterGVN *igvn = phase->is_IterGVN();
218 if (is_instance && igvn != nullptr && result->is_Phi()) {
219 PhiNode *mphi = result->as_Phi();
220 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
221 const TypePtr *t = mphi->adr_type();
222 bool do_split = false;
223 // In the following cases, Load memory input can be further optimized based on
224 // its precise address type
225 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
226 do_split = true;
227 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
228 const TypeOopPtr* mem_t =
229 t->is_oopptr()->cast_to_exactness(true)
230 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
231 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
232 if (t_oop->isa_aryptr()) {
233 mem_t = mem_t->is_aryptr()
234 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
235 ->cast_to_size(t_oop->is_aryptr()->size())
236 ->with_offset(t_oop->is_aryptr()->offset())
237 ->is_aryptr();
238 }
239 do_split = mem_t == t_oop;
240 }
241 if (do_split) {
242 // clone the Phi with our address type
243 result = mphi->split_out_instance(t_adr, igvn);
244 } else {
245 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
246 }
247 }
248 return result;
249 }
250
251 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
252 uint alias_idx = phase->C->get_alias_index(tp);
253 Node *mem = mmem;
254 #ifdef ASSERT
255 {
256 // Check that current type is consistent with the alias index used during graph construction
257 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
258 bool consistent = adr_check == nullptr || adr_check->empty() ||
259 phase->C->must_alias(adr_check, alias_idx );
260 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
261 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
262 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
263 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
264 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
265 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
266 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
267 // don't assert if it is dead code.
268 consistent = true;
269 }
270 if( !consistent ) {
271 st->print("alias_idx==%d, adr_check==", alias_idx);
272 if( adr_check == nullptr ) {
273 st->print("null");
274 } else {
275 adr_check->dump();
276 }
277 st->cr();
278 print_alias_types();
279 assert(consistent, "adr_check must match alias idx");
280 }
281 }
282 #endif
995 Node* ld = gvn.transform(load);
996 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
997 }
998
999 return load;
1000 }
1001
1002 //------------------------------hash-------------------------------------------
1003 uint LoadNode::hash() const {
1004 // unroll addition of interesting fields
1005 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1006 }
1007
1008 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1009 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1010 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1011 bool is_stable_ary = FoldStableValues &&
1012 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1013 tp->isa_aryptr()->is_stable();
1014
1015 return (eliminate_boxing && non_volatile) || is_stable_ary;
1016 }
1017
1018 return false;
1019 }
1020
1021 LoadNode* LoadNode::pin_array_access_node() const {
1022 const TypePtr* adr_type = this->adr_type();
1023 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1024 return clone_pinned();
1025 }
1026 return nullptr;
1027 }
1028
1029 // Is the value loaded previously stored by an arraycopy? If so return
1030 // a load node that reads from the source array so we may be able to
1031 // optimize out the ArrayCopy node later.
1032 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1033 Node* ld_adr = in(MemNode::Address);
1034 intptr_t ld_off = 0;
1035 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1052 assert(ld_alloc != nullptr, "need an alloc");
1053 assert(addp->is_AddP(), "address must be addp");
1054 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1055 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1056 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1057 addp->set_req(AddPNode::Base, src);
1058 addp->set_req(AddPNode::Address, src);
1059 } else {
1060 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1061 ac->as_ArrayCopy()->is_copyof_validated() ||
1062 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1063 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1064 addp->set_req(AddPNode::Base, src);
1065 addp->set_req(AddPNode::Address, src);
1066
1067 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1068 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1069 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1070
1071 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1072 uint shift = exact_log2(type2aelembytes(ary_elem));
1073
1074 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1075 #ifdef _LP64
1076 diff = phase->transform(new ConvI2LNode(diff));
1077 #endif
1078 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1079
1080 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1081 addp->set_req(AddPNode::Offset, offset);
1082 }
1083 addp = phase->transform(addp);
1084 #ifdef ASSERT
1085 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1086 ld->_adr_type = adr_type;
1087 #endif
1088 ld->set_req(MemNode::Address, addp);
1089 ld->set_req(0, ctl);
1090 ld->set_req(MemNode::Memory, mem);
1091 return ld;
1092 }
1171 // Same base, same offset.
1172 // Possible improvement for arrays: check index value instead of absolute offset.
1173
1174 // At this point we have proven something like this setup:
1175 // B = << base >>
1176 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1177 // S = StoreQ(AddP( B , #Off), V)
1178 // (Actually, we haven't yet proven the Q's are the same.)
1179 // In other words, we are loading from a casted version of
1180 // the same pointer-and-offset that we stored to.
1181 // Casted version may carry a dependency and it is respected.
1182 // Thus, we are able to replace L by V.
1183 }
1184 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1185 if (store_Opcode() != st->Opcode()) {
1186 return nullptr;
1187 }
1188 // LoadVector/StoreVector needs additional check to ensure the types match.
1189 if (st->is_StoreVector()) {
1190 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1191 const TypeVect* out_vt = as_LoadVector()->vect_type();
1192 if (in_vt != out_vt) {
1193 return nullptr;
1194 }
1195 }
1196 return st->in(MemNode::ValueIn);
1197 }
1198
1199 // A load from a freshly-created object always returns zero.
1200 // (This can happen after LoadNode::Ideal resets the load's memory input
1201 // to find_captured_store, which returned InitializeNode::zero_memory.)
1202 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1203 (st->in(0) == ld_alloc) &&
1204 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1205 // return a zero value for the load's basic type
1206 // (This is one of the few places where a generic PhaseTransform
1207 // can create new nodes. Think of it as lazily manifesting
1208 // virtually pre-existing constants.)
1209 if (memory_type() != T_VOID) {
1210 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1211 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1212 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1213 // by the ArrayCopyNode.
1214 return phase->zerocon(memory_type());
1215 }
1216 } else {
1217 // TODO: materialize all-zero vector constant
1218 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1219 }
1220 }
1221
1222 // A load from an initialization barrier can match a captured store.
1223 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1224 InitializeNode* init = st->in(0)->as_Initialize();
1225 AllocateNode* alloc = init->allocation();
1226 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1227 // examine a captured store value
1228 st = init->find_captured_store(ld_off, memory_size(), phase);
1256 //----------------------is_instance_field_load_with_local_phi------------------
1257 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1258 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1259 in(Address)->is_AddP() ) {
1260 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1261 // Only instances and boxed values.
1262 if( t_oop != nullptr &&
1263 (t_oop->is_ptr_to_boxed_value() ||
1264 t_oop->is_known_instance_field()) &&
1265 t_oop->offset() != Type::OffsetBot &&
1266 t_oop->offset() != Type::OffsetTop) {
1267 return true;
1268 }
1269 }
1270 return false;
1271 }
1272
1273 //------------------------------Identity---------------------------------------
1274 // Loads are identity if previous store is to same address
1275 Node* LoadNode::Identity(PhaseGVN* phase) {
1276 // If the previous store-maker is the right kind of Store, and the store is
1277 // to the same address, then we are equal to the value stored.
1278 Node* mem = in(Memory);
1279 Node* value = can_see_stored_value(mem, phase);
1280 if( value ) {
1281 // byte, short & char stores truncate naturally.
1282 // A load has to load the truncated value which requires
1283 // some sort of masking operation and that requires an
1284 // Ideal call instead of an Identity call.
1285 if (memory_size() < BytesPerInt) {
1286 // If the input to the store does not fit with the load's result type,
1287 // it must be truncated via an Ideal call.
1288 if (!phase->type(value)->higher_equal(phase->type(this)))
1289 return this;
1290 }
1291 // (This works even when value is a Con, but LoadNode::Value
1292 // usually runs first, producing the singleton type of the Con.)
1293 if (!has_pinned_control_dependency() || value->is_Con()) {
1294 return value;
1295 } else {
1849 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1850 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1851
1852 // Skip up past a SafePoint control. Cannot do this for Stores because
1853 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1854 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1855 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1856 !addr_mark &&
1857 (depends_only_on_test() || has_unknown_control_dependency())) {
1858 ctrl = ctrl->in(0);
1859 set_req(MemNode::Control,ctrl);
1860 progress = true;
1861 }
1862
1863 intptr_t ignore = 0;
1864 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1865 if (base != nullptr
1866 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1867 // Check for useless control edge in some common special cases
1868 if (in(MemNode::Control) != nullptr
1869 && can_remove_control()
1870 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1871 && all_controls_dominate(base, phase->C->start())) {
1872 // A method-invariant, non-null address (constant or 'this' argument).
1873 set_req(MemNode::Control, nullptr);
1874 progress = true;
1875 }
1876 }
1877
1878 Node* mem = in(MemNode::Memory);
1879 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1880
1881 if (can_reshape && (addr_t != nullptr)) {
1882 // try to optimize our memory input
1883 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1884 if (opt_mem != mem) {
1885 set_req_X(MemNode::Memory, opt_mem, phase);
1886 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1887 return this;
1888 }
2053 }
2054 }
2055
2056 // Don't do this for integer types. There is only potential profit if
2057 // the element type t is lower than _type; that is, for int types, if _type is
2058 // more restrictive than t. This only happens here if one is short and the other
2059 // char (both 16 bits), and in those cases we've made an intentional decision
2060 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2061 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2062 //
2063 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2064 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2065 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2066 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2067 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2068 // In fact, that could have been the original type of p1, and p1 could have
2069 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2070 // expression (LShiftL quux 3) independently optimized to the constant 8.
2071 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2072 && (_type->isa_vect() == nullptr)
2073 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2074 // t might actually be lower than _type, if _type is a unique
2075 // concrete subclass of abstract class t.
2076 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2077 const Type* jt = t->join_speculative(_type);
2078 // In any case, do not allow the join, per se, to empty out the type.
2079 if (jt->empty() && !t->empty()) {
2080 // This can happen if a interface-typed array narrows to a class type.
2081 jt = _type;
2082 }
2083 #ifdef ASSERT
2084 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2085 // The pointers in the autobox arrays are always non-null
2086 Node* base = adr->in(AddPNode::Base);
2087 if ((base != nullptr) && base->is_DecodeN()) {
2088 // Get LoadN node which loads IntegerCache.cache field
2089 base = base->in(1);
2090 }
2091 if ((base != nullptr) && base->is_Con()) {
2092 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2093 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2094 // It could be narrow oop
2095 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2096 }
2097 }
2098 }
2099 #endif
2100 return jt;
2101 }
2102 }
2103 } else if (tp->base() == Type::InstPtr) {
2104 assert( off != Type::OffsetBot ||
2105 // arrays can be cast to Objects
2106 !tp->isa_instptr() ||
2107 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2108 // unsafe field access may not have a constant offset
2109 C->has_unsafe_access(),
2110 "Field accesses must be precise" );
2111 // For oop loads, we expect the _type to be precise.
2112
2113 // Optimize loads from constant fields.
2114 const TypeInstPtr* tinst = tp->is_instptr();
2115 ciObject* const_oop = tinst->const_oop();
2116 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2117 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
2118 if (con_type != nullptr) {
2119 return con_type;
2120 }
2121 }
2122 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2123 assert(off != Type::OffsetBot ||
2124 !tp->isa_instklassptr() ||
2125 // arrays can be cast to Objects
2126 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2127 // also allow array-loading from the primary supertype
2128 // array during subtype checks
2129 Opcode() == Op_LoadKlass,
2130 "Field accesses must be precise");
2131 // For klass/static loads, we expect the _type to be precise
2132 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2133 /* With mirrors being an indirect in the Klass*
2134 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2135 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2136 *
2137 * So check the type and klass of the node before the LoadP.
2144 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2145 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2146 return TypeInstPtr::make(klass->java_mirror());
2147 }
2148 }
2149 }
2150
2151 const TypeKlassPtr *tkls = tp->isa_klassptr();
2152 if (tkls != nullptr) {
2153 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2154 ciKlass* klass = tkls->exact_klass();
2155 // We are loading a field from a Klass metaobject whose identity
2156 // is known at compile time (the type is "exact" or "precise").
2157 // Check for fields we know are maintained as constants by the VM.
2158 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2159 // The field is Klass::_super_check_offset. Return its (constant) value.
2160 // (Folds up type checking code.)
2161 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2162 return TypeInt::make(klass->super_check_offset());
2163 }
2164 if (UseCompactObjectHeaders) {
2165 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2166 // The field is Klass::_prototype_header. Return its (constant) value.
2167 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2168 return TypeX::make(klass->prototype_header());
2169 }
2170 }
2171 // Compute index into primary_supers array
2172 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2173 // Check for overflowing; use unsigned compare to handle the negative case.
2174 if( depth < ciKlass::primary_super_limit() ) {
2175 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2176 // (Folds up type checking code.)
2177 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2178 ciKlass *ss = klass->super_of_depth(depth);
2179 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2180 }
2181 const Type* aift = load_array_final_field(tkls, klass);
2182 if (aift != nullptr) return aift;
2183 }
2184
2225 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2226 // The key property of this type is that it folds up tests
2227 // for array-ness, since it proves that the layout_helper is positive.
2228 // Thus, a generic value like the basic object layout helper works fine.
2229 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2230 }
2231 }
2232
2233 bool is_vect = (_type->isa_vect() != nullptr);
2234 if (is_instance && !is_vect) {
2235 // If we have an instance type and our memory input is the
2236 // programs's initial memory state, there is no matching store,
2237 // so just return a zero of the appropriate type -
2238 // except if it is vectorized - then we have no zero constant.
2239 Node *mem = in(MemNode::Memory);
2240 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2241 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2242 return Type::get_zero_type(_type->basic_type());
2243 }
2244 }
2245
2246 if (!UseCompactObjectHeaders) {
2247 Node* alloc = is_new_object_mark_load();
2248 if (alloc != nullptr) {
2249 return TypeX::make(markWord::prototype().value());
2250 }
2251 }
2252
2253 return _type;
2254 }
2255
2256 //------------------------------match_edge-------------------------------------
2257 // Do we Match on this edge index or not? Match only the address.
2258 uint LoadNode::match_edge(uint idx) const {
2259 return idx == MemNode::Address;
2260 }
2261
2262 //--------------------------LoadBNode::Ideal--------------------------------------
2263 //
2264 // If the previous store is to the same address as this load,
2265 // and the value stored was larger than a byte, replace this load
2266 // with the value stored truncated to a byte. If no truncation is
2267 // needed, the replacement is done in LoadNode::Identity().
2268 //
2269 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2381 return LoadNode::Ideal(phase, can_reshape);
2382 }
2383
2384 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2385 Node* mem = in(MemNode::Memory);
2386 Node* value = can_see_stored_value(mem,phase);
2387 if (value != nullptr && value->is_Con() &&
2388 !value->bottom_type()->higher_equal(_type)) {
2389 // If the input to the store does not fit with the load's result type,
2390 // it must be truncated. We can't delay until Ideal call since
2391 // a singleton Value is needed for split_thru_phi optimization.
2392 int con = value->get_int();
2393 return TypeInt::make((con << 16) >> 16);
2394 }
2395 return LoadNode::Value(phase);
2396 }
2397
2398 //=============================================================================
2399 //----------------------------LoadKlassNode::make------------------------------
2400 // Polymorphic factory method:
2401 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2402 // sanity check the alias category against the created node type
2403 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2404 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2405 #ifdef _LP64
2406 if (adr_type->is_ptr_to_narrowklass()) {
2407 assert(UseCompressedClassPointers, "no compressed klasses");
2408 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2409 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2410 }
2411 #endif
2412 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2413 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2414 }
2415
2416 //------------------------------Value------------------------------------------
2417 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2418 return klass_value_common(phase);
2419 }
2420
2421 // In most cases, LoadKlassNode does not have the control input set. If the control
2428 // Either input is TOP ==> the result is TOP
2429 const Type *t1 = phase->type( in(MemNode::Memory) );
2430 if (t1 == Type::TOP) return Type::TOP;
2431 Node *adr = in(MemNode::Address);
2432 const Type *t2 = phase->type( adr );
2433 if (t2 == Type::TOP) return Type::TOP;
2434 const TypePtr *tp = t2->is_ptr();
2435 if (TypePtr::above_centerline(tp->ptr()) ||
2436 tp->ptr() == TypePtr::Null) return Type::TOP;
2437
2438 // Return a more precise klass, if possible
2439 const TypeInstPtr *tinst = tp->isa_instptr();
2440 if (tinst != nullptr) {
2441 ciInstanceKlass* ik = tinst->instance_klass();
2442 int offset = tinst->offset();
2443 if (ik == phase->C->env()->Class_klass()
2444 && (offset == java_lang_Class::klass_offset() ||
2445 offset == java_lang_Class::array_klass_offset())) {
2446 // We are loading a special hidden field from a Class mirror object,
2447 // the field which points to the VM's Klass metaobject.
2448 ciType* t = tinst->java_mirror_type();
2449 // java_mirror_type returns non-null for compile-time Class constants.
2450 if (t != nullptr) {
2451 // constant oop => constant klass
2452 if (offset == java_lang_Class::array_klass_offset()) {
2453 if (t->is_void()) {
2454 // We cannot create a void array. Since void is a primitive type return null
2455 // klass. Users of this result need to do a null check on the returned klass.
2456 return TypePtr::NULL_PTR;
2457 }
2458 return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2459 }
2460 if (!t->is_klass()) {
2461 // a primitive Class (e.g., int.class) has null for a klass field
2462 return TypePtr::NULL_PTR;
2463 }
2464 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2465 return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2466 }
2467 // non-constant mirror, so we can't tell what's going on
2468 }
2469 if (!tinst->is_loaded())
2470 return _type; // Bail out if not loaded
2471 if (offset == oopDesc::klass_offset_in_bytes()) {
2472 return tinst->as_klass_type(true);
2473 }
2474 }
2475
2476 // Check for loading klass from an array
2477 const TypeAryPtr *tary = tp->isa_aryptr();
2478 if (tary != nullptr &&
2479 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2480 return tary->as_klass_type(true);
2481 }
2482
2483 // Check for loading klass from an array klass
2484 const TypeKlassPtr *tkls = tp->isa_klassptr();
2485 if (tkls != nullptr && !StressReflectiveCode) {
2486 if (!tkls->is_loaded())
2487 return _type; // Bail out if not loaded
2488 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2489 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2490 // // Always returning precise element type is incorrect,
2491 // // e.g., element type could be object and array may contain strings
2492 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2493
2494 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2495 // according to the element type's subclassing.
2496 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2497 }
3294 }
3295 ss.print_cr("[TraceMergeStores]: with");
3296 merged_input_value->dump("\n", false, &ss);
3297 merged_store->dump("\n", false, &ss);
3298 tty->print("%s", ss.as_string());
3299 }
3300 #endif
3301
3302 //------------------------------Ideal------------------------------------------
3303 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3304 // When a store immediately follows a relevant allocation/initialization,
3305 // try to capture it into the initialization, or hoist it above.
3306 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3307 Node* p = MemNode::Ideal_common(phase, can_reshape);
3308 if (p) return (p == NodeSentinel) ? nullptr : p;
3309
3310 Node* mem = in(MemNode::Memory);
3311 Node* address = in(MemNode::Address);
3312 Node* value = in(MemNode::ValueIn);
3313 // Back-to-back stores to same address? Fold em up. Generally
3314 // unsafe if I have intervening uses.
3315 {
3316 Node* st = mem;
3317 // If Store 'st' has more than one use, we cannot fold 'st' away.
3318 // For example, 'st' might be the final state at a conditional
3319 // return. Or, 'st' might be used by some node which is live at
3320 // the same time 'st' is live, which might be unschedulable. So,
3321 // require exactly ONE user until such time as we clone 'mem' for
3322 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3323 // true).
3324 while (st->is_Store() && st->outcnt() == 1) {
3325 // Looking at a dead closed cycle of memory?
3326 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3327 assert(Opcode() == st->Opcode() ||
3328 st->Opcode() == Op_StoreVector ||
3329 Opcode() == Op_StoreVector ||
3330 st->Opcode() == Op_StoreVectorScatter ||
3331 Opcode() == Op_StoreVectorScatter ||
3332 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3333 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3334 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3335 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3336 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3337
3338 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3339 st->as_Store()->memory_size() <= this->memory_size()) {
3340 Node* use = st->raw_out(0);
3341 if (phase->is_IterGVN()) {
3342 phase->is_IterGVN()->rehash_node_delayed(use);
3343 }
3344 // It's OK to do this in the parser, since DU info is always accurate,
3345 // and the parser always refers to nodes via SafePointNode maps.
3346 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3347 return this;
3348 }
3349 st = st->in(MemNode::Memory);
3350 }
3351 }
3352
3353
3354 // Capture an unaliased, unconditional, simple store into an initializer.
3441 const StoreVectorNode* store_vector = as_StoreVector();
3442 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3443 const Node* store_indices = store_vector->indices();
3444 const Node* mem_indices = mem_vector->indices();
3445 const Node* store_mask = store_vector->mask();
3446 const Node* mem_mask = mem_vector->mask();
3447 // Ensure types, indices, and masks match
3448 if (store_vector->vect_type() == mem_vector->vect_type() &&
3449 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3450 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3451 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3452 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3453 result = mem;
3454 }
3455 }
3456 }
3457
3458 // Store of zero anywhere into a freshly-allocated object?
3459 // Then the store is useless.
3460 // (It must already have been captured by the InitializeNode.)
3461 if (result == this &&
3462 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3463 // a newly allocated object is already all-zeroes everywhere
3464 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
3465 result = mem;
3466 }
3467
3468 if (result == this) {
3469 // the store may also apply to zero-bits in an earlier object
3470 Node* prev_mem = find_previous_store(phase);
3471 // Steps (a), (b): Walk past independent stores to find an exact match.
3472 if (prev_mem != nullptr) {
3473 Node* prev_val = can_see_stored_value(prev_mem, phase);
3474 if (prev_val != nullptr && prev_val == val) {
3475 // prev_val and val might differ by a cast; it would be good
3476 // to keep the more informative of the two.
3477 result = mem;
3478 }
3479 }
3480 }
3481 }
3482
3483 PhaseIterGVN* igvn = phase->is_IterGVN();
3484 if (result != this && igvn != nullptr) {
3485 MemBarNode* trailing = trailing_membar();
3486 if (trailing != nullptr) {
3487 #ifdef ASSERT
3488 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
3764 // Clearing a short array is faster with stores
3765 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3766 // Already know this is a large node, do not try to ideal it
3767 if (_is_large) return nullptr;
3768
3769 const int unit = BytesPerLong;
3770 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3771 if (!t) return nullptr;
3772 if (!t->is_con()) return nullptr;
3773 intptr_t raw_count = t->get_con();
3774 intptr_t size = raw_count;
3775 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3776 // Clearing nothing uses the Identity call.
3777 // Negative clears are possible on dead ClearArrays
3778 // (see jck test stmt114.stmt11402.val).
3779 if (size <= 0 || size % unit != 0) return nullptr;
3780 intptr_t count = size / unit;
3781 // Length too long; communicate this to matchers and assemblers.
3782 // Assemblers are responsible to produce fast hardware clears for it.
3783 if (size > InitArrayShortSize) {
3784 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3785 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3786 return nullptr;
3787 }
3788 if (!IdealizeClearArrayNode) return nullptr;
3789 Node *mem = in(1);
3790 if( phase->type(mem)==Type::TOP ) return nullptr;
3791 Node *adr = in(3);
3792 const Type* at = phase->type(adr);
3793 if( at==Type::TOP ) return nullptr;
3794 const TypePtr* atp = at->isa_ptr();
3795 // adjust atp to be the correct array element address type
3796 if (atp == nullptr) atp = TypePtr::BOTTOM;
3797 else atp = atp->add_offset(Type::OffsetBot);
3798 // Get base for derived pointer purposes
3799 if( adr->Opcode() != Op_AddP ) Unimplemented();
3800 Node *base = adr->in(1);
3801
3802 Node *zero = phase->makecon(TypeLong::ZERO);
3803 Node *off = phase->MakeConX(BytesPerLong);
3804 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3805 count--;
3806 while( count-- ) {
3807 mem = phase->transform(mem);
3808 adr = phase->transform(new AddPNode(base,adr,off));
3809 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3810 }
3811 return mem;
3812 }
3813
3814 //----------------------------step_through----------------------------------
3815 // Return allocation input memory edge if it is different instance
3816 // or itself if it is the one we are looking for.
3817 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3818 Node* n = *np;
3819 assert(n->is_ClearArray(), "sanity");
3820 intptr_t offset;
3821 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3822 // This method is called only before Allocate nodes are expanded
3823 // during macro nodes expansion. Before that ClearArray nodes are
3824 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3825 // Allocate nodes are expanded) which follows allocations.
3826 assert(alloc != nullptr, "should have allocation");
3827 if (alloc->_idx == instance_id) {
3828 // Can not bypass initialization of the instance we are looking for.
3829 return false;
3830 }
3831 // Otherwise skip it.
3832 InitializeNode* init = alloc->initialization();
3833 if (init != nullptr)
3834 *np = init->in(TypeFunc::Memory);
3835 else
3836 *np = alloc->in(TypeFunc::Memory);
3837 return true;
3838 }
3839
3840 //----------------------------clear_memory-------------------------------------
3841 // Generate code to initialize object storage to zero.
3842 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3843 intptr_t start_offset,
3844 Node* end_offset,
3845 PhaseGVN* phase) {
3846 intptr_t offset = start_offset;
3847
3848 int unit = BytesPerLong;
3849 if ((offset % unit) != 0) {
3850 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3851 adr = phase->transform(adr);
3852 const TypePtr* atp = TypeRawPtr::BOTTOM;
3853 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3854 mem = phase->transform(mem);
3855 offset += BytesPerInt;
3856 }
3857 assert((offset % unit) == 0, "");
3858
3859 // Initialize the remaining stuff, if any, with a ClearArray.
3860 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
3861 }
3862
3863 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3864 Node* start_offset,
3865 Node* end_offset,
3866 PhaseGVN* phase) {
3867 if (start_offset == end_offset) {
3868 // nothing to do
3869 return mem;
3870 }
3871
3872 int unit = BytesPerLong;
3873 Node* zbase = start_offset;
3874 Node* zend = end_offset;
3875
3876 // Scale to the unit required by the CPU:
3877 if (!Matcher::init_array_count_is_in_bytes) {
3878 Node* shift = phase->intcon(exact_log2(unit));
3879 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3880 zend = phase->transform(new URShiftXNode(zend, shift) );
3881 }
3882
3883 // Bulk clear double-words
3884 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3885 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3886 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
3887 return phase->transform(mem);
3888 }
3889
3890 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3891 intptr_t start_offset,
3892 intptr_t end_offset,
3893 PhaseGVN* phase) {
3894 if (start_offset == end_offset) {
3895 // nothing to do
3896 return mem;
3897 }
3898
3899 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3900 intptr_t done_offset = end_offset;
3901 if ((done_offset % BytesPerLong) != 0) {
3902 done_offset -= BytesPerInt;
3903 }
3904 if (done_offset > start_offset) {
3905 mem = clear_memory(ctl, mem, dest,
3906 start_offset, phase->MakeConX(done_offset), phase);
3907 }
3908 if (done_offset < end_offset) { // emit the final 32-bit store
3909 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3910 adr = phase->transform(adr);
3911 const TypePtr* atp = TypeRawPtr::BOTTOM;
3912 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3913 mem = phase->transform(mem);
3914 done_offset += BytesPerInt;
3915 }
3916 assert(done_offset == end_offset, "");
3917 return mem;
3918 }
3919
3920 //=============================================================================
3921 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3922 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3923 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3924 #ifdef ASSERT
3925 , _pair_idx(0)
3926 #endif
3927 {
3928 init_class_id(Class_MemBar);
3929 Node* top = C->top();
3930 init_req(TypeFunc::I_O,top);
3931 init_req(TypeFunc::FramePtr,top);
3932 init_req(TypeFunc::ReturnAdr,top);
4038 PhaseIterGVN* igvn = phase->is_IterGVN();
4039 remove(igvn);
4040 // Must return either the original node (now dead) or a new node
4041 // (Do not return a top here, since that would break the uniqueness of top.)
4042 return new ConINode(TypeInt::ZERO);
4043 }
4044 }
4045 return progress ? this : nullptr;
4046 }
4047
4048 //------------------------------Value------------------------------------------
4049 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4050 if( !in(0) ) return Type::TOP;
4051 if( phase->type(in(0)) == Type::TOP )
4052 return Type::TOP;
4053 return TypeTuple::MEMBAR;
4054 }
4055
4056 //------------------------------match------------------------------------------
4057 // Construct projections for memory.
4058 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4059 switch (proj->_con) {
4060 case TypeFunc::Control:
4061 case TypeFunc::Memory:
4062 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4063 }
4064 ShouldNotReachHere();
4065 return nullptr;
4066 }
4067
4068 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4069 trailing->_kind = TrailingStore;
4070 leading->_kind = LeadingStore;
4071 #ifdef ASSERT
4072 trailing->_pair_idx = leading->_idx;
4073 leading->_pair_idx = leading->_idx;
4074 #endif
4075 }
4076
4077 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4078 trailing->_kind = TrailingLoadStore;
4325 return (req() > RawStores);
4326 }
4327
4328 void InitializeNode::set_complete(PhaseGVN* phase) {
4329 assert(!is_complete(), "caller responsibility");
4330 _is_complete = Complete;
4331
4332 // After this node is complete, it contains a bunch of
4333 // raw-memory initializations. There is no need for
4334 // it to have anything to do with non-raw memory effects.
4335 // Therefore, tell all non-raw users to re-optimize themselves,
4336 // after skipping the memory effects of this initialization.
4337 PhaseIterGVN* igvn = phase->is_IterGVN();
4338 if (igvn) igvn->add_users_to_worklist(this);
4339 }
4340
4341 // convenience function
4342 // return false if the init contains any stores already
4343 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4344 InitializeNode* init = initialization();
4345 if (init == nullptr || init->is_complete()) return false;
4346 init->remove_extra_zeroes();
4347 // for now, if this allocation has already collected any inits, bail:
4348 if (init->is_non_zero()) return false;
4349 init->set_complete(phase);
4350 return true;
4351 }
4352
4353 void InitializeNode::remove_extra_zeroes() {
4354 if (req() == RawStores) return;
4355 Node* zmem = zero_memory();
4356 uint fill = RawStores;
4357 for (uint i = fill; i < req(); i++) {
4358 Node* n = in(i);
4359 if (n->is_top() || n == zmem) continue; // skip
4360 if (fill < i) set_req(fill, n); // compact
4361 ++fill;
4362 }
4363 // delete any empty spaces created:
4364 while (fill < req()) {
4365 del_req(fill);
4509 // store node that we'd like to capture. We need to check
4510 // the uses of the MergeMemNode.
4511 mems.push(n);
4512 }
4513 } else if (n->is_Mem()) {
4514 Node* other_adr = n->in(MemNode::Address);
4515 if (other_adr == adr) {
4516 failed = true;
4517 break;
4518 } else {
4519 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4520 if (other_t_adr != nullptr) {
4521 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4522 if (other_alias_idx == alias_idx) {
4523 // A load from the same memory slice as the store right
4524 // after the InitializeNode. We check the control of the
4525 // object/array that is loaded from. If it's the same as
4526 // the store control then we cannot capture the store.
4527 assert(!n->is_Store(), "2 stores to same slice on same control?");
4528 Node* base = other_adr;
4529 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4530 base = base->in(AddPNode::Base);
4531 if (base != nullptr) {
4532 base = base->uncast();
4533 if (base->is_Proj() && base->in(0) == alloc) {
4534 failed = true;
4535 break;
4536 }
4537 }
4538 }
4539 }
4540 }
4541 } else {
4542 failed = true;
4543 break;
4544 }
4545 }
4546 }
4547 }
4548 if (failed) {
5095 // z's_done 12 16 16 16 12 16 12
5096 // z's_needed 12 16 16 16 16 16 16
5097 // zsize 0 0 0 0 4 0 4
5098 if (next_full_store < 0) {
5099 // Conservative tack: Zero to end of current word.
5100 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5101 } else {
5102 // Zero to beginning of next fully initialized word.
5103 // Or, don't zero at all, if we are already in that word.
5104 assert(next_full_store >= zeroes_needed, "must go forward");
5105 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5106 zeroes_needed = next_full_store;
5107 }
5108 }
5109
5110 if (zeroes_needed > zeroes_done) {
5111 intptr_t zsize = zeroes_needed - zeroes_done;
5112 // Do some incremental zeroing on rawmem, in parallel with inits.
5113 zeroes_done = align_down(zeroes_done, BytesPerInt);
5114 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5115 zeroes_done, zeroes_needed,
5116 phase);
5117 zeroes_done = zeroes_needed;
5118 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5119 do_zeroing = false; // leave the hole, next time
5120 }
5121 }
5122
5123 // Collect the store and move on:
5124 phase->replace_input_of(st, MemNode::Memory, inits);
5125 inits = st; // put it on the linearized chain
5126 set_req(i, zmem); // unhook from previous position
5127
5128 if (zeroes_done == st_off)
5129 zeroes_done = next_init_off;
5130
5131 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5132
5133 #ifdef ASSERT
5134 // Various order invariants. Weaker than stores_are_sane because
5154 remove_extra_zeroes(); // clear out all the zmems left over
5155 add_req(inits);
5156
5157 if (!(UseTLAB && ZeroTLAB)) {
5158 // If anything remains to be zeroed, zero it all now.
5159 zeroes_done = align_down(zeroes_done, BytesPerInt);
5160 // if it is the last unused 4 bytes of an instance, forget about it
5161 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5162 if (zeroes_done + BytesPerLong >= size_limit) {
5163 AllocateNode* alloc = allocation();
5164 assert(alloc != nullptr, "must be present");
5165 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5166 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5167 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5168 if (zeroes_done == k->layout_helper())
5169 zeroes_done = size_limit;
5170 }
5171 }
5172 if (zeroes_done < size_limit) {
5173 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5174 zeroes_done, size_in_bytes, phase);
5175 }
5176 }
5177
5178 set_complete(phase);
5179 return rawmem;
5180 }
5181
5182
5183 #ifdef ASSERT
5184 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5185 if (is_complete())
5186 return true; // stores could be anything at this point
5187 assert(allocation() != nullptr, "must be present");
5188 intptr_t last_off = allocation()->minimum_header_size();
5189 for (uint i = InitializeNode::RawStores; i < req(); i++) {
5190 Node* st = in(i);
5191 intptr_t st_off = get_store_offset(st, phase);
5192 if (st_off < 0) continue; // ignore dead garbage
5193 if (last_off > st_off) {
|
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "compiler/compileLog.hpp"
30 #include "gc/shared/barrierSet.hpp"
31 #include "gc/shared/c2/barrierSetC2.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "memory/allocation.inline.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "oops/objArrayKlass.hpp"
36 #include "opto/addnode.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/cfgnode.hpp"
39 #include "opto/regalloc.hpp"
40 #include "opto/compile.hpp"
41 #include "opto/connode.hpp"
42 #include "opto/convertnode.hpp"
43 #include "opto/inlinetypenode.hpp"
44 #include "opto/loopnode.hpp"
45 #include "opto/machnode.hpp"
46 #include "opto/matcher.hpp"
47 #include "opto/memnode.hpp"
48 #include "opto/mempointer.hpp"
49 #include "opto/mulnode.hpp"
50 #include "opto/narrowptrnode.hpp"
51 #include "opto/phaseX.hpp"
52 #include "opto/regmask.hpp"
53 #include "opto/rootnode.hpp"
54 #include "opto/traceMergeStoresTag.hpp"
55 #include "opto/vectornode.hpp"
56 #include "utilities/align.hpp"
57 #include "utilities/copy.hpp"
58 #include "utilities/macros.hpp"
59 #include "utilities/powerOfTwo.hpp"
60 #include "utilities/vmError.hpp"
61
62 // Portions of code courtesy of Clifford Click
63
219 bool is_instance = t_oop->is_known_instance_field();
220 PhaseIterGVN *igvn = phase->is_IterGVN();
221 if (is_instance && igvn != nullptr && result->is_Phi()) {
222 PhiNode *mphi = result->as_Phi();
223 assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
224 const TypePtr *t = mphi->adr_type();
225 bool do_split = false;
226 // In the following cases, Load memory input can be further optimized based on
227 // its precise address type
228 if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
229 do_split = true;
230 } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
231 const TypeOopPtr* mem_t =
232 t->is_oopptr()->cast_to_exactness(true)
233 ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
234 ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
235 if (t_oop->isa_aryptr()) {
236 mem_t = mem_t->is_aryptr()
237 ->cast_to_stable(t_oop->is_aryptr()->is_stable())
238 ->cast_to_size(t_oop->is_aryptr()->size())
239 ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
240 ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
241 ->with_offset(t_oop->is_aryptr()->offset())
242 ->is_aryptr();
243 }
244 do_split = mem_t == t_oop;
245 }
246 if (do_split) {
247 // clone the Phi with our address type
248 result = mphi->split_out_instance(t_adr, igvn);
249 } else {
250 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
251 }
252 }
253 return result;
254 }
255
256 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
257 uint alias_idx = phase->C->get_alias_index(tp);
258 Node *mem = mmem;
259 #ifdef ASSERT
260 {
261 // Check that current type is consistent with the alias index used during graph construction
262 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
263 bool consistent = adr_check == nullptr || adr_check->empty() ||
264 phase->C->must_alias(adr_check, alias_idx );
265 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
266 if( !consistent && adr_check != nullptr && !adr_check->empty() &&
267 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
268 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
269 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
270 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
271 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
272 // don't assert if it is dead code.
273 consistent = true;
274 }
275 if( !consistent ) {
276 st->print("alias_idx==%d, adr_check==", alias_idx);
277 if( adr_check == nullptr ) {
278 st->print("null");
279 } else {
280 adr_check->dump();
281 }
282 st->cr();
283 print_alias_types();
284 assert(consistent, "adr_check must match alias idx");
285 }
286 }
287 #endif
1000 Node* ld = gvn.transform(load);
1001 return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1002 }
1003
1004 return load;
1005 }
1006
1007 //------------------------------hash-------------------------------------------
1008 uint LoadNode::hash() const {
1009 // unroll addition of interesting fields
1010 return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1011 }
1012
1013 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1014 if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1015 bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1016 bool is_stable_ary = FoldStableValues &&
1017 (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1018 tp->isa_aryptr()->is_stable();
1019
1020 return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1021 }
1022
1023 return false;
1024 }
1025
1026 LoadNode* LoadNode::pin_array_access_node() const {
1027 const TypePtr* adr_type = this->adr_type();
1028 if (adr_type != nullptr && adr_type->isa_aryptr()) {
1029 return clone_pinned();
1030 }
1031 return nullptr;
1032 }
1033
1034 // Is the value loaded previously stored by an arraycopy? If so return
1035 // a load node that reads from the source array so we may be able to
1036 // optimize out the ArrayCopy node later.
1037 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1038 Node* ld_adr = in(MemNode::Address);
1039 intptr_t ld_off = 0;
1040 AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
1057 assert(ld_alloc != nullptr, "need an alloc");
1058 assert(addp->is_AddP(), "address must be addp");
1059 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1060 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1061 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1062 addp->set_req(AddPNode::Base, src);
1063 addp->set_req(AddPNode::Address, src);
1064 } else {
1065 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1066 ac->as_ArrayCopy()->is_copyof_validated() ||
1067 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1068 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1069 addp->set_req(AddPNode::Base, src);
1070 addp->set_req(AddPNode::Address, src);
1071
1072 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1073 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1074 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1075
1076 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1077 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1078
1079 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1080 #ifdef _LP64
1081 diff = phase->transform(new ConvI2LNode(diff));
1082 #endif
1083 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1084
1085 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1086 addp->set_req(AddPNode::Offset, offset);
1087 }
1088 addp = phase->transform(addp);
1089 #ifdef ASSERT
1090 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1091 ld->_adr_type = adr_type;
1092 #endif
1093 ld->set_req(MemNode::Address, addp);
1094 ld->set_req(0, ctl);
1095 ld->set_req(MemNode::Memory, mem);
1096 return ld;
1097 }
1176 // Same base, same offset.
1177 // Possible improvement for arrays: check index value instead of absolute offset.
1178
1179 // At this point we have proven something like this setup:
1180 // B = << base >>
1181 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1182 // S = StoreQ(AddP( B , #Off), V)
1183 // (Actually, we haven't yet proven the Q's are the same.)
1184 // In other words, we are loading from a casted version of
1185 // the same pointer-and-offset that we stored to.
1186 // Casted version may carry a dependency and it is respected.
1187 // Thus, we are able to replace L by V.
1188 }
1189 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1190 if (store_Opcode() != st->Opcode()) {
1191 return nullptr;
1192 }
1193 // LoadVector/StoreVector needs additional check to ensure the types match.
1194 if (st->is_StoreVector()) {
1195 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1196 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1197 if (in_vt != out_vt) {
1198 return nullptr;
1199 }
1200 }
1201 return st->in(MemNode::ValueIn);
1202 }
1203
1204 // A load from a freshly-created object always returns zero.
1205 // (This can happen after LoadNode::Ideal resets the load's memory input
1206 // to find_captured_store, which returned InitializeNode::zero_memory.)
1207 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1208 (st->in(0) == ld_alloc) &&
1209 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1210 // return a zero value for the load's basic type
1211 // (This is one of the few places where a generic PhaseTransform
1212 // can create new nodes. Think of it as lazily manifesting
1213 // virtually pre-existing constants.)
1214 Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1215 if (default_value != nullptr) {
1216 return default_value;
1217 }
1218 assert(ld_alloc->in(AllocateNode::RawDefaultValue) == nullptr, "default value may not be null");
1219 if (memory_type() != T_VOID) {
1220 if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1221 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1222 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1223 // by the ArrayCopyNode.
1224 return phase->zerocon(memory_type());
1225 }
1226 } else {
1227 // TODO: materialize all-zero vector constant
1228 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1229 }
1230 }
1231
1232 // A load from an initialization barrier can match a captured store.
1233 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1234 InitializeNode* init = st->in(0)->as_Initialize();
1235 AllocateNode* alloc = init->allocation();
1236 if ((alloc != nullptr) && (alloc == ld_alloc)) {
1237 // examine a captured store value
1238 st = init->find_captured_store(ld_off, memory_size(), phase);
1266 //----------------------is_instance_field_load_with_local_phi------------------
1267 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1268 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1269 in(Address)->is_AddP() ) {
1270 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1271 // Only instances and boxed values.
1272 if( t_oop != nullptr &&
1273 (t_oop->is_ptr_to_boxed_value() ||
1274 t_oop->is_known_instance_field()) &&
1275 t_oop->offset() != Type::OffsetBot &&
1276 t_oop->offset() != Type::OffsetTop) {
1277 return true;
1278 }
1279 }
1280 return false;
1281 }
1282
1283 //------------------------------Identity---------------------------------------
1284 // Loads are identity if previous store is to same address
1285 Node* LoadNode::Identity(PhaseGVN* phase) {
1286 // Loading from an InlineType? The InlineType has the values of
1287 // all fields as input. Look for the field with matching offset.
1288 Node* addr = in(Address);
1289 intptr_t offset;
1290 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1291 if (!is_mismatched_access() && base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1292 Node* value = base->as_InlineType()->field_value_by_offset((int)offset, true);
1293 if (value != nullptr) {
1294 if (Opcode() == Op_LoadN) {
1295 // Encode oop value if we are loading a narrow oop
1296 assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1297 value = phase->transform(new EncodePNode(value, bottom_type()));
1298 }
1299 return value;
1300 }
1301 }
1302
1303 // If the previous store-maker is the right kind of Store, and the store is
1304 // to the same address, then we are equal to the value stored.
1305 Node* mem = in(Memory);
1306 Node* value = can_see_stored_value(mem, phase);
1307 if( value ) {
1308 // byte, short & char stores truncate naturally.
1309 // A load has to load the truncated value which requires
1310 // some sort of masking operation and that requires an
1311 // Ideal call instead of an Identity call.
1312 if (memory_size() < BytesPerInt) {
1313 // If the input to the store does not fit with the load's result type,
1314 // it must be truncated via an Ideal call.
1315 if (!phase->type(value)->higher_equal(phase->type(this)))
1316 return this;
1317 }
1318 // (This works even when value is a Con, but LoadNode::Value
1319 // usually runs first, producing the singleton type of the Con.)
1320 if (!has_pinned_control_dependency() || value->is_Con()) {
1321 return value;
1322 } else {
1876 bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1877 phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1878
1879 // Skip up past a SafePoint control. Cannot do this for Stores because
1880 // pointer stores & cardmarks must stay on the same side of a SafePoint.
1881 if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1882 phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw &&
1883 !addr_mark &&
1884 (depends_only_on_test() || has_unknown_control_dependency())) {
1885 ctrl = ctrl->in(0);
1886 set_req(MemNode::Control,ctrl);
1887 progress = true;
1888 }
1889
1890 intptr_t ignore = 0;
1891 Node* base = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1892 if (base != nullptr
1893 && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1894 // Check for useless control edge in some common special cases
1895 if (in(MemNode::Control) != nullptr
1896 && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
1897 && can_remove_control()
1898 && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1899 && all_controls_dominate(base, phase->C->start())) {
1900 // A method-invariant, non-null address (constant or 'this' argument).
1901 set_req(MemNode::Control, nullptr);
1902 progress = true;
1903 }
1904 }
1905
1906 Node* mem = in(MemNode::Memory);
1907 const TypePtr *addr_t = phase->type(address)->isa_ptr();
1908
1909 if (can_reshape && (addr_t != nullptr)) {
1910 // try to optimize our memory input
1911 Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1912 if (opt_mem != mem) {
1913 set_req_X(MemNode::Memory, opt_mem, phase);
1914 if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1915 return this;
1916 }
2081 }
2082 }
2083
2084 // Don't do this for integer types. There is only potential profit if
2085 // the element type t is lower than _type; that is, for int types, if _type is
2086 // more restrictive than t. This only happens here if one is short and the other
2087 // char (both 16 bits), and in those cases we've made an intentional decision
2088 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2089 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2090 //
2091 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2092 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
2093 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2094 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
2095 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2096 // In fact, that could have been the original type of p1, and p1 could have
2097 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2098 // expression (LShiftL quux 3) independently optimized to the constant 8.
2099 if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2100 && (_type->isa_vect() == nullptr)
2101 && !ary->is_flat()
2102 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2103 // t might actually be lower than _type, if _type is a unique
2104 // concrete subclass of abstract class t.
2105 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
2106 const Type* jt = t->join_speculative(_type);
2107 // In any case, do not allow the join, per se, to empty out the type.
2108 if (jt->empty() && !t->empty()) {
2109 // This can happen if a interface-typed array narrows to a class type.
2110 jt = _type;
2111 }
2112 #ifdef ASSERT
2113 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2114 // The pointers in the autobox arrays are always non-null
2115 Node* base = adr->in(AddPNode::Base);
2116 if ((base != nullptr) && base->is_DecodeN()) {
2117 // Get LoadN node which loads IntegerCache.cache field
2118 base = base->in(1);
2119 }
2120 if ((base != nullptr) && base->is_Con()) {
2121 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2122 if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2123 // It could be narrow oop
2124 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2125 }
2126 }
2127 }
2128 #endif
2129 return jt;
2130 }
2131 }
2132 } else if (tp->base() == Type::InstPtr) {
2133 assert( off != Type::OffsetBot ||
2134 // arrays can be cast to Objects
2135 !tp->isa_instptr() ||
2136 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2137 // Default value load
2138 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2139 // unsafe field access may not have a constant offset
2140 C->has_unsafe_access(),
2141 "Field accesses must be precise" );
2142 // For oop loads, we expect the _type to be precise.
2143
2144 const TypeInstPtr* tinst = tp->is_instptr();
2145 BasicType bt = memory_type();
2146
2147 // Optimize loads from constant fields.
2148 ciObject* const_oop = tinst->const_oop();
2149 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2150 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2151 if (con_type != nullptr) {
2152 return con_type;
2153 }
2154 }
2155 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2156 assert(off != Type::OffsetBot ||
2157 !tp->isa_instklassptr() ||
2158 // arrays can be cast to Objects
2159 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2160 // also allow array-loading from the primary supertype
2161 // array during subtype checks
2162 Opcode() == Op_LoadKlass,
2163 "Field accesses must be precise");
2164 // For klass/static loads, we expect the _type to be precise
2165 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2166 /* With mirrors being an indirect in the Klass*
2167 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2168 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2169 *
2170 * So check the type and klass of the node before the LoadP.
2177 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2178 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2179 return TypeInstPtr::make(klass->java_mirror());
2180 }
2181 }
2182 }
2183
2184 const TypeKlassPtr *tkls = tp->isa_klassptr();
2185 if (tkls != nullptr) {
2186 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2187 ciKlass* klass = tkls->exact_klass();
2188 // We are loading a field from a Klass metaobject whose identity
2189 // is known at compile time (the type is "exact" or "precise").
2190 // Check for fields we know are maintained as constants by the VM.
2191 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2192 // The field is Klass::_super_check_offset. Return its (constant) value.
2193 // (Folds up type checking code.)
2194 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2195 return TypeInt::make(klass->super_check_offset());
2196 }
2197 if (UseCompactObjectHeaders) { // TODO: Should EnableValhalla also take this path ?
2198 if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2199 // The field is Klass::_prototype_header. Return its (constant) value.
2200 assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2201 return TypeX::make(klass->prototype_header());
2202 }
2203 }
2204 // Compute index into primary_supers array
2205 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2206 // Check for overflowing; use unsigned compare to handle the negative case.
2207 if( depth < ciKlass::primary_super_limit() ) {
2208 // The field is an element of Klass::_primary_supers. Return its (constant) value.
2209 // (Folds up type checking code.)
2210 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2211 ciKlass *ss = klass->super_of_depth(depth);
2212 return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2213 }
2214 const Type* aift = load_array_final_field(tkls, klass);
2215 if (aift != nullptr) return aift;
2216 }
2217
2258 jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2259 // The key property of this type is that it folds up tests
2260 // for array-ness, since it proves that the layout_helper is positive.
2261 // Thus, a generic value like the basic object layout helper works fine.
2262 return TypeInt::make(min_size, max_jint, Type::WidenMin);
2263 }
2264 }
2265
2266 bool is_vect = (_type->isa_vect() != nullptr);
2267 if (is_instance && !is_vect) {
2268 // If we have an instance type and our memory input is the
2269 // programs's initial memory state, there is no matching store,
2270 // so just return a zero of the appropriate type -
2271 // except if it is vectorized - then we have no zero constant.
2272 Node *mem = in(MemNode::Memory);
2273 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2274 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2275 return Type::get_zero_type(_type->basic_type());
2276 }
2277 }
2278 if (!UseCompactObjectHeaders) {
2279 Node* alloc = is_new_object_mark_load();
2280 if (alloc != nullptr) {
2281 if (EnableValhalla) {
2282 // The mark word may contain property bits (inline, flat, null-free)
2283 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2284 const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2285 if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2286 return TypeX::make(tkls->exact_klass()->prototype_header());
2287 }
2288 } else {
2289 return TypeX::make(markWord::prototype().value());
2290 }
2291 }
2292 }
2293
2294 return _type;
2295 }
2296
2297 //------------------------------match_edge-------------------------------------
2298 // Do we Match on this edge index or not? Match only the address.
2299 uint LoadNode::match_edge(uint idx) const {
2300 return idx == MemNode::Address;
2301 }
2302
2303 //--------------------------LoadBNode::Ideal--------------------------------------
2304 //
2305 // If the previous store is to the same address as this load,
2306 // and the value stored was larger than a byte, replace this load
2307 // with the value stored truncated to a byte. If no truncation is
2308 // needed, the replacement is done in LoadNode::Identity().
2309 //
2310 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2422 return LoadNode::Ideal(phase, can_reshape);
2423 }
2424
2425 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2426 Node* mem = in(MemNode::Memory);
2427 Node* value = can_see_stored_value(mem,phase);
2428 if (value != nullptr && value->is_Con() &&
2429 !value->bottom_type()->higher_equal(_type)) {
2430 // If the input to the store does not fit with the load's result type,
2431 // it must be truncated. We can't delay until Ideal call since
2432 // a singleton Value is needed for split_thru_phi optimization.
2433 int con = value->get_int();
2434 return TypeInt::make((con << 16) >> 16);
2435 }
2436 return LoadNode::Value(phase);
2437 }
2438
2439 //=============================================================================
2440 //----------------------------LoadKlassNode::make------------------------------
2441 // Polymorphic factory method:
2442 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2443 const TypeKlassPtr* tk) {
2444 // sanity check the alias category against the created node type
2445 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2446 assert(adr_type != nullptr, "expecting TypeKlassPtr");
2447 #ifdef _LP64
2448 if (adr_type->is_ptr_to_narrowklass()) {
2449 assert(UseCompressedClassPointers, "no compressed klasses");
2450 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2451 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2452 }
2453 #endif
2454 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2455 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2456 }
2457
2458 //------------------------------Value------------------------------------------
2459 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2460 return klass_value_common(phase);
2461 }
2462
2463 // In most cases, LoadKlassNode does not have the control input set. If the control
2470 // Either input is TOP ==> the result is TOP
2471 const Type *t1 = phase->type( in(MemNode::Memory) );
2472 if (t1 == Type::TOP) return Type::TOP;
2473 Node *adr = in(MemNode::Address);
2474 const Type *t2 = phase->type( adr );
2475 if (t2 == Type::TOP) return Type::TOP;
2476 const TypePtr *tp = t2->is_ptr();
2477 if (TypePtr::above_centerline(tp->ptr()) ||
2478 tp->ptr() == TypePtr::Null) return Type::TOP;
2479
2480 // Return a more precise klass, if possible
2481 const TypeInstPtr *tinst = tp->isa_instptr();
2482 if (tinst != nullptr) {
2483 ciInstanceKlass* ik = tinst->instance_klass();
2484 int offset = tinst->offset();
2485 if (ik == phase->C->env()->Class_klass()
2486 && (offset == java_lang_Class::klass_offset() ||
2487 offset == java_lang_Class::array_klass_offset())) {
2488 // We are loading a special hidden field from a Class mirror object,
2489 // the field which points to the VM's Klass metaobject.
2490 bool is_null_free_array = false;
2491 ciType* t = tinst->java_mirror_type(&is_null_free_array);
2492 // java_mirror_type returns non-null for compile-time Class constants.
2493 if (t != nullptr) {
2494 // constant oop => constant klass
2495 if (offset == java_lang_Class::array_klass_offset()) {
2496 if (t->is_void()) {
2497 // We cannot create a void array. Since void is a primitive type return null
2498 // klass. Users of this result need to do a null check on the returned klass.
2499 return TypePtr::NULL_PTR;
2500 }
2501 const TypeKlassPtr* tklass = TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2502 if (is_null_free_array) {
2503 tklass = tklass->is_aryklassptr()->cast_to_null_free();
2504 }
2505 return tklass;
2506 }
2507 if (!t->is_klass()) {
2508 // a primitive Class (e.g., int.class) has null for a klass field
2509 return TypePtr::NULL_PTR;
2510 }
2511 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2512 const TypeKlassPtr* tklass = TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2513 if (is_null_free_array) {
2514 tklass = tklass->is_aryklassptr()->cast_to_null_free();
2515 }
2516 return tklass;
2517 }
2518 // non-constant mirror, so we can't tell what's going on
2519 }
2520 if (!tinst->is_loaded())
2521 return _type; // Bail out if not loaded
2522 if (offset == oopDesc::klass_offset_in_bytes()) {
2523 return tinst->as_klass_type(true);
2524 }
2525 }
2526
2527 // Check for loading klass from an array
2528 const TypeAryPtr* tary = tp->isa_aryptr();
2529 if (tary != nullptr &&
2530 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2531 return tary->as_klass_type(true);
2532 }
2533
2534 // Check for loading klass from an array klass
2535 const TypeKlassPtr *tkls = tp->isa_klassptr();
2536 if (tkls != nullptr && !StressReflectiveCode) {
2537 if (!tkls->is_loaded())
2538 return _type; // Bail out if not loaded
2539 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2540 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2541 // // Always returning precise element type is incorrect,
2542 // // e.g., element type could be object and array may contain strings
2543 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2544
2545 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2546 // according to the element type's subclassing.
2547 return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2548 }
3345 }
3346 ss.print_cr("[TraceMergeStores]: with");
3347 merged_input_value->dump("\n", false, &ss);
3348 merged_store->dump("\n", false, &ss);
3349 tty->print("%s", ss.as_string());
3350 }
3351 #endif
3352
3353 //------------------------------Ideal------------------------------------------
3354 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3355 // When a store immediately follows a relevant allocation/initialization,
3356 // try to capture it into the initialization, or hoist it above.
3357 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3358 Node* p = MemNode::Ideal_common(phase, can_reshape);
3359 if (p) return (p == NodeSentinel) ? nullptr : p;
3360
3361 Node* mem = in(MemNode::Memory);
3362 Node* address = in(MemNode::Address);
3363 Node* value = in(MemNode::ValueIn);
3364 // Back-to-back stores to same address? Fold em up. Generally
3365 // unsafe if I have intervening uses...
3366 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3367 Node* st = mem;
3368 // If Store 'st' has more than one use, we cannot fold 'st' away.
3369 // For example, 'st' might be the final state at a conditional
3370 // return. Or, 'st' might be used by some node which is live at
3371 // the same time 'st' is live, which might be unschedulable. So,
3372 // require exactly ONE user until such time as we clone 'mem' for
3373 // each of 'mem's uses (thus making the exactly-1-user-rule hold
3374 // true).
3375 while (st->is_Store() && st->outcnt() == 1) {
3376 // Looking at a dead closed cycle of memory?
3377 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3378 assert(Opcode() == st->Opcode() ||
3379 st->Opcode() == Op_StoreVector ||
3380 Opcode() == Op_StoreVector ||
3381 st->Opcode() == Op_StoreVectorScatter ||
3382 Opcode() == Op_StoreVectorScatter ||
3383 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3384 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3385 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3386 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3387 (st->adr_type()->isa_aryptr() && st->adr_type()->is_aryptr()->is_flat()) || // TODO 8343835
3388 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3389 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3390
3391 if (st->in(MemNode::Address)->eqv_uncast(address) &&
3392 st->as_Store()->memory_size() <= this->memory_size()) {
3393 Node* use = st->raw_out(0);
3394 if (phase->is_IterGVN()) {
3395 phase->is_IterGVN()->rehash_node_delayed(use);
3396 }
3397 // It's OK to do this in the parser, since DU info is always accurate,
3398 // and the parser always refers to nodes via SafePointNode maps.
3399 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3400 return this;
3401 }
3402 st = st->in(MemNode::Memory);
3403 }
3404 }
3405
3406
3407 // Capture an unaliased, unconditional, simple store into an initializer.
3494 const StoreVectorNode* store_vector = as_StoreVector();
3495 const StoreVectorNode* mem_vector = mem->as_StoreVector();
3496 const Node* store_indices = store_vector->indices();
3497 const Node* mem_indices = mem_vector->indices();
3498 const Node* store_mask = store_vector->mask();
3499 const Node* mem_mask = mem_vector->mask();
3500 // Ensure types, indices, and masks match
3501 if (store_vector->vect_type() == mem_vector->vect_type() &&
3502 ((store_indices == nullptr) == (mem_indices == nullptr) &&
3503 (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3504 ((store_mask == nullptr) == (mem_mask == nullptr) &&
3505 (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3506 result = mem;
3507 }
3508 }
3509 }
3510
3511 // Store of zero anywhere into a freshly-allocated object?
3512 // Then the store is useless.
3513 // (It must already have been captured by the InitializeNode.)
3514 if (result == this && ReduceFieldZeroing) {
3515 // a newly allocated object is already all-zeroes everywhere
3516 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3517 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
3518 result = mem;
3519 }
3520
3521 if (result == this && phase->type(val)->is_zero_type()) {
3522 // the store may also apply to zero-bits in an earlier object
3523 Node* prev_mem = find_previous_store(phase);
3524 // Steps (a), (b): Walk past independent stores to find an exact match.
3525 if (prev_mem != nullptr) {
3526 Node* prev_val = can_see_stored_value(prev_mem, phase);
3527 if (prev_val != nullptr && prev_val == val) {
3528 // prev_val and val might differ by a cast; it would be good
3529 // to keep the more informative of the two.
3530 result = mem;
3531 }
3532 }
3533 }
3534 }
3535
3536 PhaseIterGVN* igvn = phase->is_IterGVN();
3537 if (result != this && igvn != nullptr) {
3538 MemBarNode* trailing = trailing_membar();
3539 if (trailing != nullptr) {
3540 #ifdef ASSERT
3541 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
3817 // Clearing a short array is faster with stores
3818 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3819 // Already know this is a large node, do not try to ideal it
3820 if (_is_large) return nullptr;
3821
3822 const int unit = BytesPerLong;
3823 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3824 if (!t) return nullptr;
3825 if (!t->is_con()) return nullptr;
3826 intptr_t raw_count = t->get_con();
3827 intptr_t size = raw_count;
3828 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3829 // Clearing nothing uses the Identity call.
3830 // Negative clears are possible on dead ClearArrays
3831 // (see jck test stmt114.stmt11402.val).
3832 if (size <= 0 || size % unit != 0) return nullptr;
3833 intptr_t count = size / unit;
3834 // Length too long; communicate this to matchers and assemblers.
3835 // Assemblers are responsible to produce fast hardware clears for it.
3836 if (size > InitArrayShortSize) {
3837 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3838 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3839 return nullptr;
3840 }
3841 if (!IdealizeClearArrayNode) return nullptr;
3842 Node *mem = in(1);
3843 if( phase->type(mem)==Type::TOP ) return nullptr;
3844 Node *adr = in(3);
3845 const Type* at = phase->type(adr);
3846 if( at==Type::TOP ) return nullptr;
3847 const TypePtr* atp = at->isa_ptr();
3848 // adjust atp to be the correct array element address type
3849 if (atp == nullptr) atp = TypePtr::BOTTOM;
3850 else atp = atp->add_offset(Type::OffsetBot);
3851 // Get base for derived pointer purposes
3852 if( adr->Opcode() != Op_AddP ) Unimplemented();
3853 Node *base = adr->in(1);
3854
3855 Node *val = in(4);
3856 Node *off = phase->MakeConX(BytesPerLong);
3857 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3858 count--;
3859 while( count-- ) {
3860 mem = phase->transform(mem);
3861 adr = phase->transform(new AddPNode(base,adr,off));
3862 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3863 }
3864 return mem;
3865 }
3866
3867 //----------------------------step_through----------------------------------
3868 // Return allocation input memory edge if it is different instance
3869 // or itself if it is the one we are looking for.
3870 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
3871 Node* n = *np;
3872 assert(n->is_ClearArray(), "sanity");
3873 intptr_t offset;
3874 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3875 // This method is called only before Allocate nodes are expanded
3876 // during macro nodes expansion. Before that ClearArray nodes are
3877 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3878 // Allocate nodes are expanded) which follows allocations.
3879 assert(alloc != nullptr, "should have allocation");
3880 if (alloc->_idx == instance_id) {
3881 // Can not bypass initialization of the instance we are looking for.
3882 return false;
3883 }
3884 // Otherwise skip it.
3885 InitializeNode* init = alloc->initialization();
3886 if (init != nullptr)
3887 *np = init->in(TypeFunc::Memory);
3888 else
3889 *np = alloc->in(TypeFunc::Memory);
3890 return true;
3891 }
3892
3893 //----------------------------clear_memory-------------------------------------
3894 // Generate code to initialize object storage to zero.
3895 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3896 Node* val,
3897 Node* raw_val,
3898 intptr_t start_offset,
3899 Node* end_offset,
3900 PhaseGVN* phase) {
3901 intptr_t offset = start_offset;
3902
3903 int unit = BytesPerLong;
3904 if ((offset % unit) != 0) {
3905 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3906 adr = phase->transform(adr);
3907 const TypePtr* atp = TypeRawPtr::BOTTOM;
3908 if (val != nullptr) {
3909 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3910 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3911 } else {
3912 assert(raw_val == nullptr, "val may not be null");
3913 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3914 }
3915 mem = phase->transform(mem);
3916 offset += BytesPerInt;
3917 }
3918 assert((offset % unit) == 0, "");
3919
3920 // Initialize the remaining stuff, if any, with a ClearArray.
3921 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3922 }
3923
3924 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3925 Node* raw_val,
3926 Node* start_offset,
3927 Node* end_offset,
3928 PhaseGVN* phase) {
3929 if (start_offset == end_offset) {
3930 // nothing to do
3931 return mem;
3932 }
3933
3934 int unit = BytesPerLong;
3935 Node* zbase = start_offset;
3936 Node* zend = end_offset;
3937
3938 // Scale to the unit required by the CPU:
3939 if (!Matcher::init_array_count_is_in_bytes) {
3940 Node* shift = phase->intcon(exact_log2(unit));
3941 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3942 zend = phase->transform(new URShiftXNode(zend, shift) );
3943 }
3944
3945 // Bulk clear double-words
3946 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3947 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3948 if (raw_val == nullptr) {
3949 raw_val = phase->MakeConX(0);
3950 }
3951 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3952 return phase->transform(mem);
3953 }
3954
3955 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3956 Node* val,
3957 Node* raw_val,
3958 intptr_t start_offset,
3959 intptr_t end_offset,
3960 PhaseGVN* phase) {
3961 if (start_offset == end_offset) {
3962 // nothing to do
3963 return mem;
3964 }
3965
3966 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3967 intptr_t done_offset = end_offset;
3968 if ((done_offset % BytesPerLong) != 0) {
3969 done_offset -= BytesPerInt;
3970 }
3971 if (done_offset > start_offset) {
3972 mem = clear_memory(ctl, mem, dest, val, raw_val,
3973 start_offset, phase->MakeConX(done_offset), phase);
3974 }
3975 if (done_offset < end_offset) { // emit the final 32-bit store
3976 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3977 adr = phase->transform(adr);
3978 const TypePtr* atp = TypeRawPtr::BOTTOM;
3979 if (val != nullptr) {
3980 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3981 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3982 } else {
3983 assert(raw_val == nullptr, "val may not be null");
3984 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3985 }
3986 mem = phase->transform(mem);
3987 done_offset += BytesPerInt;
3988 }
3989 assert(done_offset == end_offset, "");
3990 return mem;
3991 }
3992
3993 //=============================================================================
3994 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3995 : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
3996 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3997 #ifdef ASSERT
3998 , _pair_idx(0)
3999 #endif
4000 {
4001 init_class_id(Class_MemBar);
4002 Node* top = C->top();
4003 init_req(TypeFunc::I_O,top);
4004 init_req(TypeFunc::FramePtr,top);
4005 init_req(TypeFunc::ReturnAdr,top);
4111 PhaseIterGVN* igvn = phase->is_IterGVN();
4112 remove(igvn);
4113 // Must return either the original node (now dead) or a new node
4114 // (Do not return a top here, since that would break the uniqueness of top.)
4115 return new ConINode(TypeInt::ZERO);
4116 }
4117 }
4118 return progress ? this : nullptr;
4119 }
4120
4121 //------------------------------Value------------------------------------------
4122 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4123 if( !in(0) ) return Type::TOP;
4124 if( phase->type(in(0)) == Type::TOP )
4125 return Type::TOP;
4126 return TypeTuple::MEMBAR;
4127 }
4128
4129 //------------------------------match------------------------------------------
4130 // Construct projections for memory.
4131 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4132 switch (proj->_con) {
4133 case TypeFunc::Control:
4134 case TypeFunc::Memory:
4135 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4136 }
4137 ShouldNotReachHere();
4138 return nullptr;
4139 }
4140
4141 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4142 trailing->_kind = TrailingStore;
4143 leading->_kind = LeadingStore;
4144 #ifdef ASSERT
4145 trailing->_pair_idx = leading->_idx;
4146 leading->_pair_idx = leading->_idx;
4147 #endif
4148 }
4149
4150 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4151 trailing->_kind = TrailingLoadStore;
4398 return (req() > RawStores);
4399 }
4400
4401 void InitializeNode::set_complete(PhaseGVN* phase) {
4402 assert(!is_complete(), "caller responsibility");
4403 _is_complete = Complete;
4404
4405 // After this node is complete, it contains a bunch of
4406 // raw-memory initializations. There is no need for
4407 // it to have anything to do with non-raw memory effects.
4408 // Therefore, tell all non-raw users to re-optimize themselves,
4409 // after skipping the memory effects of this initialization.
4410 PhaseIterGVN* igvn = phase->is_IterGVN();
4411 if (igvn) igvn->add_users_to_worklist(this);
4412 }
4413
4414 // convenience function
4415 // return false if the init contains any stores already
4416 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4417 InitializeNode* init = initialization();
4418 if (init == nullptr || init->is_complete()) {
4419 return false;
4420 }
4421 init->remove_extra_zeroes();
4422 // for now, if this allocation has already collected any inits, bail:
4423 if (init->is_non_zero()) return false;
4424 init->set_complete(phase);
4425 return true;
4426 }
4427
4428 void InitializeNode::remove_extra_zeroes() {
4429 if (req() == RawStores) return;
4430 Node* zmem = zero_memory();
4431 uint fill = RawStores;
4432 for (uint i = fill; i < req(); i++) {
4433 Node* n = in(i);
4434 if (n->is_top() || n == zmem) continue; // skip
4435 if (fill < i) set_req(fill, n); // compact
4436 ++fill;
4437 }
4438 // delete any empty spaces created:
4439 while (fill < req()) {
4440 del_req(fill);
4584 // store node that we'd like to capture. We need to check
4585 // the uses of the MergeMemNode.
4586 mems.push(n);
4587 }
4588 } else if (n->is_Mem()) {
4589 Node* other_adr = n->in(MemNode::Address);
4590 if (other_adr == adr) {
4591 failed = true;
4592 break;
4593 } else {
4594 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4595 if (other_t_adr != nullptr) {
4596 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4597 if (other_alias_idx == alias_idx) {
4598 // A load from the same memory slice as the store right
4599 // after the InitializeNode. We check the control of the
4600 // object/array that is loaded from. If it's the same as
4601 // the store control then we cannot capture the store.
4602 assert(!n->is_Store(), "2 stores to same slice on same control?");
4603 Node* base = other_adr;
4604 if (base->is_Phi()) {
4605 // In rare case, base may be a PhiNode and it may read
4606 // the same memory slice between InitializeNode and store.
4607 failed = true;
4608 break;
4609 }
4610 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4611 base = base->in(AddPNode::Base);
4612 if (base != nullptr) {
4613 base = base->uncast();
4614 if (base->is_Proj() && base->in(0) == alloc) {
4615 failed = true;
4616 break;
4617 }
4618 }
4619 }
4620 }
4621 }
4622 } else {
4623 failed = true;
4624 break;
4625 }
4626 }
4627 }
4628 }
4629 if (failed) {
5176 // z's_done 12 16 16 16 12 16 12
5177 // z's_needed 12 16 16 16 16 16 16
5178 // zsize 0 0 0 0 4 0 4
5179 if (next_full_store < 0) {
5180 // Conservative tack: Zero to end of current word.
5181 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5182 } else {
5183 // Zero to beginning of next fully initialized word.
5184 // Or, don't zero at all, if we are already in that word.
5185 assert(next_full_store >= zeroes_needed, "must go forward");
5186 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5187 zeroes_needed = next_full_store;
5188 }
5189 }
5190
5191 if (zeroes_needed > zeroes_done) {
5192 intptr_t zsize = zeroes_needed - zeroes_done;
5193 // Do some incremental zeroing on rawmem, in parallel with inits.
5194 zeroes_done = align_down(zeroes_done, BytesPerInt);
5195 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5196 allocation()->in(AllocateNode::DefaultValue),
5197 allocation()->in(AllocateNode::RawDefaultValue),
5198 zeroes_done, zeroes_needed,
5199 phase);
5200 zeroes_done = zeroes_needed;
5201 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5202 do_zeroing = false; // leave the hole, next time
5203 }
5204 }
5205
5206 // Collect the store and move on:
5207 phase->replace_input_of(st, MemNode::Memory, inits);
5208 inits = st; // put it on the linearized chain
5209 set_req(i, zmem); // unhook from previous position
5210
5211 if (zeroes_done == st_off)
5212 zeroes_done = next_init_off;
5213
5214 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5215
5216 #ifdef ASSERT
5217 // Various order invariants. Weaker than stores_are_sane because
5237 remove_extra_zeroes(); // clear out all the zmems left over
5238 add_req(inits);
5239
5240 if (!(UseTLAB && ZeroTLAB)) {
5241 // If anything remains to be zeroed, zero it all now.
5242 zeroes_done = align_down(zeroes_done, BytesPerInt);
5243 // if it is the last unused 4 bytes of an instance, forget about it
5244 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5245 if (zeroes_done + BytesPerLong >= size_limit) {
5246 AllocateNode* alloc = allocation();
5247 assert(alloc != nullptr, "must be present");
5248 if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5249 Node* klass_node = alloc->in(AllocateNode::KlassNode);
5250 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5251 if (zeroes_done == k->layout_helper())
5252 zeroes_done = size_limit;
5253 }
5254 }
5255 if (zeroes_done < size_limit) {
5256 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5257 allocation()->in(AllocateNode::DefaultValue),
5258 allocation()->in(AllocateNode::RawDefaultValue),
5259 zeroes_done, size_in_bytes, phase);
5260 }
5261 }
5262
5263 set_complete(phase);
5264 return rawmem;
5265 }
5266
5267
5268 #ifdef ASSERT
5269 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5270 if (is_complete())
5271 return true; // stores could be anything at this point
5272 assert(allocation() != nullptr, "must be present");
5273 intptr_t last_off = allocation()->minimum_header_size();
5274 for (uint i = InitializeNode::RawStores; i < req(); i++) {
5275 Node* st = in(i);
5276 intptr_t st_off = get_store_offset(st, phase);
5277 if (st_off < 0) continue; // ignore dead garbage
5278 if (last_off > st_off) {
|