6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "gc/shared/tlab_globals.hpp"
31 #include "memory/allocation.inline.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "oops/objArrayKlass.hpp"
34 #include "opto/addnode.hpp"
35 #include "opto/arraycopynode.hpp"
36 #include "opto/cfgnode.hpp"
37 #include "opto/regalloc.hpp"
38 #include "opto/compile.hpp"
39 #include "opto/connode.hpp"
40 #include "opto/convertnode.hpp"
41 #include "opto/loopnode.hpp"
42 #include "opto/machnode.hpp"
43 #include "opto/matcher.hpp"
44 #include "opto/memnode.hpp"
45 #include "opto/mulnode.hpp"
46 #include "opto/narrowptrnode.hpp"
47 #include "opto/phaseX.hpp"
48 #include "opto/regmask.hpp"
49 #include "opto/rootnode.hpp"
50 #include "opto/vectornode.hpp"
51 #include "utilities/align.hpp"
52 #include "utilities/copy.hpp"
53 #include "utilities/macros.hpp"
54 #include "utilities/powerOfTwo.hpp"
55 #include "utilities/vmError.hpp"
56
57 // Portions of code courtesy of Clifford Click
58
59 // Optimization - Graph Style
60
223 // clone the Phi with our address type
224 result = mphi->split_out_instance(t_adr, igvn);
225 } else {
226 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
227 }
228 }
229 return result;
230 }
231
232 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
233 uint alias_idx = phase->C->get_alias_index(tp);
234 Node *mem = mmem;
235 #ifdef ASSERT
236 {
237 // Check that current type is consistent with the alias index used during graph construction
238 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
239 bool consistent = adr_check == NULL || adr_check->empty() ||
240 phase->C->must_alias(adr_check, alias_idx );
241 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
242 if( !consistent && adr_check != NULL && !adr_check->empty() &&
243 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
244 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
245 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
246 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
247 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
248 // don't assert if it is dead code.
249 consistent = true;
250 }
251 if( !consistent ) {
252 st->print("alias_idx==%d, adr_check==", alias_idx);
253 if( adr_check == NULL ) {
254 st->print("NULL");
255 } else {
256 adr_check->dump();
257 }
258 st->cr();
259 print_alias_types();
260 assert(consistent, "adr_check must match alias idx");
261 }
262 }
263 #endif
881 "use LoadKlassNode instead");
882 assert(!(adr_type->isa_aryptr() &&
883 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
884 "use LoadRangeNode instead");
885 // Check control edge of raw loads
886 assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
887 // oop will be recorded in oop map if load crosses safepoint
888 rt->isa_oopptr() || is_immutable_value(adr),
889 "raw memory operations should have control edge");
890 LoadNode* load = NULL;
891 switch (bt) {
892 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
893 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
894 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
895 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
896 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
897 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
898 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
899 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
900 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
901 case T_OBJECT:
902 #ifdef _LP64
903 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
904 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
905 } else
906 #endif
907 {
908 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
909 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
910 }
911 break;
912 default:
913 ShouldNotReachHere();
914 break;
915 }
916 assert(load != NULL, "LoadNode should have been created");
917 if (unaligned) {
918 load->set_unaligned_access();
919 }
920 if (mismatched) {
976 assert(ld_alloc != NULL, "need an alloc");
977 assert(addp->is_AddP(), "address must be addp");
978 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
979 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
980 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
981 addp->set_req(AddPNode::Base, src);
982 addp->set_req(AddPNode::Address, src);
983 } else {
984 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
985 ac->as_ArrayCopy()->is_copyof_validated() ||
986 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
987 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
988 addp->set_req(AddPNode::Base, src);
989 addp->set_req(AddPNode::Address, src);
990
991 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
992 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
993 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
994
995 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
996 uint shift = exact_log2(type2aelembytes(ary_elem));
997
998 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
999 #ifdef _LP64
1000 diff = phase->transform(new ConvI2LNode(diff));
1001 #endif
1002 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1003
1004 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1005 addp->set_req(AddPNode::Offset, offset);
1006 }
1007 addp = phase->transform(addp);
1008 #ifdef ASSERT
1009 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1010 ld->_adr_type = adr_type;
1011 #endif
1012 ld->set_req(MemNode::Address, addp);
1013 ld->set_req(0, ctl);
1014 ld->set_req(MemNode::Memory, mem);
1015 // load depends on the tests that validate the arraycopy
1016 ld->_control_dependency = UnknownControl;
1097 // Same base, same offset.
1098 // Possible improvement for arrays: check index value instead of absolute offset.
1099
1100 // At this point we have proven something like this setup:
1101 // B = << base >>
1102 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1103 // S = StoreQ(AddP( B , #Off), V)
1104 // (Actually, we haven't yet proven the Q's are the same.)
1105 // In other words, we are loading from a casted version of
1106 // the same pointer-and-offset that we stored to.
1107 // Casted version may carry a dependency and it is respected.
1108 // Thus, we are able to replace L by V.
1109 }
1110 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1111 if (store_Opcode() != st->Opcode()) {
1112 return NULL;
1113 }
1114 // LoadVector/StoreVector needs additional check to ensure the types match.
1115 if (st->is_StoreVector()) {
1116 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1117 const TypeVect* out_vt = as_LoadVector()->vect_type();
1118 if (in_vt != out_vt) {
1119 return NULL;
1120 }
1121 }
1122 return st->in(MemNode::ValueIn);
1123 }
1124
1125 // A load from a freshly-created object always returns zero.
1126 // (This can happen after LoadNode::Ideal resets the load's memory input
1127 // to find_captured_store, which returned InitializeNode::zero_memory.)
1128 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1129 (st->in(0) == ld_alloc) &&
1130 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1131 // return a zero value for the load's basic type
1132 // (This is one of the few places where a generic PhaseTransform
1133 // can create new nodes. Think of it as lazily manifesting
1134 // virtually pre-existing constants.)
1135 if (memory_type() != T_VOID) {
1136 if (ReduceBulkZeroing || find_array_copy_clone(phase, ld_alloc, in(MemNode::Memory)) == NULL) {
1137 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1138 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1139 // by the ArrayCopyNode.
1140 return phase->zerocon(memory_type());
1141 }
1142 } else {
1143 // TODO: materialize all-zero vector constant
1144 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1145 }
1146 }
1147
1148 // A load from an initialization barrier can match a captured store.
1149 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1150 InitializeNode* init = st->in(0)->as_Initialize();
1151 AllocateNode* alloc = init->allocation();
1152 if ((alloc != NULL) && (alloc == ld_alloc)) {
1153 // examine a captured store value
1154 st = init->find_captured_store(ld_off, memory_size(), phase);
1182 //----------------------is_instance_field_load_with_local_phi------------------
1183 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1184 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1185 in(Address)->is_AddP() ) {
1186 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1187 // Only instances and boxed values.
1188 if( t_oop != NULL &&
1189 (t_oop->is_ptr_to_boxed_value() ||
1190 t_oop->is_known_instance_field()) &&
1191 t_oop->offset() != Type::OffsetBot &&
1192 t_oop->offset() != Type::OffsetTop) {
1193 return true;
1194 }
1195 }
1196 return false;
1197 }
1198
1199 //------------------------------Identity---------------------------------------
1200 // Loads are identity if previous store is to same address
1201 Node* LoadNode::Identity(PhaseGVN* phase) {
1202 // If the previous store-maker is the right kind of Store, and the store is
1203 // to the same address, then we are equal to the value stored.
1204 Node* mem = in(Memory);
1205 Node* value = can_see_stored_value(mem, phase);
1206 if( value ) {
1207 // byte, short & char stores truncate naturally.
1208 // A load has to load the truncated value which requires
1209 // some sort of masking operation and that requires an
1210 // Ideal call instead of an Identity call.
1211 if (memory_size() < BytesPerInt) {
1212 // If the input to the store does not fit with the load's result type,
1213 // it must be truncated via an Ideal call.
1214 if (!phase->type(value)->higher_equal(phase->type(this)))
1215 return this;
1216 }
1217 // (This works even when value is a Con, but LoadNode::Value
1218 // usually runs first, producing the singleton type of the Con.)
1219 return value;
1220 }
1221
1891 }
1892 }
1893
1894 // Don't do this for integer types. There is only potential profit if
1895 // the element type t is lower than _type; that is, for int types, if _type is
1896 // more restrictive than t. This only happens here if one is short and the other
1897 // char (both 16 bits), and in those cases we've made an intentional decision
1898 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1899 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1900 //
1901 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1902 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
1903 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1904 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
1905 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1906 // In fact, that could have been the original type of p1, and p1 could have
1907 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1908 // expression (LShiftL quux 3) independently optimized to the constant 8.
1909 if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1910 && (_type->isa_vect() == NULL)
1911 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1912 // t might actually be lower than _type, if _type is a unique
1913 // concrete subclass of abstract class t.
1914 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
1915 const Type* jt = t->join_speculative(_type);
1916 // In any case, do not allow the join, per se, to empty out the type.
1917 if (jt->empty() && !t->empty()) {
1918 // This can happen if a interface-typed array narrows to a class type.
1919 jt = _type;
1920 }
1921 #ifdef ASSERT
1922 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1923 // The pointers in the autobox arrays are always non-null
1924 Node* base = adr->in(AddPNode::Base);
1925 if ((base != NULL) && base->is_DecodeN()) {
1926 // Get LoadN node which loads IntegerCache.cache field
1927 base = base->in(1);
1928 }
1929 if ((base != NULL) && base->is_Con()) {
1930 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1931 if ((base_type != NULL) && base_type->is_autobox_cache()) {
1932 // It could be narrow oop
1933 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1934 }
1935 }
1936 }
1937 #endif
1938 return jt;
1939 }
1940 }
1941 } else if (tp->base() == Type::InstPtr) {
1942 assert( off != Type::OffsetBot ||
1943 // arrays can be cast to Objects
1944 !tp->isa_instptr() ||
1945 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
1946 // unsafe field access may not have a constant offset
1947 C->has_unsafe_access(),
1948 "Field accesses must be precise" );
1949 // For oop loads, we expect the _type to be precise.
1950
1951 // Optimize loads from constant fields.
1952 const TypeInstPtr* tinst = tp->is_instptr();
1953 ciObject* const_oop = tinst->const_oop();
1954 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1955 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
1956 if (con_type != NULL) {
1957 return con_type;
1958 }
1959 }
1960 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
1961 assert(off != Type::OffsetBot ||
1962 !tp->isa_instklassptr() ||
1963 // arrays can be cast to Objects
1964 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
1965 // also allow array-loading from the primary supertype
1966 // array during subtype checks
1967 Opcode() == Op_LoadKlass,
1968 "Field accesses must be precise");
1969 // For klass/static loads, we expect the _type to be precise
1970 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
1971 /* With mirrors being an indirect in the Klass*
1972 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
1973 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
1974 *
1975 * So check the type and klass of the node before the LoadP.
1976 */
1977 Node* adr2 = adr->in(MemNode::Address);
1978 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
1979 if (tkls != NULL && !StressReflectiveCode) {
1980 if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
1981 ciKlass* klass = tkls->exact_klass();
1982 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1983 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1984 return TypeInstPtr::make(klass->java_mirror());
1985 }
1986 }
1987 }
1988
1989 const TypeKlassPtr *tkls = tp->isa_klassptr();
1990 if (tkls != NULL && !StressReflectiveCode) {
1991 if (tkls->is_loaded() && tkls->klass_is_exact()) {
1992 ciKlass* klass = tkls->exact_klass();
1993 // We are loading a field from a Klass metaobject whose identity
1994 // is known at compile time (the type is "exact" or "precise").
1995 // Check for fields we know are maintained as constants by the VM.
1996 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
1997 // The field is Klass::_super_check_offset. Return its (constant) value.
1998 // (Folds up type checking code.)
1999 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2000 return TypeInt::make(klass->super_check_offset());
2001 }
2002 // Compute index into primary_supers array
2003 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2004 // Check for overflowing; use unsigned compare to handle the negative case.
2070 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2071 Node* value = can_see_stored_value(mem,phase);
2072 if (value != NULL && value->is_Con()) {
2073 assert(value->bottom_type()->higher_equal(_type),"sanity");
2074 return value->bottom_type();
2075 }
2076 }
2077
2078 bool is_vect = (_type->isa_vect() != NULL);
2079 if (is_instance && !is_vect) {
2080 // If we have an instance type and our memory input is the
2081 // programs's initial memory state, there is no matching store,
2082 // so just return a zero of the appropriate type -
2083 // except if it is vectorized - then we have no zero constant.
2084 Node *mem = in(MemNode::Memory);
2085 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2086 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2087 return Type::get_zero_type(_type->basic_type());
2088 }
2089 }
2090
2091 Node* alloc = is_new_object_mark_load(phase);
2092 if (alloc != NULL) {
2093 return TypeX::make(markWord::prototype().value());
2094 }
2095
2096 return _type;
2097 }
2098
2099 //------------------------------match_edge-------------------------------------
2100 // Do we Match on this edge index or not? Match only the address.
2101 uint LoadNode::match_edge(uint idx) const {
2102 return idx == MemNode::Address;
2103 }
2104
2105 //--------------------------LoadBNode::Ideal--------------------------------------
2106 //
2107 // If the previous store is to the same address as this load,
2108 // and the value stored was larger than a byte, replace this load
2109 // with the value stored truncated to a byte. If no truncation is
2110 // needed, the replacement is done in LoadNode::Identity().
2111 //
2112 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2113 Node* mem = in(MemNode::Memory);
2224 return LoadNode::Ideal(phase, can_reshape);
2225 }
2226
2227 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2228 Node* mem = in(MemNode::Memory);
2229 Node* value = can_see_stored_value(mem,phase);
2230 if (value != NULL && value->is_Con() &&
2231 !value->bottom_type()->higher_equal(_type)) {
2232 // If the input to the store does not fit with the load's result type,
2233 // it must be truncated. We can't delay until Ideal call since
2234 // a singleton Value is needed for split_thru_phi optimization.
2235 int con = value->get_int();
2236 return TypeInt::make((con << 16) >> 16);
2237 }
2238 return LoadNode::Value(phase);
2239 }
2240
2241 //=============================================================================
2242 //----------------------------LoadKlassNode::make------------------------------
2243 // Polymorphic factory method:
2244 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2245 // sanity check the alias category against the created node type
2246 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2247 assert(adr_type != NULL, "expecting TypeKlassPtr");
2248 #ifdef _LP64
2249 if (adr_type->is_ptr_to_narrowklass()) {
2250 assert(UseCompressedClassPointers, "no compressed klasses");
2251 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2252 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2253 }
2254 #endif
2255 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2256 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2257 }
2258
2259 //------------------------------Value------------------------------------------
2260 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2261 return klass_value_common(phase);
2262 }
2263
2264 // In most cases, LoadKlassNode does not have the control input set. If the control
2271 // Either input is TOP ==> the result is TOP
2272 const Type *t1 = phase->type( in(MemNode::Memory) );
2273 if (t1 == Type::TOP) return Type::TOP;
2274 Node *adr = in(MemNode::Address);
2275 const Type *t2 = phase->type( adr );
2276 if (t2 == Type::TOP) return Type::TOP;
2277 const TypePtr *tp = t2->is_ptr();
2278 if (TypePtr::above_centerline(tp->ptr()) ||
2279 tp->ptr() == TypePtr::Null) return Type::TOP;
2280
2281 // Return a more precise klass, if possible
2282 const TypeInstPtr *tinst = tp->isa_instptr();
2283 if (tinst != NULL) {
2284 ciInstanceKlass* ik = tinst->instance_klass();
2285 int offset = tinst->offset();
2286 if (ik == phase->C->env()->Class_klass()
2287 && (offset == java_lang_Class::klass_offset() ||
2288 offset == java_lang_Class::array_klass_offset())) {
2289 // We are loading a special hidden field from a Class mirror object,
2290 // the field which points to the VM's Klass metaobject.
2291 ciType* t = tinst->java_mirror_type();
2292 // java_mirror_type returns non-null for compile-time Class constants.
2293 if (t != NULL) {
2294 // constant oop => constant klass
2295 if (offset == java_lang_Class::array_klass_offset()) {
2296 if (t->is_void()) {
2297 // We cannot create a void array. Since void is a primitive type return null
2298 // klass. Users of this result need to do a null check on the returned klass.
2299 return TypePtr::NULL_PTR;
2300 }
2301 return TypeKlassPtr::make(ciArrayKlass::make(t));
2302 }
2303 if (!t->is_klass()) {
2304 // a primitive Class (e.g., int.class) has NULL for a klass field
2305 return TypePtr::NULL_PTR;
2306 }
2307 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2308 return TypeKlassPtr::make(t->as_klass());
2309 }
2310 // non-constant mirror, so we can't tell what's going on
2311 }
2312 if (!tinst->is_loaded())
2313 return _type; // Bail out if not loaded
2314 if (offset == oopDesc::klass_offset_in_bytes()) {
2315 return tinst->as_klass_type(true);
2316 }
2317 }
2318
2319 // Check for loading klass from an array
2320 const TypeAryPtr *tary = tp->isa_aryptr();
2321 if (tary != NULL && tary->elem() != Type::BOTTOM &&
2322 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2323 return tary->as_klass_type(true);
2324 }
2325
2326 // Check for loading klass from an array klass
2327 const TypeKlassPtr *tkls = tp->isa_klassptr();
2328 if (tkls != NULL && !StressReflectiveCode) {
2329 if (!tkls->is_loaded())
2330 return _type; // Bail out if not loaded
2331 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2332 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2333 // // Always returning precise element type is incorrect,
2334 // // e.g., element type could be object and array may contain strings
2335 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2336
2337 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2338 // according to the element type's subclassing.
2339 return tkls->is_aryklassptr()->elem();
2340 }
2526 //=============================================================================
2527 //---------------------------StoreNode::make-----------------------------------
2528 // Polymorphic factory method:
2529 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2530 assert((mo == unordered || mo == release), "unexpected");
2531 Compile* C = gvn.C;
2532 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2533 ctl != NULL, "raw memory operations should have control edge");
2534
2535 switch (bt) {
2536 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2537 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2538 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2539 case T_CHAR:
2540 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2541 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2542 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2543 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2544 case T_METADATA:
2545 case T_ADDRESS:
2546 case T_OBJECT:
2547 #ifdef _LP64
2548 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2549 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2550 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2551 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2552 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2553 adr->bottom_type()->isa_rawptr())) {
2554 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2555 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2556 }
2557 #endif
2558 {
2559 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2560 }
2561 default:
2562 ShouldNotReachHere();
2563 return (StoreNode*)NULL;
2564 }
2565 }
2576
2577 // Since they are not commoned, do not hash them:
2578 return NO_HASH;
2579 }
2580
2581 //------------------------------Ideal------------------------------------------
2582 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2583 // When a store immediately follows a relevant allocation/initialization,
2584 // try to capture it into the initialization, or hoist it above.
2585 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2586 Node* p = MemNode::Ideal_common(phase, can_reshape);
2587 if (p) return (p == NodeSentinel) ? NULL : p;
2588
2589 Node* mem = in(MemNode::Memory);
2590 Node* address = in(MemNode::Address);
2591 Node* value = in(MemNode::ValueIn);
2592 // Back-to-back stores to same address? Fold em up. Generally
2593 // unsafe if I have intervening uses... Also disallowed for StoreCM
2594 // since they must follow each StoreP operation. Redundant StoreCMs
2595 // are eliminated just before matching in final_graph_reshape.
2596 {
2597 Node* st = mem;
2598 // If Store 'st' has more than one use, we cannot fold 'st' away.
2599 // For example, 'st' might be the final state at a conditional
2600 // return. Or, 'st' might be used by some node which is live at
2601 // the same time 'st' is live, which might be unschedulable. So,
2602 // require exactly ONE user until such time as we clone 'mem' for
2603 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2604 // true).
2605 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2606 // Looking at a dead closed cycle of memory?
2607 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2608 assert(Opcode() == st->Opcode() ||
2609 st->Opcode() == Op_StoreVector ||
2610 Opcode() == Op_StoreVector ||
2611 st->Opcode() == Op_StoreVectorScatter ||
2612 Opcode() == Op_StoreVectorScatter ||
2613 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2614 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2615 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2616 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2617 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2618
2619 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2620 st->as_Store()->memory_size() <= this->memory_size()) {
2621 Node* use = st->raw_out(0);
2622 if (phase->is_IterGVN()) {
2623 phase->is_IterGVN()->rehash_node_delayed(use);
2624 }
2625 // It's OK to do this in the parser, since DU info is always accurate,
2626 // and the parser always refers to nodes via SafePointNode maps.
2627 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2628 return this;
2629 }
2630 st = st->in(MemNode::Memory);
2631 }
2632 }
2633
2634
2635 // Capture an unaliased, unconditional, simple store into an initializer.
2692 // Load then Store? Then the Store is useless
2693 if (val->is_Load() &&
2694 val->in(MemNode::Address)->eqv_uncast(adr) &&
2695 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2696 val->as_Load()->store_Opcode() == Opcode()) {
2697 result = mem;
2698 }
2699
2700 // Two stores in a row of the same value?
2701 if (result == this &&
2702 mem->is_Store() &&
2703 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2704 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2705 mem->Opcode() == Opcode()) {
2706 result = mem;
2707 }
2708
2709 // Store of zero anywhere into a freshly-allocated object?
2710 // Then the store is useless.
2711 // (It must already have been captured by the InitializeNode.)
2712 if (result == this &&
2713 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2714 // a newly allocated object is already all-zeroes everywhere
2715 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
2716 result = mem;
2717 }
2718
2719 if (result == this) {
2720 // the store may also apply to zero-bits in an earlier object
2721 Node* prev_mem = find_previous_store(phase);
2722 // Steps (a), (b): Walk past independent stores to find an exact match.
2723 if (prev_mem != NULL) {
2724 Node* prev_val = can_see_stored_value(prev_mem, phase);
2725 if (prev_val != NULL && prev_val == val) {
2726 // prev_val and val might differ by a cast; it would be good
2727 // to keep the more informative of the two.
2728 result = mem;
2729 }
2730 }
2731 }
2732 }
2733
2734 PhaseIterGVN* igvn = phase->is_IterGVN();
2735 if (result != this && igvn != NULL) {
2736 MemBarNode* trailing = trailing_membar();
2737 if (trailing != NULL) {
2738 #ifdef ASSERT
2739 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2884 Node* StoreCMNode::Identity(PhaseGVN* phase) {
2885 // No need to card mark when storing a null ptr
2886 Node* my_store = in(MemNode::OopStore);
2887 if (my_store->is_Store()) {
2888 const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
2889 if( t1 == TypePtr::NULL_PTR ) {
2890 return in(MemNode::Memory);
2891 }
2892 }
2893 return this;
2894 }
2895
2896 //=============================================================================
2897 //------------------------------Ideal---------------------------------------
2898 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
2899 Node* progress = StoreNode::Ideal(phase, can_reshape);
2900 if (progress != NULL) return progress;
2901
2902 Node* my_store = in(MemNode::OopStore);
2903 if (my_store->is_MergeMem()) {
2904 Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
2905 set_req_X(MemNode::OopStore, mem, phase);
2906 return this;
2907 }
2908
2909 return NULL;
2910 }
2911
2912 //------------------------------Value-----------------------------------------
2913 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
2914 // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
2915 // If extra input is TOP ==> the result is TOP
2916 const Type* t = phase->type(in(MemNode::OopStore));
2917 if (t == Type::TOP) {
2918 return Type::TOP;
2919 }
2920 return StoreNode::Value(phase);
2921 }
2922
2923
2924 //=============================================================================
2925 //----------------------------------SCMemProjNode------------------------------
2926 const Type* SCMemProjNode::Value(PhaseGVN* phase) const
3045 // Clearing a short array is faster with stores
3046 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3047 // Already know this is a large node, do not try to ideal it
3048 if (_is_large) return NULL;
3049
3050 const int unit = BytesPerLong;
3051 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3052 if (!t) return NULL;
3053 if (!t->is_con()) return NULL;
3054 intptr_t raw_count = t->get_con();
3055 intptr_t size = raw_count;
3056 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3057 // Clearing nothing uses the Identity call.
3058 // Negative clears are possible on dead ClearArrays
3059 // (see jck test stmt114.stmt11402.val).
3060 if (size <= 0 || size % unit != 0) return NULL;
3061 intptr_t count = size / unit;
3062 // Length too long; communicate this to matchers and assemblers.
3063 // Assemblers are responsible to produce fast hardware clears for it.
3064 if (size > InitArrayShortSize) {
3065 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3066 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3067 return NULL;
3068 }
3069 if (!IdealizeClearArrayNode) return NULL;
3070 Node *mem = in(1);
3071 if( phase->type(mem)==Type::TOP ) return NULL;
3072 Node *adr = in(3);
3073 const Type* at = phase->type(adr);
3074 if( at==Type::TOP ) return NULL;
3075 const TypePtr* atp = at->isa_ptr();
3076 // adjust atp to be the correct array element address type
3077 if (atp == NULL) atp = TypePtr::BOTTOM;
3078 else atp = atp->add_offset(Type::OffsetBot);
3079 // Get base for derived pointer purposes
3080 if( adr->Opcode() != Op_AddP ) Unimplemented();
3081 Node *base = adr->in(1);
3082
3083 Node *zero = phase->makecon(TypeLong::ZERO);
3084 Node *off = phase->MakeConX(BytesPerLong);
3085 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3086 count--;
3087 while( count-- ) {
3088 mem = phase->transform(mem);
3089 adr = phase->transform(new AddPNode(base,adr,off));
3090 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3091 }
3092 return mem;
3093 }
3094
3095 //----------------------------step_through----------------------------------
3096 // Return allocation input memory edge if it is different instance
3097 // or itself if it is the one we are looking for.
3098 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
3099 Node* n = *np;
3100 assert(n->is_ClearArray(), "sanity");
3101 intptr_t offset;
3102 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3103 // This method is called only before Allocate nodes are expanded
3104 // during macro nodes expansion. Before that ClearArray nodes are
3105 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3106 // Allocate nodes are expanded) which follows allocations.
3107 assert(alloc != NULL, "should have allocation");
3108 if (alloc->_idx == instance_id) {
3109 // Can not bypass initialization of the instance we are looking for.
3110 return false;
3111 }
3112 // Otherwise skip it.
3113 InitializeNode* init = alloc->initialization();
3114 if (init != NULL)
3115 *np = init->in(TypeFunc::Memory);
3116 else
3117 *np = alloc->in(TypeFunc::Memory);
3118 return true;
3119 }
3120
3121 //----------------------------clear_memory-------------------------------------
3122 // Generate code to initialize object storage to zero.
3123 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3124 intptr_t start_offset,
3125 Node* end_offset,
3126 PhaseGVN* phase) {
3127 intptr_t offset = start_offset;
3128
3129 int unit = BytesPerLong;
3130 if ((offset % unit) != 0) {
3131 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3132 adr = phase->transform(adr);
3133 const TypePtr* atp = TypeRawPtr::BOTTOM;
3134 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3135 mem = phase->transform(mem);
3136 offset += BytesPerInt;
3137 }
3138 assert((offset % unit) == 0, "");
3139
3140 // Initialize the remaining stuff, if any, with a ClearArray.
3141 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
3142 }
3143
3144 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3145 Node* start_offset,
3146 Node* end_offset,
3147 PhaseGVN* phase) {
3148 if (start_offset == end_offset) {
3149 // nothing to do
3150 return mem;
3151 }
3152
3153 int unit = BytesPerLong;
3154 Node* zbase = start_offset;
3155 Node* zend = end_offset;
3156
3157 // Scale to the unit required by the CPU:
3158 if (!Matcher::init_array_count_is_in_bytes) {
3159 Node* shift = phase->intcon(exact_log2(unit));
3160 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3161 zend = phase->transform(new URShiftXNode(zend, shift) );
3162 }
3163
3164 // Bulk clear double-words
3165 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3166 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3167 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
3168 return phase->transform(mem);
3169 }
3170
3171 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3172 intptr_t start_offset,
3173 intptr_t end_offset,
3174 PhaseGVN* phase) {
3175 if (start_offset == end_offset) {
3176 // nothing to do
3177 return mem;
3178 }
3179
3180 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3181 intptr_t done_offset = end_offset;
3182 if ((done_offset % BytesPerLong) != 0) {
3183 done_offset -= BytesPerInt;
3184 }
3185 if (done_offset > start_offset) {
3186 mem = clear_memory(ctl, mem, dest,
3187 start_offset, phase->MakeConX(done_offset), phase);
3188 }
3189 if (done_offset < end_offset) { // emit the final 32-bit store
3190 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3191 adr = phase->transform(adr);
3192 const TypePtr* atp = TypeRawPtr::BOTTOM;
3193 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3194 mem = phase->transform(mem);
3195 done_offset += BytesPerInt;
3196 }
3197 assert(done_offset == end_offset, "");
3198 return mem;
3199 }
3200
3201 //=============================================================================
3202 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3203 : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3204 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3205 #ifdef ASSERT
3206 , _pair_idx(0)
3207 #endif
3208 {
3209 init_class_id(Class_MemBar);
3210 Node* top = C->top();
3211 init_req(TypeFunc::I_O,top);
3212 init_req(TypeFunc::FramePtr,top);
3213 init_req(TypeFunc::ReturnAdr,top);
3319 PhaseIterGVN* igvn = phase->is_IterGVN();
3320 remove(igvn);
3321 // Must return either the original node (now dead) or a new node
3322 // (Do not return a top here, since that would break the uniqueness of top.)
3323 return new ConINode(TypeInt::ZERO);
3324 }
3325 }
3326 return progress ? this : NULL;
3327 }
3328
3329 //------------------------------Value------------------------------------------
3330 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3331 if( !in(0) ) return Type::TOP;
3332 if( phase->type(in(0)) == Type::TOP )
3333 return Type::TOP;
3334 return TypeTuple::MEMBAR;
3335 }
3336
3337 //------------------------------match------------------------------------------
3338 // Construct projections for memory.
3339 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3340 switch (proj->_con) {
3341 case TypeFunc::Control:
3342 case TypeFunc::Memory:
3343 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3344 }
3345 ShouldNotReachHere();
3346 return NULL;
3347 }
3348
3349 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3350 trailing->_kind = TrailingStore;
3351 leading->_kind = LeadingStore;
3352 #ifdef ASSERT
3353 trailing->_pair_idx = leading->_idx;
3354 leading->_pair_idx = leading->_idx;
3355 #endif
3356 }
3357
3358 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3359 trailing->_kind = TrailingLoadStore;
3626 return (req() > RawStores);
3627 }
3628
3629 void InitializeNode::set_complete(PhaseGVN* phase) {
3630 assert(!is_complete(), "caller responsibility");
3631 _is_complete = Complete;
3632
3633 // After this node is complete, it contains a bunch of
3634 // raw-memory initializations. There is no need for
3635 // it to have anything to do with non-raw memory effects.
3636 // Therefore, tell all non-raw users to re-optimize themselves,
3637 // after skipping the memory effects of this initialization.
3638 PhaseIterGVN* igvn = phase->is_IterGVN();
3639 if (igvn) igvn->add_users_to_worklist(this);
3640 }
3641
3642 // convenience function
3643 // return false if the init contains any stores already
3644 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3645 InitializeNode* init = initialization();
3646 if (init == NULL || init->is_complete()) return false;
3647 init->remove_extra_zeroes();
3648 // for now, if this allocation has already collected any inits, bail:
3649 if (init->is_non_zero()) return false;
3650 init->set_complete(phase);
3651 return true;
3652 }
3653
3654 void InitializeNode::remove_extra_zeroes() {
3655 if (req() == RawStores) return;
3656 Node* zmem = zero_memory();
3657 uint fill = RawStores;
3658 for (uint i = fill; i < req(); i++) {
3659 Node* n = in(i);
3660 if (n->is_top() || n == zmem) continue; // skip
3661 if (fill < i) set_req(fill, n); // compact
3662 ++fill;
3663 }
3664 // delete any empty spaces created:
3665 while (fill < req()) {
3666 del_req(fill);
3804 // store node that we'd like to capture. We need to check
3805 // the uses of the MergeMemNode.
3806 mems.push(n);
3807 }
3808 } else if (n->is_Mem()) {
3809 Node* other_adr = n->in(MemNode::Address);
3810 if (other_adr == adr) {
3811 failed = true;
3812 break;
3813 } else {
3814 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3815 if (other_t_adr != NULL) {
3816 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3817 if (other_alias_idx == alias_idx) {
3818 // A load from the same memory slice as the store right
3819 // after the InitializeNode. We check the control of the
3820 // object/array that is loaded from. If it's the same as
3821 // the store control then we cannot capture the store.
3822 assert(!n->is_Store(), "2 stores to same slice on same control?");
3823 Node* base = other_adr;
3824 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3825 base = base->in(AddPNode::Base);
3826 if (base != NULL) {
3827 base = base->uncast();
3828 if (base->is_Proj() && base->in(0) == alloc) {
3829 failed = true;
3830 break;
3831 }
3832 }
3833 }
3834 }
3835 }
3836 } else {
3837 failed = true;
3838 break;
3839 }
3840 }
3841 }
3842 }
3843 if (failed) {
4388 // z's_done 12 16 16 16 12 16 12
4389 // z's_needed 12 16 16 16 16 16 16
4390 // zsize 0 0 0 0 4 0 4
4391 if (next_full_store < 0) {
4392 // Conservative tack: Zero to end of current word.
4393 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4394 } else {
4395 // Zero to beginning of next fully initialized word.
4396 // Or, don't zero at all, if we are already in that word.
4397 assert(next_full_store >= zeroes_needed, "must go forward");
4398 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4399 zeroes_needed = next_full_store;
4400 }
4401 }
4402
4403 if (zeroes_needed > zeroes_done) {
4404 intptr_t zsize = zeroes_needed - zeroes_done;
4405 // Do some incremental zeroing on rawmem, in parallel with inits.
4406 zeroes_done = align_down(zeroes_done, BytesPerInt);
4407 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4408 zeroes_done, zeroes_needed,
4409 phase);
4410 zeroes_done = zeroes_needed;
4411 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4412 do_zeroing = false; // leave the hole, next time
4413 }
4414 }
4415
4416 // Collect the store and move on:
4417 phase->replace_input_of(st, MemNode::Memory, inits);
4418 inits = st; // put it on the linearized chain
4419 set_req(i, zmem); // unhook from previous position
4420
4421 if (zeroes_done == st_off)
4422 zeroes_done = next_init_off;
4423
4424 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4425
4426 #ifdef ASSERT
4427 // Various order invariants. Weaker than stores_are_sane because
4447 remove_extra_zeroes(); // clear out all the zmems left over
4448 add_req(inits);
4449
4450 if (!(UseTLAB && ZeroTLAB)) {
4451 // If anything remains to be zeroed, zero it all now.
4452 zeroes_done = align_down(zeroes_done, BytesPerInt);
4453 // if it is the last unused 4 bytes of an instance, forget about it
4454 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4455 if (zeroes_done + BytesPerLong >= size_limit) {
4456 AllocateNode* alloc = allocation();
4457 assert(alloc != NULL, "must be present");
4458 if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4459 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4460 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4461 if (zeroes_done == k->layout_helper())
4462 zeroes_done = size_limit;
4463 }
4464 }
4465 if (zeroes_done < size_limit) {
4466 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4467 zeroes_done, size_in_bytes, phase);
4468 }
4469 }
4470
4471 set_complete(phase);
4472 return rawmem;
4473 }
4474
4475
4476 #ifdef ASSERT
4477 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4478 if (is_complete())
4479 return true; // stores could be anything at this point
4480 assert(allocation() != NULL, "must be present");
4481 intptr_t last_off = allocation()->minimum_header_size();
4482 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4483 Node* st = in(i);
4484 intptr_t st_off = get_store_offset(st, phase);
4485 if (st_off < 0) continue; // ignore dead garbage
4486 if (last_off > st_off) {
|
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciFlatArrayKlass.hpp"
27 #include "classfile/javaClasses.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "compiler/compileLog.hpp"
30 #include "gc/shared/barrierSet.hpp"
31 #include "gc/shared/c2/barrierSetC2.hpp"
32 #include "gc/shared/tlab_globals.hpp"
33 #include "memory/allocation.inline.hpp"
34 #include "memory/resourceArea.hpp"
35 #include "oops/objArrayKlass.hpp"
36 #include "opto/addnode.hpp"
37 #include "opto/arraycopynode.hpp"
38 #include "opto/cfgnode.hpp"
39 #include "opto/regalloc.hpp"
40 #include "opto/compile.hpp"
41 #include "opto/connode.hpp"
42 #include "opto/convertnode.hpp"
43 #include "opto/inlinetypenode.hpp"
44 #include "opto/loopnode.hpp"
45 #include "opto/machnode.hpp"
46 #include "opto/matcher.hpp"
47 #include "opto/memnode.hpp"
48 #include "opto/mulnode.hpp"
49 #include "opto/narrowptrnode.hpp"
50 #include "opto/phaseX.hpp"
51 #include "opto/regmask.hpp"
52 #include "opto/rootnode.hpp"
53 #include "opto/vectornode.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/macros.hpp"
57 #include "utilities/powerOfTwo.hpp"
58 #include "utilities/vmError.hpp"
59
60 // Portions of code courtesy of Clifford Click
61
62 // Optimization - Graph Style
63
226 // clone the Phi with our address type
227 result = mphi->split_out_instance(t_adr, igvn);
228 } else {
229 assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
230 }
231 }
232 return result;
233 }
234
235 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
236 uint alias_idx = phase->C->get_alias_index(tp);
237 Node *mem = mmem;
238 #ifdef ASSERT
239 {
240 // Check that current type is consistent with the alias index used during graph construction
241 assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
242 bool consistent = adr_check == NULL || adr_check->empty() ||
243 phase->C->must_alias(adr_check, alias_idx );
244 // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
245 if( !consistent && adr_check != NULL && !adr_check->empty() &&
246 tp->isa_aryptr() && tp->offset() == Type::OffsetBot &&
247 adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
248 ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
249 adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
250 adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
251 // don't assert if it is dead code.
252 consistent = true;
253 }
254 if( !consistent ) {
255 st->print("alias_idx==%d, adr_check==", alias_idx);
256 if( adr_check == NULL ) {
257 st->print("NULL");
258 } else {
259 adr_check->dump();
260 }
261 st->cr();
262 print_alias_types();
263 assert(consistent, "adr_check must match alias idx");
264 }
265 }
266 #endif
884 "use LoadKlassNode instead");
885 assert(!(adr_type->isa_aryptr() &&
886 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
887 "use LoadRangeNode instead");
888 // Check control edge of raw loads
889 assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
890 // oop will be recorded in oop map if load crosses safepoint
891 rt->isa_oopptr() || is_immutable_value(adr),
892 "raw memory operations should have control edge");
893 LoadNode* load = NULL;
894 switch (bt) {
895 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
896 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
897 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
898 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
899 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
900 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
901 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
902 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency, require_atomic_access); break;
903 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
904 case T_PRIMITIVE_OBJECT:
905 case T_OBJECT:
906 #ifdef _LP64
907 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
908 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
909 } else
910 #endif
911 {
912 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
913 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
914 }
915 break;
916 default:
917 ShouldNotReachHere();
918 break;
919 }
920 assert(load != NULL, "LoadNode should have been created");
921 if (unaligned) {
922 load->set_unaligned_access();
923 }
924 if (mismatched) {
980 assert(ld_alloc != NULL, "need an alloc");
981 assert(addp->is_AddP(), "address must be addp");
982 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
983 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
984 assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
985 addp->set_req(AddPNode::Base, src);
986 addp->set_req(AddPNode::Address, src);
987 } else {
988 assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
989 ac->as_ArrayCopy()->is_copyof_validated() ||
990 ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
991 assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
992 addp->set_req(AddPNode::Base, src);
993 addp->set_req(AddPNode::Address, src);
994
995 const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
996 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
997 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
998
999 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1000 uint shift = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1001
1002 Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1003 #ifdef _LP64
1004 diff = phase->transform(new ConvI2LNode(diff));
1005 #endif
1006 diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1007
1008 Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1009 addp->set_req(AddPNode::Offset, offset);
1010 }
1011 addp = phase->transform(addp);
1012 #ifdef ASSERT
1013 const TypePtr* adr_type = phase->type(addp)->is_ptr();
1014 ld->_adr_type = adr_type;
1015 #endif
1016 ld->set_req(MemNode::Address, addp);
1017 ld->set_req(0, ctl);
1018 ld->set_req(MemNode::Memory, mem);
1019 // load depends on the tests that validate the arraycopy
1020 ld->_control_dependency = UnknownControl;
1101 // Same base, same offset.
1102 // Possible improvement for arrays: check index value instead of absolute offset.
1103
1104 // At this point we have proven something like this setup:
1105 // B = << base >>
1106 // L = LoadQ(AddP(Check/CastPP(B), #Off))
1107 // S = StoreQ(AddP( B , #Off), V)
1108 // (Actually, we haven't yet proven the Q's are the same.)
1109 // In other words, we are loading from a casted version of
1110 // the same pointer-and-offset that we stored to.
1111 // Casted version may carry a dependency and it is respected.
1112 // Thus, we are able to replace L by V.
1113 }
1114 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1115 if (store_Opcode() != st->Opcode()) {
1116 return NULL;
1117 }
1118 // LoadVector/StoreVector needs additional check to ensure the types match.
1119 if (st->is_StoreVector()) {
1120 const TypeVect* in_vt = st->as_StoreVector()->vect_type();
1121 const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1122 if (in_vt != out_vt) {
1123 return NULL;
1124 }
1125 }
1126 return st->in(MemNode::ValueIn);
1127 }
1128
1129 // A load from a freshly-created object always returns zero.
1130 // (This can happen after LoadNode::Ideal resets the load's memory input
1131 // to find_captured_store, which returned InitializeNode::zero_memory.)
1132 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1133 (st->in(0) == ld_alloc) &&
1134 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1135 // return a zero value for the load's basic type
1136 // (This is one of the few places where a generic PhaseTransform
1137 // can create new nodes. Think of it as lazily manifesting
1138 // virtually pre-existing constants.)
1139 assert(memory_type() != T_PRIMITIVE_OBJECT, "should not be used for inline types");
1140 Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1141 if (default_value != NULL) {
1142 return default_value;
1143 }
1144 assert(ld_alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
1145 if (memory_type() != T_VOID) {
1146 if (ReduceBulkZeroing || find_array_copy_clone(phase, ld_alloc, in(MemNode::Memory)) == NULL) {
1147 // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1148 // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1149 // by the ArrayCopyNode.
1150 return phase->zerocon(memory_type());
1151 }
1152 } else {
1153 // TODO: materialize all-zero vector constant
1154 assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1155 }
1156 }
1157
1158 // A load from an initialization barrier can match a captured store.
1159 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1160 InitializeNode* init = st->in(0)->as_Initialize();
1161 AllocateNode* alloc = init->allocation();
1162 if ((alloc != NULL) && (alloc == ld_alloc)) {
1163 // examine a captured store value
1164 st = init->find_captured_store(ld_off, memory_size(), phase);
1192 //----------------------is_instance_field_load_with_local_phi------------------
1193 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1194 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1195 in(Address)->is_AddP() ) {
1196 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1197 // Only instances and boxed values.
1198 if( t_oop != NULL &&
1199 (t_oop->is_ptr_to_boxed_value() ||
1200 t_oop->is_known_instance_field()) &&
1201 t_oop->offset() != Type::OffsetBot &&
1202 t_oop->offset() != Type::OffsetTop) {
1203 return true;
1204 }
1205 }
1206 return false;
1207 }
1208
1209 //------------------------------Identity---------------------------------------
1210 // Loads are identity if previous store is to same address
1211 Node* LoadNode::Identity(PhaseGVN* phase) {
1212 // Loading from an InlineTypePtr? The InlineTypePtr has the values of
1213 // all fields as input. Look for the field with matching offset.
1214 Node* addr = in(Address);
1215 intptr_t offset;
1216 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1217 if (base != NULL && base->is_InlineTypePtr() && offset > oopDesc::klass_offset_in_bytes()) {
1218 Node* value = base->as_InlineTypePtr()->field_value_by_offset((int)offset, true);
1219 if (value->is_InlineType()) {
1220 // Non-flattened inline type field
1221 InlineTypeNode* vt = value->as_InlineType();
1222 if (vt->is_allocated(phase)) {
1223 value = vt->get_oop();
1224 } else {
1225 // Not yet allocated, bail out
1226 value = NULL;
1227 }
1228 }
1229 if (value != NULL) {
1230 if (Opcode() == Op_LoadN) {
1231 // Encode oop value if we are loading a narrow oop
1232 assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1233 value = phase->transform(new EncodePNode(value, bottom_type()));
1234 }
1235 return value;
1236 }
1237 }
1238
1239 // If the previous store-maker is the right kind of Store, and the store is
1240 // to the same address, then we are equal to the value stored.
1241 Node* mem = in(Memory);
1242 Node* value = can_see_stored_value(mem, phase);
1243 if( value ) {
1244 // byte, short & char stores truncate naturally.
1245 // A load has to load the truncated value which requires
1246 // some sort of masking operation and that requires an
1247 // Ideal call instead of an Identity call.
1248 if (memory_size() < BytesPerInt) {
1249 // If the input to the store does not fit with the load's result type,
1250 // it must be truncated via an Ideal call.
1251 if (!phase->type(value)->higher_equal(phase->type(this)))
1252 return this;
1253 }
1254 // (This works even when value is a Con, but LoadNode::Value
1255 // usually runs first, producing the singleton type of the Con.)
1256 return value;
1257 }
1258
1928 }
1929 }
1930
1931 // Don't do this for integer types. There is only potential profit if
1932 // the element type t is lower than _type; that is, for int types, if _type is
1933 // more restrictive than t. This only happens here if one is short and the other
1934 // char (both 16 bits), and in those cases we've made an intentional decision
1935 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1936 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1937 //
1938 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1939 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
1940 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1941 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
1942 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1943 // In fact, that could have been the original type of p1, and p1 could have
1944 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1945 // expression (LShiftL quux 3) independently optimized to the constant 8.
1946 if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1947 && (_type->isa_vect() == NULL)
1948 && t->isa_inlinetype() == NULL
1949 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1950 // t might actually be lower than _type, if _type is a unique
1951 // concrete subclass of abstract class t.
1952 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
1953 const Type* jt = t->join_speculative(_type);
1954 // In any case, do not allow the join, per se, to empty out the type.
1955 if (jt->empty() && !t->empty()) {
1956 // This can happen if a interface-typed array narrows to a class type.
1957 jt = _type;
1958 }
1959 #ifdef ASSERT
1960 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1961 // The pointers in the autobox arrays are always non-null
1962 Node* base = adr->in(AddPNode::Base);
1963 if ((base != NULL) && base->is_DecodeN()) {
1964 // Get LoadN node which loads IntegerCache.cache field
1965 base = base->in(1);
1966 }
1967 if ((base != NULL) && base->is_Con()) {
1968 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1969 if ((base_type != NULL) && base_type->is_autobox_cache()) {
1970 // It could be narrow oop
1971 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1972 }
1973 }
1974 }
1975 #endif
1976 return jt;
1977 }
1978 }
1979 } else if (tp->base() == Type::InstPtr) {
1980 assert( off != Type::OffsetBot ||
1981 // arrays can be cast to Objects
1982 !tp->isa_instptr() ||
1983 tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
1984 // Default value load
1985 tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
1986 // unsafe field access may not have a constant offset
1987 C->has_unsafe_access(),
1988 "Field accesses must be precise" );
1989 // For oop loads, we expect the _type to be precise.
1990
1991 const TypeInstPtr* tinst = tp->is_instptr();
1992 BasicType bt = memory_type();
1993
1994 // Optimize loads from constant fields.
1995 ciObject* const_oop = tinst->const_oop();
1996 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1997 ciType* mirror_type = const_oop->as_instance()->java_mirror_type();
1998 if (mirror_type != NULL) {
1999 const Type* const_oop = NULL;
2000 ciInlineKlass* vk = mirror_type->is_inlinetype() ? mirror_type->as_inline_klass() : NULL;
2001 // Fold default value loads
2002 if (vk != NULL && off == vk->default_value_offset()) {
2003 const_oop = TypeInstPtr::make(vk->default_instance());
2004 }
2005 // Fold class mirror loads
2006 if (off == java_lang_Class::primary_mirror_offset()) {
2007 const_oop = (vk == NULL) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->ref_instance());
2008 } else if (off == java_lang_Class::secondary_mirror_offset()) {
2009 const_oop = (vk == NULL) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->val_instance());
2010 }
2011 if (const_oop != NULL) {
2012 return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
2013 }
2014 }
2015 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2016 if (con_type != NULL) {
2017 return con_type;
2018 }
2019 }
2020 } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2021 assert(off != Type::OffsetBot ||
2022 !tp->isa_instklassptr() ||
2023 // arrays can be cast to Objects
2024 tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2025 // also allow array-loading from the primary supertype
2026 // array during subtype checks
2027 Opcode() == Op_LoadKlass,
2028 "Field accesses must be precise");
2029 // For klass/static loads, we expect the _type to be precise
2030 } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
2031 if (adr->is_Load() && off == 0) {
2032 /* With mirrors being an indirect in the Klass*
2033 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2034 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2035 *
2036 * So check the type and klass of the node before the LoadP.
2037 */
2038 Node* adr2 = adr->in(MemNode::Address);
2039 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2040 if (tkls != NULL) {
2041 if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2042 ciKlass* klass = tkls->exact_klass();
2043 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2044 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2045 return TypeInstPtr::make(klass->java_mirror());
2046 }
2047 }
2048 } else {
2049 // Check for a load of the default value offset from the InlineKlassFixedBlock:
2050 // LoadI(LoadP(inline_klass, adr_inlineklass_fixed_block_offset), default_value_offset_offset)
2051 intptr_t offset = 0;
2052 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2053 if (base != NULL && base->is_Load() && offset == in_bytes(InlineKlass::default_value_offset_offset())) {
2054 const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
2055 if (tkls != NULL && tkls->is_loaded() && tkls->klass_is_exact() && tkls->exact_klass()->is_inlinetype() &&
2056 tkls->offset() == in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())) {
2057 assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
2058 assert(Opcode() == Op_LoadI, "must load an int from fixed block");
2059 return TypeInt::make(tkls->exact_klass()->as_inline_klass()->default_value_offset());
2060 }
2061 }
2062 }
2063 }
2064
2065 const TypeKlassPtr *tkls = tp->isa_klassptr();
2066 if (tkls != NULL && !StressReflectiveCode) {
2067 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2068 ciKlass* klass = tkls->exact_klass();
2069 // We are loading a field from a Klass metaobject whose identity
2070 // is known at compile time (the type is "exact" or "precise").
2071 // Check for fields we know are maintained as constants by the VM.
2072 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2073 // The field is Klass::_super_check_offset. Return its (constant) value.
2074 // (Folds up type checking code.)
2075 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2076 return TypeInt::make(klass->super_check_offset());
2077 }
2078 // Compute index into primary_supers array
2079 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2080 // Check for overflowing; use unsigned compare to handle the negative case.
2146 if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2147 Node* value = can_see_stored_value(mem,phase);
2148 if (value != NULL && value->is_Con()) {
2149 assert(value->bottom_type()->higher_equal(_type),"sanity");
2150 return value->bottom_type();
2151 }
2152 }
2153
2154 bool is_vect = (_type->isa_vect() != NULL);
2155 if (is_instance && !is_vect) {
2156 // If we have an instance type and our memory input is the
2157 // programs's initial memory state, there is no matching store,
2158 // so just return a zero of the appropriate type -
2159 // except if it is vectorized - then we have no zero constant.
2160 Node *mem = in(MemNode::Memory);
2161 if (mem->is_Parm() && mem->in(0)->is_Start()) {
2162 assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2163 return Type::get_zero_type(_type->basic_type());
2164 }
2165 }
2166 Node* alloc = is_new_object_mark_load(phase);
2167 if (alloc != NULL) {
2168 if (EnableValhalla) {
2169 // The mark word may contain property bits (inline, flat, null-free)
2170 Node* klass_node = alloc->in(AllocateNode::KlassNode);
2171 const TypeKlassPtr* tkls = phase->type(klass_node)->is_klassptr();
2172 if (tkls->is_loaded() && tkls->klass_is_exact()) {
2173 return TypeX::make(tkls->exact_klass()->prototype_header().value());
2174 }
2175 } else {
2176 return TypeX::make(markWord::prototype().value());
2177 }
2178 }
2179
2180 return _type;
2181 }
2182
2183 //------------------------------match_edge-------------------------------------
2184 // Do we Match on this edge index or not? Match only the address.
2185 uint LoadNode::match_edge(uint idx) const {
2186 return idx == MemNode::Address;
2187 }
2188
2189 //--------------------------LoadBNode::Ideal--------------------------------------
2190 //
2191 // If the previous store is to the same address as this load,
2192 // and the value stored was larger than a byte, replace this load
2193 // with the value stored truncated to a byte. If no truncation is
2194 // needed, the replacement is done in LoadNode::Identity().
2195 //
2196 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2197 Node* mem = in(MemNode::Memory);
2308 return LoadNode::Ideal(phase, can_reshape);
2309 }
2310
2311 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2312 Node* mem = in(MemNode::Memory);
2313 Node* value = can_see_stored_value(mem,phase);
2314 if (value != NULL && value->is_Con() &&
2315 !value->bottom_type()->higher_equal(_type)) {
2316 // If the input to the store does not fit with the load's result type,
2317 // it must be truncated. We can't delay until Ideal call since
2318 // a singleton Value is needed for split_thru_phi optimization.
2319 int con = value->get_int();
2320 return TypeInt::make((con << 16) >> 16);
2321 }
2322 return LoadNode::Value(phase);
2323 }
2324
2325 //=============================================================================
2326 //----------------------------LoadKlassNode::make------------------------------
2327 // Polymorphic factory method:
2328 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2329 const TypeKlassPtr* tk) {
2330 // sanity check the alias category against the created node type
2331 const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2332 assert(adr_type != NULL, "expecting TypeKlassPtr");
2333 #ifdef _LP64
2334 if (adr_type->is_ptr_to_narrowklass()) {
2335 assert(UseCompressedClassPointers, "no compressed klasses");
2336 Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2337 return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2338 }
2339 #endif
2340 assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2341 return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2342 }
2343
2344 //------------------------------Value------------------------------------------
2345 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2346 return klass_value_common(phase);
2347 }
2348
2349 // In most cases, LoadKlassNode does not have the control input set. If the control
2356 // Either input is TOP ==> the result is TOP
2357 const Type *t1 = phase->type( in(MemNode::Memory) );
2358 if (t1 == Type::TOP) return Type::TOP;
2359 Node *adr = in(MemNode::Address);
2360 const Type *t2 = phase->type( adr );
2361 if (t2 == Type::TOP) return Type::TOP;
2362 const TypePtr *tp = t2->is_ptr();
2363 if (TypePtr::above_centerline(tp->ptr()) ||
2364 tp->ptr() == TypePtr::Null) return Type::TOP;
2365
2366 // Return a more precise klass, if possible
2367 const TypeInstPtr *tinst = tp->isa_instptr();
2368 if (tinst != NULL) {
2369 ciInstanceKlass* ik = tinst->instance_klass();
2370 int offset = tinst->offset();
2371 if (ik == phase->C->env()->Class_klass()
2372 && (offset == java_lang_Class::klass_offset() ||
2373 offset == java_lang_Class::array_klass_offset())) {
2374 // We are loading a special hidden field from a Class mirror object,
2375 // the field which points to the VM's Klass metaobject.
2376 bool null_free = false;
2377 ciType* t = tinst->java_mirror_type(&null_free);
2378 // java_mirror_type returns non-null for compile-time Class constants.
2379 if (t != NULL) {
2380 // constant oop => constant klass
2381 if (offset == java_lang_Class::array_klass_offset()) {
2382 if (t->is_void()) {
2383 // We cannot create a void array. Since void is a primitive type return null
2384 // klass. Users of this result need to do a null check on the returned klass.
2385 return TypePtr::NULL_PTR;
2386 }
2387 return TypeKlassPtr::make(ciArrayKlass::make(t, null_free));
2388 }
2389 if (!t->is_klass()) {
2390 // a primitive Class (e.g., int.class) has NULL for a klass field
2391 return TypePtr::NULL_PTR;
2392 }
2393 // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2394 return TypeKlassPtr::make(t->as_klass());
2395 }
2396 // non-constant mirror, so we can't tell what's going on
2397 }
2398 if (!tinst->is_loaded())
2399 return _type; // Bail out if not loaded
2400 if (offset == oopDesc::klass_offset_in_bytes()) {
2401 return tinst->as_klass_type(true);
2402 }
2403 }
2404
2405 // Check for loading klass from an array
2406 const TypeAryPtr* tary = tp->isa_aryptr();
2407 if (tary != NULL && tary->elem() != Type::BOTTOM &&
2408 tary->offset() == oopDesc::klass_offset_in_bytes()) {
2409 return tary->as_klass_type(true);
2410 }
2411
2412 // Check for loading klass from an array klass
2413 const TypeKlassPtr *tkls = tp->isa_klassptr();
2414 if (tkls != NULL && !StressReflectiveCode) {
2415 if (!tkls->is_loaded())
2416 return _type; // Bail out if not loaded
2417 if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2418 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2419 // // Always returning precise element type is incorrect,
2420 // // e.g., element type could be object and array may contain strings
2421 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2422
2423 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2424 // according to the element type's subclassing.
2425 return tkls->is_aryklassptr()->elem();
2426 }
2612 //=============================================================================
2613 //---------------------------StoreNode::make-----------------------------------
2614 // Polymorphic factory method:
2615 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2616 assert((mo == unordered || mo == release), "unexpected");
2617 Compile* C = gvn.C;
2618 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2619 ctl != NULL, "raw memory operations should have control edge");
2620
2621 switch (bt) {
2622 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2623 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2624 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2625 case T_CHAR:
2626 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2627 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2628 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2629 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2630 case T_METADATA:
2631 case T_ADDRESS:
2632 case T_PRIMITIVE_OBJECT:
2633 case T_OBJECT:
2634 #ifdef _LP64
2635 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2636 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2637 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2638 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2639 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2640 adr->bottom_type()->isa_rawptr())) {
2641 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2642 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2643 }
2644 #endif
2645 {
2646 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2647 }
2648 default:
2649 ShouldNotReachHere();
2650 return (StoreNode*)NULL;
2651 }
2652 }
2663
2664 // Since they are not commoned, do not hash them:
2665 return NO_HASH;
2666 }
2667
2668 //------------------------------Ideal------------------------------------------
2669 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2670 // When a store immediately follows a relevant allocation/initialization,
2671 // try to capture it into the initialization, or hoist it above.
2672 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2673 Node* p = MemNode::Ideal_common(phase, can_reshape);
2674 if (p) return (p == NodeSentinel) ? NULL : p;
2675
2676 Node* mem = in(MemNode::Memory);
2677 Node* address = in(MemNode::Address);
2678 Node* value = in(MemNode::ValueIn);
2679 // Back-to-back stores to same address? Fold em up. Generally
2680 // unsafe if I have intervening uses... Also disallowed for StoreCM
2681 // since they must follow each StoreP operation. Redundant StoreCMs
2682 // are eliminated just before matching in final_graph_reshape.
2683 if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
2684 Node* st = mem;
2685 // If Store 'st' has more than one use, we cannot fold 'st' away.
2686 // For example, 'st' might be the final state at a conditional
2687 // return. Or, 'st' might be used by some node which is live at
2688 // the same time 'st' is live, which might be unschedulable. So,
2689 // require exactly ONE user until such time as we clone 'mem' for
2690 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2691 // true).
2692 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2693 // Looking at a dead closed cycle of memory?
2694 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2695 assert(Opcode() == st->Opcode() ||
2696 st->Opcode() == Op_StoreVector ||
2697 Opcode() == Op_StoreVector ||
2698 st->Opcode() == Op_StoreVectorScatter ||
2699 Opcode() == Op_StoreVectorScatter ||
2700 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2701 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2702 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2703 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
2704 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2705 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2706
2707 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2708 st->as_Store()->memory_size() <= this->memory_size()) {
2709 Node* use = st->raw_out(0);
2710 if (phase->is_IterGVN()) {
2711 phase->is_IterGVN()->rehash_node_delayed(use);
2712 }
2713 // It's OK to do this in the parser, since DU info is always accurate,
2714 // and the parser always refers to nodes via SafePointNode maps.
2715 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2716 return this;
2717 }
2718 st = st->in(MemNode::Memory);
2719 }
2720 }
2721
2722
2723 // Capture an unaliased, unconditional, simple store into an initializer.
2780 // Load then Store? Then the Store is useless
2781 if (val->is_Load() &&
2782 val->in(MemNode::Address)->eqv_uncast(adr) &&
2783 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2784 val->as_Load()->store_Opcode() == Opcode()) {
2785 result = mem;
2786 }
2787
2788 // Two stores in a row of the same value?
2789 if (result == this &&
2790 mem->is_Store() &&
2791 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2792 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2793 mem->Opcode() == Opcode()) {
2794 result = mem;
2795 }
2796
2797 // Store of zero anywhere into a freshly-allocated object?
2798 // Then the store is useless.
2799 // (It must already have been captured by the InitializeNode.)
2800 if (result == this && ReduceFieldZeroing) {
2801 // a newly allocated object is already all-zeroes everywhere
2802 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
2803 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
2804 assert(!phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == NULL, "storing null to inline type array is forbidden");
2805 result = mem;
2806 }
2807
2808 if (result == this && phase->type(val)->is_zero_type()) {
2809 // the store may also apply to zero-bits in an earlier object
2810 Node* prev_mem = find_previous_store(phase);
2811 // Steps (a), (b): Walk past independent stores to find an exact match.
2812 if (prev_mem != NULL) {
2813 Node* prev_val = can_see_stored_value(prev_mem, phase);
2814 if (prev_val != NULL && prev_val == val) {
2815 // prev_val and val might differ by a cast; it would be good
2816 // to keep the more informative of the two.
2817 result = mem;
2818 }
2819 }
2820 }
2821 }
2822
2823 PhaseIterGVN* igvn = phase->is_IterGVN();
2824 if (result != this && igvn != NULL) {
2825 MemBarNode* trailing = trailing_membar();
2826 if (trailing != NULL) {
2827 #ifdef ASSERT
2828 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2973 Node* StoreCMNode::Identity(PhaseGVN* phase) {
2974 // No need to card mark when storing a null ptr
2975 Node* my_store = in(MemNode::OopStore);
2976 if (my_store->is_Store()) {
2977 const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
2978 if( t1 == TypePtr::NULL_PTR ) {
2979 return in(MemNode::Memory);
2980 }
2981 }
2982 return this;
2983 }
2984
2985 //=============================================================================
2986 //------------------------------Ideal---------------------------------------
2987 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
2988 Node* progress = StoreNode::Ideal(phase, can_reshape);
2989 if (progress != NULL) return progress;
2990
2991 Node* my_store = in(MemNode::OopStore);
2992 if (my_store->is_MergeMem()) {
2993 if (oop_alias_idx() != phase->C->get_alias_index(TypeAryPtr::INLINES) ||
2994 phase->C->flattened_accesses_share_alias()) {
2995 // The alias that was recorded is no longer accurate enough.
2996 Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
2997 set_req_X(MemNode::OopStore, mem, phase);
2998 return this;
2999 }
3000 }
3001
3002 return NULL;
3003 }
3004
3005 //------------------------------Value-----------------------------------------
3006 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
3007 // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
3008 // If extra input is TOP ==> the result is TOP
3009 const Type* t = phase->type(in(MemNode::OopStore));
3010 if (t == Type::TOP) {
3011 return Type::TOP;
3012 }
3013 return StoreNode::Value(phase);
3014 }
3015
3016
3017 //=============================================================================
3018 //----------------------------------SCMemProjNode------------------------------
3019 const Type* SCMemProjNode::Value(PhaseGVN* phase) const
3138 // Clearing a short array is faster with stores
3139 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3140 // Already know this is a large node, do not try to ideal it
3141 if (_is_large) return NULL;
3142
3143 const int unit = BytesPerLong;
3144 const TypeX* t = phase->type(in(2))->isa_intptr_t();
3145 if (!t) return NULL;
3146 if (!t->is_con()) return NULL;
3147 intptr_t raw_count = t->get_con();
3148 intptr_t size = raw_count;
3149 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3150 // Clearing nothing uses the Identity call.
3151 // Negative clears are possible on dead ClearArrays
3152 // (see jck test stmt114.stmt11402.val).
3153 if (size <= 0 || size % unit != 0) return NULL;
3154 intptr_t count = size / unit;
3155 // Length too long; communicate this to matchers and assemblers.
3156 // Assemblers are responsible to produce fast hardware clears for it.
3157 if (size > InitArrayShortSize) {
3158 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3159 } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3160 return NULL;
3161 }
3162 if (!IdealizeClearArrayNode) return NULL;
3163 Node *mem = in(1);
3164 if( phase->type(mem)==Type::TOP ) return NULL;
3165 Node *adr = in(3);
3166 const Type* at = phase->type(adr);
3167 if( at==Type::TOP ) return NULL;
3168 const TypePtr* atp = at->isa_ptr();
3169 // adjust atp to be the correct array element address type
3170 if (atp == NULL) atp = TypePtr::BOTTOM;
3171 else atp = atp->add_offset(Type::OffsetBot);
3172 // Get base for derived pointer purposes
3173 if( adr->Opcode() != Op_AddP ) Unimplemented();
3174 Node *base = adr->in(1);
3175
3176 Node *val = in(4);
3177 Node *off = phase->MakeConX(BytesPerLong);
3178 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3179 count--;
3180 while( count-- ) {
3181 mem = phase->transform(mem);
3182 adr = phase->transform(new AddPNode(base,adr,off));
3183 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3184 }
3185 return mem;
3186 }
3187
3188 //----------------------------step_through----------------------------------
3189 // Return allocation input memory edge if it is different instance
3190 // or itself if it is the one we are looking for.
3191 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
3192 Node* n = *np;
3193 assert(n->is_ClearArray(), "sanity");
3194 intptr_t offset;
3195 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3196 // This method is called only before Allocate nodes are expanded
3197 // during macro nodes expansion. Before that ClearArray nodes are
3198 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3199 // Allocate nodes are expanded) which follows allocations.
3200 assert(alloc != NULL, "should have allocation");
3201 if (alloc->_idx == instance_id) {
3202 // Can not bypass initialization of the instance we are looking for.
3203 return false;
3204 }
3205 // Otherwise skip it.
3206 InitializeNode* init = alloc->initialization();
3207 if (init != NULL)
3208 *np = init->in(TypeFunc::Memory);
3209 else
3210 *np = alloc->in(TypeFunc::Memory);
3211 return true;
3212 }
3213
3214 //----------------------------clear_memory-------------------------------------
3215 // Generate code to initialize object storage to zero.
3216 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3217 Node* val,
3218 Node* raw_val,
3219 intptr_t start_offset,
3220 Node* end_offset,
3221 PhaseGVN* phase) {
3222 intptr_t offset = start_offset;
3223
3224 int unit = BytesPerLong;
3225 if ((offset % unit) != 0) {
3226 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3227 adr = phase->transform(adr);
3228 const TypePtr* atp = TypeRawPtr::BOTTOM;
3229 if (val != NULL) {
3230 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3231 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3232 } else {
3233 assert(raw_val == NULL, "val may not be null");
3234 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3235 }
3236 mem = phase->transform(mem);
3237 offset += BytesPerInt;
3238 }
3239 assert((offset % unit) == 0, "");
3240
3241 // Initialize the remaining stuff, if any, with a ClearArray.
3242 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3243 }
3244
3245 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3246 Node* raw_val,
3247 Node* start_offset,
3248 Node* end_offset,
3249 PhaseGVN* phase) {
3250 if (start_offset == end_offset) {
3251 // nothing to do
3252 return mem;
3253 }
3254
3255 int unit = BytesPerLong;
3256 Node* zbase = start_offset;
3257 Node* zend = end_offset;
3258
3259 // Scale to the unit required by the CPU:
3260 if (!Matcher::init_array_count_is_in_bytes) {
3261 Node* shift = phase->intcon(exact_log2(unit));
3262 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3263 zend = phase->transform(new URShiftXNode(zend, shift) );
3264 }
3265
3266 // Bulk clear double-words
3267 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3268 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3269 if (raw_val == NULL) {
3270 raw_val = phase->MakeConX(0);
3271 }
3272 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3273 return phase->transform(mem);
3274 }
3275
3276 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3277 Node* val,
3278 Node* raw_val,
3279 intptr_t start_offset,
3280 intptr_t end_offset,
3281 PhaseGVN* phase) {
3282 if (start_offset == end_offset) {
3283 // nothing to do
3284 return mem;
3285 }
3286
3287 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3288 intptr_t done_offset = end_offset;
3289 if ((done_offset % BytesPerLong) != 0) {
3290 done_offset -= BytesPerInt;
3291 }
3292 if (done_offset > start_offset) {
3293 mem = clear_memory(ctl, mem, dest, val, raw_val,
3294 start_offset, phase->MakeConX(done_offset), phase);
3295 }
3296 if (done_offset < end_offset) { // emit the final 32-bit store
3297 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3298 adr = phase->transform(adr);
3299 const TypePtr* atp = TypeRawPtr::BOTTOM;
3300 if (val != NULL) {
3301 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3302 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3303 } else {
3304 assert(raw_val == NULL, "val may not be null");
3305 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3306 }
3307 mem = phase->transform(mem);
3308 done_offset += BytesPerInt;
3309 }
3310 assert(done_offset == end_offset, "");
3311 return mem;
3312 }
3313
3314 //=============================================================================
3315 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3316 : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3317 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3318 #ifdef ASSERT
3319 , _pair_idx(0)
3320 #endif
3321 {
3322 init_class_id(Class_MemBar);
3323 Node* top = C->top();
3324 init_req(TypeFunc::I_O,top);
3325 init_req(TypeFunc::FramePtr,top);
3326 init_req(TypeFunc::ReturnAdr,top);
3432 PhaseIterGVN* igvn = phase->is_IterGVN();
3433 remove(igvn);
3434 // Must return either the original node (now dead) or a new node
3435 // (Do not return a top here, since that would break the uniqueness of top.)
3436 return new ConINode(TypeInt::ZERO);
3437 }
3438 }
3439 return progress ? this : NULL;
3440 }
3441
3442 //------------------------------Value------------------------------------------
3443 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3444 if( !in(0) ) return Type::TOP;
3445 if( phase->type(in(0)) == Type::TOP )
3446 return Type::TOP;
3447 return TypeTuple::MEMBAR;
3448 }
3449
3450 //------------------------------match------------------------------------------
3451 // Construct projections for memory.
3452 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
3453 switch (proj->_con) {
3454 case TypeFunc::Control:
3455 case TypeFunc::Memory:
3456 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3457 }
3458 ShouldNotReachHere();
3459 return NULL;
3460 }
3461
3462 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3463 trailing->_kind = TrailingStore;
3464 leading->_kind = LeadingStore;
3465 #ifdef ASSERT
3466 trailing->_pair_idx = leading->_idx;
3467 leading->_pair_idx = leading->_idx;
3468 #endif
3469 }
3470
3471 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3472 trailing->_kind = TrailingLoadStore;
3739 return (req() > RawStores);
3740 }
3741
3742 void InitializeNode::set_complete(PhaseGVN* phase) {
3743 assert(!is_complete(), "caller responsibility");
3744 _is_complete = Complete;
3745
3746 // After this node is complete, it contains a bunch of
3747 // raw-memory initializations. There is no need for
3748 // it to have anything to do with non-raw memory effects.
3749 // Therefore, tell all non-raw users to re-optimize themselves,
3750 // after skipping the memory effects of this initialization.
3751 PhaseIterGVN* igvn = phase->is_IterGVN();
3752 if (igvn) igvn->add_users_to_worklist(this);
3753 }
3754
3755 // convenience function
3756 // return false if the init contains any stores already
3757 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3758 InitializeNode* init = initialization();
3759 if (init == NULL || init->is_complete()) {
3760 return false;
3761 }
3762 init->remove_extra_zeroes();
3763 // for now, if this allocation has already collected any inits, bail:
3764 if (init->is_non_zero()) return false;
3765 init->set_complete(phase);
3766 return true;
3767 }
3768
3769 void InitializeNode::remove_extra_zeroes() {
3770 if (req() == RawStores) return;
3771 Node* zmem = zero_memory();
3772 uint fill = RawStores;
3773 for (uint i = fill; i < req(); i++) {
3774 Node* n = in(i);
3775 if (n->is_top() || n == zmem) continue; // skip
3776 if (fill < i) set_req(fill, n); // compact
3777 ++fill;
3778 }
3779 // delete any empty spaces created:
3780 while (fill < req()) {
3781 del_req(fill);
3919 // store node that we'd like to capture. We need to check
3920 // the uses of the MergeMemNode.
3921 mems.push(n);
3922 }
3923 } else if (n->is_Mem()) {
3924 Node* other_adr = n->in(MemNode::Address);
3925 if (other_adr == adr) {
3926 failed = true;
3927 break;
3928 } else {
3929 const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3930 if (other_t_adr != NULL) {
3931 int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3932 if (other_alias_idx == alias_idx) {
3933 // A load from the same memory slice as the store right
3934 // after the InitializeNode. We check the control of the
3935 // object/array that is loaded from. If it's the same as
3936 // the store control then we cannot capture the store.
3937 assert(!n->is_Store(), "2 stores to same slice on same control?");
3938 Node* base = other_adr;
3939 if (base->is_Phi()) {
3940 // In rare case, base may be a PhiNode and it may read
3941 // the same memory slice between InitializeNode and store.
3942 failed = true;
3943 break;
3944 }
3945 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3946 base = base->in(AddPNode::Base);
3947 if (base != NULL) {
3948 base = base->uncast();
3949 if (base->is_Proj() && base->in(0) == alloc) {
3950 failed = true;
3951 break;
3952 }
3953 }
3954 }
3955 }
3956 }
3957 } else {
3958 failed = true;
3959 break;
3960 }
3961 }
3962 }
3963 }
3964 if (failed) {
4509 // z's_done 12 16 16 16 12 16 12
4510 // z's_needed 12 16 16 16 16 16 16
4511 // zsize 0 0 0 0 4 0 4
4512 if (next_full_store < 0) {
4513 // Conservative tack: Zero to end of current word.
4514 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4515 } else {
4516 // Zero to beginning of next fully initialized word.
4517 // Or, don't zero at all, if we are already in that word.
4518 assert(next_full_store >= zeroes_needed, "must go forward");
4519 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4520 zeroes_needed = next_full_store;
4521 }
4522 }
4523
4524 if (zeroes_needed > zeroes_done) {
4525 intptr_t zsize = zeroes_needed - zeroes_done;
4526 // Do some incremental zeroing on rawmem, in parallel with inits.
4527 zeroes_done = align_down(zeroes_done, BytesPerInt);
4528 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4529 allocation()->in(AllocateNode::DefaultValue),
4530 allocation()->in(AllocateNode::RawDefaultValue),
4531 zeroes_done, zeroes_needed,
4532 phase);
4533 zeroes_done = zeroes_needed;
4534 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4535 do_zeroing = false; // leave the hole, next time
4536 }
4537 }
4538
4539 // Collect the store and move on:
4540 phase->replace_input_of(st, MemNode::Memory, inits);
4541 inits = st; // put it on the linearized chain
4542 set_req(i, zmem); // unhook from previous position
4543
4544 if (zeroes_done == st_off)
4545 zeroes_done = next_init_off;
4546
4547 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4548
4549 #ifdef ASSERT
4550 // Various order invariants. Weaker than stores_are_sane because
4570 remove_extra_zeroes(); // clear out all the zmems left over
4571 add_req(inits);
4572
4573 if (!(UseTLAB && ZeroTLAB)) {
4574 // If anything remains to be zeroed, zero it all now.
4575 zeroes_done = align_down(zeroes_done, BytesPerInt);
4576 // if it is the last unused 4 bytes of an instance, forget about it
4577 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4578 if (zeroes_done + BytesPerLong >= size_limit) {
4579 AllocateNode* alloc = allocation();
4580 assert(alloc != NULL, "must be present");
4581 if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4582 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4583 ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4584 if (zeroes_done == k->layout_helper())
4585 zeroes_done = size_limit;
4586 }
4587 }
4588 if (zeroes_done < size_limit) {
4589 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4590 allocation()->in(AllocateNode::DefaultValue),
4591 allocation()->in(AllocateNode::RawDefaultValue),
4592 zeroes_done, size_in_bytes, phase);
4593 }
4594 }
4595
4596 set_complete(phase);
4597 return rawmem;
4598 }
4599
4600
4601 #ifdef ASSERT
4602 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4603 if (is_complete())
4604 return true; // stores could be anything at this point
4605 assert(allocation() != NULL, "must be present");
4606 intptr_t last_off = allocation()->minimum_header_size();
4607 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4608 Node* st = in(i);
4609 intptr_t st_off = get_store_offset(st, phase);
4610 if (st_off < 0) continue; // ignore dead garbage
4611 if (last_off > st_off) {
|