< prev index next >

src/hotspot/share/opto/memnode.cpp

Print this page

   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"

  26 #include "classfile/javaClasses.hpp"

  27 #include "compiler/compileLog.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/c2/barrierSetC2.hpp"
  30 #include "gc/shared/tlab_globals.hpp"
  31 #include "memory/allocation.inline.hpp"
  32 #include "memory/resourceArea.hpp"
  33 #include "oops/objArrayKlass.hpp"
  34 #include "opto/addnode.hpp"
  35 #include "opto/arraycopynode.hpp"
  36 #include "opto/cfgnode.hpp"
  37 #include "opto/regalloc.hpp"
  38 #include "opto/compile.hpp"
  39 #include "opto/connode.hpp"
  40 #include "opto/convertnode.hpp"

  41 #include "opto/loopnode.hpp"
  42 #include "opto/machnode.hpp"
  43 #include "opto/matcher.hpp"
  44 #include "opto/memnode.hpp"
  45 #include "opto/mulnode.hpp"
  46 #include "opto/narrowptrnode.hpp"
  47 #include "opto/phaseX.hpp"
  48 #include "opto/regmask.hpp"
  49 #include "opto/rootnode.hpp"
  50 #include "opto/vectornode.hpp"
  51 #include "utilities/align.hpp"
  52 #include "utilities/copy.hpp"
  53 #include "utilities/macros.hpp"
  54 #include "utilities/powerOfTwo.hpp"
  55 #include "utilities/vmError.hpp"
  56 
  57 // Portions of code courtesy of Clifford Click
  58 
  59 // Optimization - Graph Style
  60 

 223       // clone the Phi with our address type
 224       result = mphi->split_out_instance(t_adr, igvn);
 225     } else {
 226       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 227     }
 228   }
 229   return result;
 230 }
 231 
 232 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 233   uint alias_idx = phase->C->get_alias_index(tp);
 234   Node *mem = mmem;
 235 #ifdef ASSERT
 236   {
 237     // Check that current type is consistent with the alias index used during graph construction
 238     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 239     bool consistent =  adr_check == NULL || adr_check->empty() ||
 240                        phase->C->must_alias(adr_check, alias_idx );
 241     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 242     if( !consistent && adr_check != NULL && !adr_check->empty() &&
 243                tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 244         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 245         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 246           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 247           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 248       // don't assert if it is dead code.
 249       consistent = true;
 250     }
 251     if( !consistent ) {
 252       st->print("alias_idx==%d, adr_check==", alias_idx);
 253       if( adr_check == NULL ) {
 254         st->print("NULL");
 255       } else {
 256         adr_check->dump();
 257       }
 258       st->cr();
 259       print_alias_types();
 260       assert(consistent, "adr_check must match alias idx");
 261     }
 262   }
 263 #endif

 881          "use LoadKlassNode instead");
 882   assert(!(adr_type->isa_aryptr() &&
 883            adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
 884          "use LoadRangeNode instead");
 885   // Check control edge of raw loads
 886   assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
 887           // oop will be recorded in oop map if load crosses safepoint
 888           rt->isa_oopptr() || is_immutable_value(adr),
 889           "raw memory operations should have control edge");
 890   LoadNode* load = NULL;
 891   switch (bt) {
 892   case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 893   case T_BYTE:    load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 894   case T_INT:     load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 895   case T_CHAR:    load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 896   case T_SHORT:   load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 897   case T_LONG:    load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
 898   case T_FLOAT:   load = new LoadFNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
 899   case T_DOUBLE:  load = new LoadDNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency, require_atomic_access); break;
 900   case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(),  mo, control_dependency); break;

 901   case T_OBJECT:
 902 #ifdef _LP64
 903     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 904       load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
 905     } else
 906 #endif
 907     {
 908       assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
 909       load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
 910     }
 911     break;
 912   default:
 913     ShouldNotReachHere();
 914     break;
 915   }
 916   assert(load != NULL, "LoadNode should have been created");
 917   if (unaligned) {
 918     load->set_unaligned_access();
 919   }
 920   if (mismatched) {

 976       assert(ld_alloc != NULL, "need an alloc");
 977       assert(addp->is_AddP(), "address must be addp");
 978       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 979       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
 980       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
 981       addp->set_req(AddPNode::Base, src);
 982       addp->set_req(AddPNode::Address, src);
 983     } else {
 984       assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
 985              ac->as_ArrayCopy()->is_copyof_validated() ||
 986              ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
 987       assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
 988       addp->set_req(AddPNode::Base, src);
 989       addp->set_req(AddPNode::Address, src);
 990 
 991       const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
 992       BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
 993       if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
 994 
 995       uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
 996       uint shift  = exact_log2(type2aelembytes(ary_elem));
 997 
 998       Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
 999 #ifdef _LP64
1000       diff = phase->transform(new ConvI2LNode(diff));
1001 #endif
1002       diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1003 
1004       Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1005       addp->set_req(AddPNode::Offset, offset);
1006     }
1007     addp = phase->transform(addp);
1008 #ifdef ASSERT
1009     const TypePtr* adr_type = phase->type(addp)->is_ptr();
1010     ld->_adr_type = adr_type;
1011 #endif
1012     ld->set_req(MemNode::Address, addp);
1013     ld->set_req(0, ctl);
1014     ld->set_req(MemNode::Memory, mem);
1015     // load depends on the tests that validate the arraycopy
1016     ld->_control_dependency = UnknownControl;

1097         // Same base, same offset.
1098         // Possible improvement for arrays: check index value instead of absolute offset.
1099 
1100         // At this point we have proven something like this setup:
1101         //   B = << base >>
1102         //   L =  LoadQ(AddP(Check/CastPP(B), #Off))
1103         //   S = StoreQ(AddP(             B , #Off), V)
1104         // (Actually, we haven't yet proven the Q's are the same.)
1105         // In other words, we are loading from a casted version of
1106         // the same pointer-and-offset that we stored to.
1107         // Casted version may carry a dependency and it is respected.
1108         // Thus, we are able to replace L by V.
1109       }
1110       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1111       if (store_Opcode() != st->Opcode()) {
1112         return NULL;
1113       }
1114       // LoadVector/StoreVector needs additional check to ensure the types match.
1115       if (st->is_StoreVector()) {
1116         const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
1117         const TypeVect* out_vt = as_LoadVector()->vect_type();
1118         if (in_vt != out_vt) {
1119           return NULL;
1120         }
1121       }
1122       return st->in(MemNode::ValueIn);
1123     }
1124 
1125     // A load from a freshly-created object always returns zero.
1126     // (This can happen after LoadNode::Ideal resets the load's memory input
1127     // to find_captured_store, which returned InitializeNode::zero_memory.)
1128     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1129         (st->in(0) == ld_alloc) &&
1130         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1131       // return a zero value for the load's basic type
1132       // (This is one of the few places where a generic PhaseTransform
1133       // can create new nodes.  Think of it as lazily manifesting
1134       // virtually pre-existing constants.)






1135       if (memory_type() != T_VOID) {
1136         if (ReduceBulkZeroing || find_array_copy_clone(phase, ld_alloc, in(MemNode::Memory)) == NULL) {
1137           // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1138           // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1139           // by the ArrayCopyNode.
1140           return phase->zerocon(memory_type());
1141         }
1142       } else {
1143         // TODO: materialize all-zero vector constant
1144         assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1145       }
1146     }
1147 
1148     // A load from an initialization barrier can match a captured store.
1149     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1150       InitializeNode* init = st->in(0)->as_Initialize();
1151       AllocateNode* alloc = init->allocation();
1152       if ((alloc != NULL) && (alloc == ld_alloc)) {
1153         // examine a captured store value
1154         st = init->find_captured_store(ld_off, memory_size(), phase);

1182 //----------------------is_instance_field_load_with_local_phi------------------
1183 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1184   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1185       in(Address)->is_AddP() ) {
1186     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1187     // Only instances and boxed values.
1188     if( t_oop != NULL &&
1189         (t_oop->is_ptr_to_boxed_value() ||
1190          t_oop->is_known_instance_field()) &&
1191         t_oop->offset() != Type::OffsetBot &&
1192         t_oop->offset() != Type::OffsetTop) {
1193       return true;
1194     }
1195   }
1196   return false;
1197 }
1198 
1199 //------------------------------Identity---------------------------------------
1200 // Loads are identity if previous store is to same address
1201 Node* LoadNode::Identity(PhaseGVN* phase) {



























1202   // If the previous store-maker is the right kind of Store, and the store is
1203   // to the same address, then we are equal to the value stored.
1204   Node* mem = in(Memory);
1205   Node* value = can_see_stored_value(mem, phase);
1206   if( value ) {
1207     // byte, short & char stores truncate naturally.
1208     // A load has to load the truncated value which requires
1209     // some sort of masking operation and that requires an
1210     // Ideal call instead of an Identity call.
1211     if (memory_size() < BytesPerInt) {
1212       // If the input to the store does not fit with the load's result type,
1213       // it must be truncated via an Ideal call.
1214       if (!phase->type(value)->higher_equal(phase->type(this)))
1215         return this;
1216     }
1217     // (This works even when value is a Con, but LoadNode::Value
1218     // usually runs first, producing the singleton type of the Con.)
1219     return value;
1220   }
1221 

1891       }
1892     }
1893 
1894     // Don't do this for integer types. There is only potential profit if
1895     // the element type t is lower than _type; that is, for int types, if _type is
1896     // more restrictive than t.  This only happens here if one is short and the other
1897     // char (both 16 bits), and in those cases we've made an intentional decision
1898     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1899     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1900     //
1901     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1902     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
1903     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1904     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
1905     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1906     // In fact, that could have been the original type of p1, and p1 could have
1907     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1908     // expression (LShiftL quux 3) independently optimized to the constant 8.
1909     if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1910         && (_type->isa_vect() == NULL)

1911         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1912       // t might actually be lower than _type, if _type is a unique
1913       // concrete subclass of abstract class t.
1914       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
1915         const Type* jt = t->join_speculative(_type);
1916         // In any case, do not allow the join, per se, to empty out the type.
1917         if (jt->empty() && !t->empty()) {
1918           // This can happen if a interface-typed array narrows to a class type.
1919           jt = _type;
1920         }
1921 #ifdef ASSERT
1922         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1923           // The pointers in the autobox arrays are always non-null
1924           Node* base = adr->in(AddPNode::Base);
1925           if ((base != NULL) && base->is_DecodeN()) {
1926             // Get LoadN node which loads IntegerCache.cache field
1927             base = base->in(1);
1928           }
1929           if ((base != NULL) && base->is_Con()) {
1930             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1931             if ((base_type != NULL) && base_type->is_autobox_cache()) {
1932               // It could be narrow oop
1933               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1934             }
1935           }
1936         }
1937 #endif
1938         return jt;
1939       }
1940     }
1941   } else if (tp->base() == Type::InstPtr) {
1942     assert( off != Type::OffsetBot ||
1943             // arrays can be cast to Objects
1944             !tp->isa_instptr() ||
1945             tp->is_instptr()->instance_klass()->is_java_lang_Object() ||


1946             // unsafe field access may not have a constant offset
1947             C->has_unsafe_access(),
1948             "Field accesses must be precise" );
1949     // For oop loads, we expect the _type to be precise.
1950 
1951     // Optimize loads from constant fields.
1952     const TypeInstPtr* tinst = tp->is_instptr();



1953     ciObject* const_oop = tinst->const_oop();
1954     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1955       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());


















1956       if (con_type != NULL) {
1957         return con_type;
1958       }
1959     }
1960   } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
1961     assert(off != Type::OffsetBot ||
1962             !tp->isa_instklassptr() ||
1963            // arrays can be cast to Objects
1964            tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
1965            // also allow array-loading from the primary supertype
1966            // array during subtype checks
1967            Opcode() == Op_LoadKlass,
1968            "Field accesses must be precise");
1969     // For klass/static loads, we expect the _type to be precise
1970   } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
1971     /* With mirrors being an indirect in the Klass*
1972      * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
1973      * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
1974      *
1975      * So check the type and klass of the node before the LoadP.
1976      */
1977     Node* adr2 = adr->in(MemNode::Address);
1978     const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
1979     if (tkls != NULL && !StressReflectiveCode) {
1980       if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
1981         ciKlass* klass = tkls->exact_klass();
1982         assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1983         assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1984         return TypeInstPtr::make(klass->java_mirror());
















1985       }
1986     }
1987   }
1988 
1989   const TypeKlassPtr *tkls = tp->isa_klassptr();
1990   if (tkls != NULL && !StressReflectiveCode) {
1991     if (tkls->is_loaded() && tkls->klass_is_exact()) {
1992       ciKlass* klass = tkls->exact_klass();
1993       // We are loading a field from a Klass metaobject whose identity
1994       // is known at compile time (the type is "exact" or "precise").
1995       // Check for fields we know are maintained as constants by the VM.
1996       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
1997         // The field is Klass::_super_check_offset.  Return its (constant) value.
1998         // (Folds up type checking code.)
1999         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2000         return TypeInt::make(klass->super_check_offset());
2001       }
2002       // Compute index into primary_supers array
2003       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2004       // Check for overflowing; use unsigned compare to handle the negative case.

2070   if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2071     Node* value = can_see_stored_value(mem,phase);
2072     if (value != NULL && value->is_Con()) {
2073       assert(value->bottom_type()->higher_equal(_type),"sanity");
2074       return value->bottom_type();
2075     }
2076   }
2077 
2078   bool is_vect = (_type->isa_vect() != NULL);
2079   if (is_instance && !is_vect) {
2080     // If we have an instance type and our memory input is the
2081     // programs's initial memory state, there is no matching store,
2082     // so just return a zero of the appropriate type -
2083     // except if it is vectorized - then we have no zero constant.
2084     Node *mem = in(MemNode::Memory);
2085     if (mem->is_Parm() && mem->in(0)->is_Start()) {
2086       assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2087       return Type::get_zero_type(_type->basic_type());
2088     }
2089   }
2090 
2091   Node* alloc = is_new_object_mark_load(phase);
2092   if (alloc != NULL) {
2093     return TypeX::make(markWord::prototype().value());









2094   }
2095 
2096   return _type;
2097 }
2098 
2099 //------------------------------match_edge-------------------------------------
2100 // Do we Match on this edge index or not?  Match only the address.
2101 uint LoadNode::match_edge(uint idx) const {
2102   return idx == MemNode::Address;
2103 }
2104 
2105 //--------------------------LoadBNode::Ideal--------------------------------------
2106 //
2107 //  If the previous store is to the same address as this load,
2108 //  and the value stored was larger than a byte, replace this load
2109 //  with the value stored truncated to a byte.  If no truncation is
2110 //  needed, the replacement is done in LoadNode::Identity().
2111 //
2112 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2113   Node* mem = in(MemNode::Memory);

2224   return LoadNode::Ideal(phase, can_reshape);
2225 }
2226 
2227 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2228   Node* mem = in(MemNode::Memory);
2229   Node* value = can_see_stored_value(mem,phase);
2230   if (value != NULL && value->is_Con() &&
2231       !value->bottom_type()->higher_equal(_type)) {
2232     // If the input to the store does not fit with the load's result type,
2233     // it must be truncated. We can't delay until Ideal call since
2234     // a singleton Value is needed for split_thru_phi optimization.
2235     int con = value->get_int();
2236     return TypeInt::make((con << 16) >> 16);
2237   }
2238   return LoadNode::Value(phase);
2239 }
2240 
2241 //=============================================================================
2242 //----------------------------LoadKlassNode::make------------------------------
2243 // Polymorphic factory method:
2244 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {

2245   // sanity check the alias category against the created node type
2246   const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2247   assert(adr_type != NULL, "expecting TypeKlassPtr");
2248 #ifdef _LP64
2249   if (adr_type->is_ptr_to_narrowklass()) {
2250     assert(UseCompressedClassPointers, "no compressed klasses");
2251     Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2252     return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2253   }
2254 #endif
2255   assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2256   return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2257 }
2258 
2259 //------------------------------Value------------------------------------------
2260 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2261   return klass_value_common(phase);
2262 }
2263 
2264 // In most cases, LoadKlassNode does not have the control input set. If the control

2271   // Either input is TOP ==> the result is TOP
2272   const Type *t1 = phase->type( in(MemNode::Memory) );
2273   if (t1 == Type::TOP)  return Type::TOP;
2274   Node *adr = in(MemNode::Address);
2275   const Type *t2 = phase->type( adr );
2276   if (t2 == Type::TOP)  return Type::TOP;
2277   const TypePtr *tp = t2->is_ptr();
2278   if (TypePtr::above_centerline(tp->ptr()) ||
2279       tp->ptr() == TypePtr::Null)  return Type::TOP;
2280 
2281   // Return a more precise klass, if possible
2282   const TypeInstPtr *tinst = tp->isa_instptr();
2283   if (tinst != NULL) {
2284     ciInstanceKlass* ik = tinst->instance_klass();
2285     int offset = tinst->offset();
2286     if (ik == phase->C->env()->Class_klass()
2287         && (offset == java_lang_Class::klass_offset() ||
2288             offset == java_lang_Class::array_klass_offset())) {
2289       // We are loading a special hidden field from a Class mirror object,
2290       // the field which points to the VM's Klass metaobject.
2291       ciType* t = tinst->java_mirror_type();

2292       // java_mirror_type returns non-null for compile-time Class constants.
2293       if (t != NULL) {
2294         // constant oop => constant klass
2295         if (offset == java_lang_Class::array_klass_offset()) {
2296           if (t->is_void()) {
2297             // We cannot create a void array.  Since void is a primitive type return null
2298             // klass.  Users of this result need to do a null check on the returned klass.
2299             return TypePtr::NULL_PTR;
2300           }
2301           return TypeKlassPtr::make(ciArrayKlass::make(t));
2302         }
2303         if (!t->is_klass()) {
2304           // a primitive Class (e.g., int.class) has NULL for a klass field
2305           return TypePtr::NULL_PTR;
2306         }
2307         // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2308         return TypeKlassPtr::make(t->as_klass());
2309       }
2310       // non-constant mirror, so we can't tell what's going on
2311     }
2312     if (!tinst->is_loaded())
2313       return _type;             // Bail out if not loaded
2314     if (offset == oopDesc::klass_offset_in_bytes()) {
2315       return tinst->as_klass_type(true);
2316     }
2317   }
2318 
2319   // Check for loading klass from an array
2320   const TypeAryPtr *tary = tp->isa_aryptr();
2321   if (tary != NULL && tary->elem() != Type::BOTTOM &&
2322       tary->offset() == oopDesc::klass_offset_in_bytes()) {
2323     return tary->as_klass_type(true);
2324   }
2325 
2326   // Check for loading klass from an array klass
2327   const TypeKlassPtr *tkls = tp->isa_klassptr();
2328   if (tkls != NULL && !StressReflectiveCode) {
2329     if (!tkls->is_loaded())
2330      return _type;             // Bail out if not loaded
2331     if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2332         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2333       // // Always returning precise element type is incorrect,
2334       // // e.g., element type could be object and array may contain strings
2335       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2336 
2337       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2338       // according to the element type's subclassing.
2339       return tkls->is_aryklassptr()->elem();
2340     }

2526 //=============================================================================
2527 //---------------------------StoreNode::make-----------------------------------
2528 // Polymorphic factory method:
2529 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2530   assert((mo == unordered || mo == release), "unexpected");
2531   Compile* C = gvn.C;
2532   assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2533          ctl != NULL, "raw memory operations should have control edge");
2534 
2535   switch (bt) {
2536   case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2537   case T_BYTE:    return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2538   case T_INT:     return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2539   case T_CHAR:
2540   case T_SHORT:   return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2541   case T_LONG:    return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2542   case T_FLOAT:   return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2543   case T_DOUBLE:  return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2544   case T_METADATA:
2545   case T_ADDRESS:

2546   case T_OBJECT:
2547 #ifdef _LP64
2548     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2549       val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2550       return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2551     } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2552                (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2553                 adr->bottom_type()->isa_rawptr())) {
2554       val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2555       return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2556     }
2557 #endif
2558     {
2559       return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2560     }
2561   default:
2562     ShouldNotReachHere();
2563     return (StoreNode*)NULL;
2564   }
2565 }

2576 
2577   // Since they are not commoned, do not hash them:
2578   return NO_HASH;
2579 }
2580 
2581 //------------------------------Ideal------------------------------------------
2582 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2583 // When a store immediately follows a relevant allocation/initialization,
2584 // try to capture it into the initialization, or hoist it above.
2585 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2586   Node* p = MemNode::Ideal_common(phase, can_reshape);
2587   if (p)  return (p == NodeSentinel) ? NULL : p;
2588 
2589   Node* mem     = in(MemNode::Memory);
2590   Node* address = in(MemNode::Address);
2591   Node* value   = in(MemNode::ValueIn);
2592   // Back-to-back stores to same address?  Fold em up.  Generally
2593   // unsafe if I have intervening uses...  Also disallowed for StoreCM
2594   // since they must follow each StoreP operation.  Redundant StoreCMs
2595   // are eliminated just before matching in final_graph_reshape.
2596   {
2597     Node* st = mem;
2598     // If Store 'st' has more than one use, we cannot fold 'st' away.
2599     // For example, 'st' might be the final state at a conditional
2600     // return.  Or, 'st' might be used by some node which is live at
2601     // the same time 'st' is live, which might be unschedulable.  So,
2602     // require exactly ONE user until such time as we clone 'mem' for
2603     // each of 'mem's uses (thus making the exactly-1-user-rule hold
2604     // true).
2605     while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2606       // Looking at a dead closed cycle of memory?
2607       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2608       assert(Opcode() == st->Opcode() ||
2609              st->Opcode() == Op_StoreVector ||
2610              Opcode() == Op_StoreVector ||
2611              st->Opcode() == Op_StoreVectorScatter ||
2612              Opcode() == Op_StoreVectorScatter ||
2613              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2614              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2615              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy

2616              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2617              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2618 
2619       if (st->in(MemNode::Address)->eqv_uncast(address) &&
2620           st->as_Store()->memory_size() <= this->memory_size()) {
2621         Node* use = st->raw_out(0);
2622         if (phase->is_IterGVN()) {
2623           phase->is_IterGVN()->rehash_node_delayed(use);
2624         }
2625         // It's OK to do this in the parser, since DU info is always accurate,
2626         // and the parser always refers to nodes via SafePointNode maps.
2627         use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2628         return this;
2629       }
2630       st = st->in(MemNode::Memory);
2631     }
2632   }
2633 
2634 
2635   // Capture an unaliased, unconditional, simple store into an initializer.

2692   // Load then Store?  Then the Store is useless
2693   if (val->is_Load() &&
2694       val->in(MemNode::Address)->eqv_uncast(adr) &&
2695       val->in(MemNode::Memory )->eqv_uncast(mem) &&
2696       val->as_Load()->store_Opcode() == Opcode()) {
2697     result = mem;
2698   }
2699 
2700   // Two stores in a row of the same value?
2701   if (result == this &&
2702       mem->is_Store() &&
2703       mem->in(MemNode::Address)->eqv_uncast(adr) &&
2704       mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2705       mem->Opcode() == Opcode()) {
2706     result = mem;
2707   }
2708 
2709   // Store of zero anywhere into a freshly-allocated object?
2710   // Then the store is useless.
2711   // (It must already have been captured by the InitializeNode.)
2712   if (result == this &&
2713       ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2714     // a newly allocated object is already all-zeroes everywhere
2715     if (mem->is_Proj() && mem->in(0)->is_Allocate()) {


2716       result = mem;
2717     }
2718 
2719     if (result == this) {
2720       // the store may also apply to zero-bits in an earlier object
2721       Node* prev_mem = find_previous_store(phase);
2722       // Steps (a), (b):  Walk past independent stores to find an exact match.
2723       if (prev_mem != NULL) {
2724         Node* prev_val = can_see_stored_value(prev_mem, phase);
2725         if (prev_val != NULL && prev_val == val) {
2726           // prev_val and val might differ by a cast; it would be good
2727           // to keep the more informative of the two.
2728           result = mem;
2729         }
2730       }
2731     }
2732   }
2733 
2734   PhaseIterGVN* igvn = phase->is_IterGVN();
2735   if (result != this && igvn != NULL) {
2736     MemBarNode* trailing = trailing_membar();
2737     if (trailing != NULL) {
2738 #ifdef ASSERT
2739       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();

2884 Node* StoreCMNode::Identity(PhaseGVN* phase) {
2885   // No need to card mark when storing a null ptr
2886   Node* my_store = in(MemNode::OopStore);
2887   if (my_store->is_Store()) {
2888     const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
2889     if( t1 == TypePtr::NULL_PTR ) {
2890       return in(MemNode::Memory);
2891     }
2892   }
2893   return this;
2894 }
2895 
2896 //=============================================================================
2897 //------------------------------Ideal---------------------------------------
2898 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
2899   Node* progress = StoreNode::Ideal(phase, can_reshape);
2900   if (progress != NULL) return progress;
2901 
2902   Node* my_store = in(MemNode::OopStore);
2903   if (my_store->is_MergeMem()) {
2904     Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
2905     set_req_X(MemNode::OopStore, mem, phase);
2906     return this;




2907   }
2908 
2909   return NULL;
2910 }
2911 
2912 //------------------------------Value-----------------------------------------
2913 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
2914   // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
2915   // If extra input is TOP ==> the result is TOP
2916   const Type* t = phase->type(in(MemNode::OopStore));
2917   if (t == Type::TOP) {
2918     return Type::TOP;
2919   }
2920   return StoreNode::Value(phase);
2921 }
2922 
2923 
2924 //=============================================================================
2925 //----------------------------------SCMemProjNode------------------------------
2926 const Type* SCMemProjNode::Value(PhaseGVN* phase) const

3045 // Clearing a short array is faster with stores
3046 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3047   // Already know this is a large node, do not try to ideal it
3048   if (_is_large) return NULL;
3049 
3050   const int unit = BytesPerLong;
3051   const TypeX* t = phase->type(in(2))->isa_intptr_t();
3052   if (!t)  return NULL;
3053   if (!t->is_con())  return NULL;
3054   intptr_t raw_count = t->get_con();
3055   intptr_t size = raw_count;
3056   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3057   // Clearing nothing uses the Identity call.
3058   // Negative clears are possible on dead ClearArrays
3059   // (see jck test stmt114.stmt11402.val).
3060   if (size <= 0 || size % unit != 0)  return NULL;
3061   intptr_t count = size / unit;
3062   // Length too long; communicate this to matchers and assemblers.
3063   // Assemblers are responsible to produce fast hardware clears for it.
3064   if (size > InitArrayShortSize) {
3065     return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
3066   } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3067     return NULL;
3068   }
3069   if (!IdealizeClearArrayNode) return NULL;
3070   Node *mem = in(1);
3071   if( phase->type(mem)==Type::TOP ) return NULL;
3072   Node *adr = in(3);
3073   const Type* at = phase->type(adr);
3074   if( at==Type::TOP ) return NULL;
3075   const TypePtr* atp = at->isa_ptr();
3076   // adjust atp to be the correct array element address type
3077   if (atp == NULL)  atp = TypePtr::BOTTOM;
3078   else              atp = atp->add_offset(Type::OffsetBot);
3079   // Get base for derived pointer purposes
3080   if( adr->Opcode() != Op_AddP ) Unimplemented();
3081   Node *base = adr->in(1);
3082 
3083   Node *zero = phase->makecon(TypeLong::ZERO);
3084   Node *off  = phase->MakeConX(BytesPerLong);
3085   mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3086   count--;
3087   while( count-- ) {
3088     mem = phase->transform(mem);
3089     adr = phase->transform(new AddPNode(base,adr,off));
3090     mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
3091   }
3092   return mem;
3093 }
3094 
3095 //----------------------------step_through----------------------------------
3096 // Return allocation input memory edge if it is different instance
3097 // or itself if it is the one we are looking for.
3098 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
3099   Node* n = *np;
3100   assert(n->is_ClearArray(), "sanity");
3101   intptr_t offset;
3102   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3103   // This method is called only before Allocate nodes are expanded
3104   // during macro nodes expansion. Before that ClearArray nodes are
3105   // only generated in PhaseMacroExpand::generate_arraycopy() (before
3106   // Allocate nodes are expanded) which follows allocations.
3107   assert(alloc != NULL, "should have allocation");
3108   if (alloc->_idx == instance_id) {
3109     // Can not bypass initialization of the instance we are looking for.
3110     return false;
3111   }
3112   // Otherwise skip it.
3113   InitializeNode* init = alloc->initialization();
3114   if (init != NULL)
3115     *np = init->in(TypeFunc::Memory);
3116   else
3117     *np = alloc->in(TypeFunc::Memory);
3118   return true;
3119 }
3120 
3121 //----------------------------clear_memory-------------------------------------
3122 // Generate code to initialize object storage to zero.
3123 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


3124                                    intptr_t start_offset,
3125                                    Node* end_offset,
3126                                    PhaseGVN* phase) {
3127   intptr_t offset = start_offset;
3128 
3129   int unit = BytesPerLong;
3130   if ((offset % unit) != 0) {
3131     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3132     adr = phase->transform(adr);
3133     const TypePtr* atp = TypeRawPtr::BOTTOM;
3134     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






3135     mem = phase->transform(mem);
3136     offset += BytesPerInt;
3137   }
3138   assert((offset % unit) == 0, "");
3139 
3140   // Initialize the remaining stuff, if any, with a ClearArray.
3141   return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
3142 }
3143 
3144 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,

3145                                    Node* start_offset,
3146                                    Node* end_offset,
3147                                    PhaseGVN* phase) {
3148   if (start_offset == end_offset) {
3149     // nothing to do
3150     return mem;
3151   }
3152 
3153   int unit = BytesPerLong;
3154   Node* zbase = start_offset;
3155   Node* zend  = end_offset;
3156 
3157   // Scale to the unit required by the CPU:
3158   if (!Matcher::init_array_count_is_in_bytes) {
3159     Node* shift = phase->intcon(exact_log2(unit));
3160     zbase = phase->transform(new URShiftXNode(zbase, shift) );
3161     zend  = phase->transform(new URShiftXNode(zend,  shift) );
3162   }
3163 
3164   // Bulk clear double-words
3165   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3166   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3167   mem = new ClearArrayNode(ctl, mem, zsize, adr, false);



3168   return phase->transform(mem);
3169 }
3170 
3171 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


3172                                    intptr_t start_offset,
3173                                    intptr_t end_offset,
3174                                    PhaseGVN* phase) {
3175   if (start_offset == end_offset) {
3176     // nothing to do
3177     return mem;
3178   }
3179 
3180   assert((end_offset % BytesPerInt) == 0, "odd end offset");
3181   intptr_t done_offset = end_offset;
3182   if ((done_offset % BytesPerLong) != 0) {
3183     done_offset -= BytesPerInt;
3184   }
3185   if (done_offset > start_offset) {
3186     mem = clear_memory(ctl, mem, dest,
3187                        start_offset, phase->MakeConX(done_offset), phase);
3188   }
3189   if (done_offset < end_offset) { // emit the final 32-bit store
3190     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3191     adr = phase->transform(adr);
3192     const TypePtr* atp = TypeRawPtr::BOTTOM;
3193     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






3194     mem = phase->transform(mem);
3195     done_offset += BytesPerInt;
3196   }
3197   assert(done_offset == end_offset, "");
3198   return mem;
3199 }
3200 
3201 //=============================================================================
3202 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3203   : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3204     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3205 #ifdef ASSERT
3206   , _pair_idx(0)
3207 #endif
3208 {
3209   init_class_id(Class_MemBar);
3210   Node* top = C->top();
3211   init_req(TypeFunc::I_O,top);
3212   init_req(TypeFunc::FramePtr,top);
3213   init_req(TypeFunc::ReturnAdr,top);

3319       PhaseIterGVN* igvn = phase->is_IterGVN();
3320       remove(igvn);
3321       // Must return either the original node (now dead) or a new node
3322       // (Do not return a top here, since that would break the uniqueness of top.)
3323       return new ConINode(TypeInt::ZERO);
3324     }
3325   }
3326   return progress ? this : NULL;
3327 }
3328 
3329 //------------------------------Value------------------------------------------
3330 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3331   if( !in(0) ) return Type::TOP;
3332   if( phase->type(in(0)) == Type::TOP )
3333     return Type::TOP;
3334   return TypeTuple::MEMBAR;
3335 }
3336 
3337 //------------------------------match------------------------------------------
3338 // Construct projections for memory.
3339 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3340   switch (proj->_con) {
3341   case TypeFunc::Control:
3342   case TypeFunc::Memory:
3343     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3344   }
3345   ShouldNotReachHere();
3346   return NULL;
3347 }
3348 
3349 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3350   trailing->_kind = TrailingStore;
3351   leading->_kind = LeadingStore;
3352 #ifdef ASSERT
3353   trailing->_pair_idx = leading->_idx;
3354   leading->_pair_idx = leading->_idx;
3355 #endif
3356 }
3357 
3358 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3359   trailing->_kind = TrailingLoadStore;

3626   return (req() > RawStores);
3627 }
3628 
3629 void InitializeNode::set_complete(PhaseGVN* phase) {
3630   assert(!is_complete(), "caller responsibility");
3631   _is_complete = Complete;
3632 
3633   // After this node is complete, it contains a bunch of
3634   // raw-memory initializations.  There is no need for
3635   // it to have anything to do with non-raw memory effects.
3636   // Therefore, tell all non-raw users to re-optimize themselves,
3637   // after skipping the memory effects of this initialization.
3638   PhaseIterGVN* igvn = phase->is_IterGVN();
3639   if (igvn)  igvn->add_users_to_worklist(this);
3640 }
3641 
3642 // convenience function
3643 // return false if the init contains any stores already
3644 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3645   InitializeNode* init = initialization();
3646   if (init == NULL || init->is_complete())  return false;


3647   init->remove_extra_zeroes();
3648   // for now, if this allocation has already collected any inits, bail:
3649   if (init->is_non_zero())  return false;
3650   init->set_complete(phase);
3651   return true;
3652 }
3653 
3654 void InitializeNode::remove_extra_zeroes() {
3655   if (req() == RawStores)  return;
3656   Node* zmem = zero_memory();
3657   uint fill = RawStores;
3658   for (uint i = fill; i < req(); i++) {
3659     Node* n = in(i);
3660     if (n->is_top() || n == zmem)  continue;  // skip
3661     if (fill < i)  set_req(fill, n);          // compact
3662     ++fill;
3663   }
3664   // delete any empty spaces created:
3665   while (fill < req()) {
3666     del_req(fill);

3804             // store node that we'd like to capture. We need to check
3805             // the uses of the MergeMemNode.
3806             mems.push(n);
3807           }
3808         } else if (n->is_Mem()) {
3809           Node* other_adr = n->in(MemNode::Address);
3810           if (other_adr == adr) {
3811             failed = true;
3812             break;
3813           } else {
3814             const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3815             if (other_t_adr != NULL) {
3816               int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3817               if (other_alias_idx == alias_idx) {
3818                 // A load from the same memory slice as the store right
3819                 // after the InitializeNode. We check the control of the
3820                 // object/array that is loaded from. If it's the same as
3821                 // the store control then we cannot capture the store.
3822                 assert(!n->is_Store(), "2 stores to same slice on same control?");
3823                 Node* base = other_adr;






3824                 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3825                 base = base->in(AddPNode::Base);
3826                 if (base != NULL) {
3827                   base = base->uncast();
3828                   if (base->is_Proj() && base->in(0) == alloc) {
3829                     failed = true;
3830                     break;
3831                   }
3832                 }
3833               }
3834             }
3835           }
3836         } else {
3837           failed = true;
3838           break;
3839         }
3840       }
3841     }
3842   }
3843   if (failed) {

4388         //   z's_done      12  16  16  16    12  16    12
4389         //   z's_needed    12  16  16  16    16  16    16
4390         //   zsize          0   0   0   0     4   0     4
4391         if (next_full_store < 0) {
4392           // Conservative tack:  Zero to end of current word.
4393           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4394         } else {
4395           // Zero to beginning of next fully initialized word.
4396           // Or, don't zero at all, if we are already in that word.
4397           assert(next_full_store >= zeroes_needed, "must go forward");
4398           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4399           zeroes_needed = next_full_store;
4400         }
4401       }
4402 
4403       if (zeroes_needed > zeroes_done) {
4404         intptr_t zsize = zeroes_needed - zeroes_done;
4405         // Do some incremental zeroing on rawmem, in parallel with inits.
4406         zeroes_done = align_down(zeroes_done, BytesPerInt);
4407         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


4408                                               zeroes_done, zeroes_needed,
4409                                               phase);
4410         zeroes_done = zeroes_needed;
4411         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4412           do_zeroing = false;   // leave the hole, next time
4413       }
4414     }
4415 
4416     // Collect the store and move on:
4417     phase->replace_input_of(st, MemNode::Memory, inits);
4418     inits = st;                 // put it on the linearized chain
4419     set_req(i, zmem);           // unhook from previous position
4420 
4421     if (zeroes_done == st_off)
4422       zeroes_done = next_init_off;
4423 
4424     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4425 
4426     #ifdef ASSERT
4427     // Various order invariants.  Weaker than stores_are_sane because

4447   remove_extra_zeroes();        // clear out all the zmems left over
4448   add_req(inits);
4449 
4450   if (!(UseTLAB && ZeroTLAB)) {
4451     // If anything remains to be zeroed, zero it all now.
4452     zeroes_done = align_down(zeroes_done, BytesPerInt);
4453     // if it is the last unused 4 bytes of an instance, forget about it
4454     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4455     if (zeroes_done + BytesPerLong >= size_limit) {
4456       AllocateNode* alloc = allocation();
4457       assert(alloc != NULL, "must be present");
4458       if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4459         Node* klass_node = alloc->in(AllocateNode::KlassNode);
4460         ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4461         if (zeroes_done == k->layout_helper())
4462           zeroes_done = size_limit;
4463       }
4464     }
4465     if (zeroes_done < size_limit) {
4466       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


4467                                             zeroes_done, size_in_bytes, phase);
4468     }
4469   }
4470 
4471   set_complete(phase);
4472   return rawmem;
4473 }
4474 
4475 
4476 #ifdef ASSERT
4477 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4478   if (is_complete())
4479     return true;                // stores could be anything at this point
4480   assert(allocation() != NULL, "must be present");
4481   intptr_t last_off = allocation()->minimum_header_size();
4482   for (uint i = InitializeNode::RawStores; i < req(); i++) {
4483     Node* st = in(i);
4484     intptr_t st_off = get_store_offset(st, phase);
4485     if (st_off < 0)  continue;  // ignore dead garbage
4486     if (last_off > st_off) {

   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "ci/ciFlatArrayKlass.hpp"
  27 #include "classfile/javaClasses.hpp"
  28 #include "classfile/systemDictionary.hpp"
  29 #include "compiler/compileLog.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/c2/barrierSetC2.hpp"
  32 #include "gc/shared/tlab_globals.hpp"
  33 #include "memory/allocation.inline.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "oops/objArrayKlass.hpp"
  36 #include "opto/addnode.hpp"
  37 #include "opto/arraycopynode.hpp"
  38 #include "opto/cfgnode.hpp"
  39 #include "opto/regalloc.hpp"
  40 #include "opto/compile.hpp"
  41 #include "opto/connode.hpp"
  42 #include "opto/convertnode.hpp"
  43 #include "opto/inlinetypenode.hpp"
  44 #include "opto/loopnode.hpp"
  45 #include "opto/machnode.hpp"
  46 #include "opto/matcher.hpp"
  47 #include "opto/memnode.hpp"
  48 #include "opto/mulnode.hpp"
  49 #include "opto/narrowptrnode.hpp"
  50 #include "opto/phaseX.hpp"
  51 #include "opto/regmask.hpp"
  52 #include "opto/rootnode.hpp"
  53 #include "opto/vectornode.hpp"
  54 #include "utilities/align.hpp"
  55 #include "utilities/copy.hpp"
  56 #include "utilities/macros.hpp"
  57 #include "utilities/powerOfTwo.hpp"
  58 #include "utilities/vmError.hpp"
  59 
  60 // Portions of code courtesy of Clifford Click
  61 
  62 // Optimization - Graph Style
  63 

 226       // clone the Phi with our address type
 227       result = mphi->split_out_instance(t_adr, igvn);
 228     } else {
 229       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 230     }
 231   }
 232   return result;
 233 }
 234 
 235 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 236   uint alias_idx = phase->C->get_alias_index(tp);
 237   Node *mem = mmem;
 238 #ifdef ASSERT
 239   {
 240     // Check that current type is consistent with the alias index used during graph construction
 241     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 242     bool consistent =  adr_check == NULL || adr_check->empty() ||
 243                        phase->C->must_alias(adr_check, alias_idx );
 244     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 245     if( !consistent && adr_check != NULL && !adr_check->empty() &&
 246         tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 247         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 248         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 249           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 250           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 251       // don't assert if it is dead code.
 252       consistent = true;
 253     }
 254     if( !consistent ) {
 255       st->print("alias_idx==%d, adr_check==", alias_idx);
 256       if( adr_check == NULL ) {
 257         st->print("NULL");
 258       } else {
 259         adr_check->dump();
 260       }
 261       st->cr();
 262       print_alias_types();
 263       assert(consistent, "adr_check must match alias idx");
 264     }
 265   }
 266 #endif

 884          "use LoadKlassNode instead");
 885   assert(!(adr_type->isa_aryptr() &&
 886            adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
 887          "use LoadRangeNode instead");
 888   // Check control edge of raw loads
 889   assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
 890           // oop will be recorded in oop map if load crosses safepoint
 891           rt->isa_oopptr() || is_immutable_value(adr),
 892           "raw memory operations should have control edge");
 893   LoadNode* load = NULL;
 894   switch (bt) {
 895   case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 896   case T_BYTE:    load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 897   case T_INT:     load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 898   case T_CHAR:    load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 899   case T_SHORT:   load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 900   case T_LONG:    load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
 901   case T_FLOAT:   load = new LoadFNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
 902   case T_DOUBLE:  load = new LoadDNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency, require_atomic_access); break;
 903   case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(),  mo, control_dependency); break;
 904   case T_PRIMITIVE_OBJECT:
 905   case T_OBJECT:
 906 #ifdef _LP64
 907     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 908       load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
 909     } else
 910 #endif
 911     {
 912       assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
 913       load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
 914     }
 915     break;
 916   default:
 917     ShouldNotReachHere();
 918     break;
 919   }
 920   assert(load != NULL, "LoadNode should have been created");
 921   if (unaligned) {
 922     load->set_unaligned_access();
 923   }
 924   if (mismatched) {

 980       assert(ld_alloc != NULL, "need an alloc");
 981       assert(addp->is_AddP(), "address must be addp");
 982       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 983       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
 984       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
 985       addp->set_req(AddPNode::Base, src);
 986       addp->set_req(AddPNode::Address, src);
 987     } else {
 988       assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
 989              ac->as_ArrayCopy()->is_copyof_validated() ||
 990              ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
 991       assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
 992       addp->set_req(AddPNode::Base, src);
 993       addp->set_req(AddPNode::Address, src);
 994 
 995       const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
 996       BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
 997       if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
 998 
 999       uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1000       uint shift  = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));
1001 
1002       Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1003 #ifdef _LP64
1004       diff = phase->transform(new ConvI2LNode(diff));
1005 #endif
1006       diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1007 
1008       Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1009       addp->set_req(AddPNode::Offset, offset);
1010     }
1011     addp = phase->transform(addp);
1012 #ifdef ASSERT
1013     const TypePtr* adr_type = phase->type(addp)->is_ptr();
1014     ld->_adr_type = adr_type;
1015 #endif
1016     ld->set_req(MemNode::Address, addp);
1017     ld->set_req(0, ctl);
1018     ld->set_req(MemNode::Memory, mem);
1019     // load depends on the tests that validate the arraycopy
1020     ld->_control_dependency = UnknownControl;

1101         // Same base, same offset.
1102         // Possible improvement for arrays: check index value instead of absolute offset.
1103 
1104         // At this point we have proven something like this setup:
1105         //   B = << base >>
1106         //   L =  LoadQ(AddP(Check/CastPP(B), #Off))
1107         //   S = StoreQ(AddP(             B , #Off), V)
1108         // (Actually, we haven't yet proven the Q's are the same.)
1109         // In other words, we are loading from a casted version of
1110         // the same pointer-and-offset that we stored to.
1111         // Casted version may carry a dependency and it is respected.
1112         // Thus, we are able to replace L by V.
1113       }
1114       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1115       if (store_Opcode() != st->Opcode()) {
1116         return NULL;
1117       }
1118       // LoadVector/StoreVector needs additional check to ensure the types match.
1119       if (st->is_StoreVector()) {
1120         const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
1121         const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1122         if (in_vt != out_vt) {
1123           return NULL;
1124         }
1125       }
1126       return st->in(MemNode::ValueIn);
1127     }
1128 
1129     // A load from a freshly-created object always returns zero.
1130     // (This can happen after LoadNode::Ideal resets the load's memory input
1131     // to find_captured_store, which returned InitializeNode::zero_memory.)
1132     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1133         (st->in(0) == ld_alloc) &&
1134         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1135       // return a zero value for the load's basic type
1136       // (This is one of the few places where a generic PhaseTransform
1137       // can create new nodes.  Think of it as lazily manifesting
1138       // virtually pre-existing constants.)
1139       assert(memory_type() != T_PRIMITIVE_OBJECT, "should not be used for inline types");
1140       Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1141       if (default_value != NULL) {
1142         return default_value;
1143       }
1144       assert(ld_alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
1145       if (memory_type() != T_VOID) {
1146         if (ReduceBulkZeroing || find_array_copy_clone(phase, ld_alloc, in(MemNode::Memory)) == NULL) {
1147           // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1148           // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1149           // by the ArrayCopyNode.
1150           return phase->zerocon(memory_type());
1151         }
1152       } else {
1153         // TODO: materialize all-zero vector constant
1154         assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1155       }
1156     }
1157 
1158     // A load from an initialization barrier can match a captured store.
1159     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1160       InitializeNode* init = st->in(0)->as_Initialize();
1161       AllocateNode* alloc = init->allocation();
1162       if ((alloc != NULL) && (alloc == ld_alloc)) {
1163         // examine a captured store value
1164         st = init->find_captured_store(ld_off, memory_size(), phase);

1192 //----------------------is_instance_field_load_with_local_phi------------------
1193 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1194   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1195       in(Address)->is_AddP() ) {
1196     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1197     // Only instances and boxed values.
1198     if( t_oop != NULL &&
1199         (t_oop->is_ptr_to_boxed_value() ||
1200          t_oop->is_known_instance_field()) &&
1201         t_oop->offset() != Type::OffsetBot &&
1202         t_oop->offset() != Type::OffsetTop) {
1203       return true;
1204     }
1205   }
1206   return false;
1207 }
1208 
1209 //------------------------------Identity---------------------------------------
1210 // Loads are identity if previous store is to same address
1211 Node* LoadNode::Identity(PhaseGVN* phase) {
1212   // Loading from an InlineTypePtr? The InlineTypePtr has the values of
1213   // all fields as input. Look for the field with matching offset.
1214   Node* addr = in(Address);
1215   intptr_t offset;
1216   Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1217   if (base != NULL && base->is_InlineTypePtr() && offset > oopDesc::klass_offset_in_bytes()) {
1218     Node* value = base->as_InlineTypePtr()->field_value_by_offset((int)offset, true);
1219     if (value->is_InlineType()) {
1220       // Non-flattened inline type field
1221       InlineTypeNode* vt = value->as_InlineType();
1222       if (vt->is_allocated(phase)) {
1223         value = vt->get_oop();
1224       } else {
1225         // Not yet allocated, bail out
1226         value = NULL;
1227       }
1228     }
1229     if (value != NULL) {
1230       if (Opcode() == Op_LoadN) {
1231         // Encode oop value if we are loading a narrow oop
1232         assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1233         value = phase->transform(new EncodePNode(value, bottom_type()));
1234       }
1235       return value;
1236     }
1237   }
1238 
1239   // If the previous store-maker is the right kind of Store, and the store is
1240   // to the same address, then we are equal to the value stored.
1241   Node* mem = in(Memory);
1242   Node* value = can_see_stored_value(mem, phase);
1243   if( value ) {
1244     // byte, short & char stores truncate naturally.
1245     // A load has to load the truncated value which requires
1246     // some sort of masking operation and that requires an
1247     // Ideal call instead of an Identity call.
1248     if (memory_size() < BytesPerInt) {
1249       // If the input to the store does not fit with the load's result type,
1250       // it must be truncated via an Ideal call.
1251       if (!phase->type(value)->higher_equal(phase->type(this)))
1252         return this;
1253     }
1254     // (This works even when value is a Con, but LoadNode::Value
1255     // usually runs first, producing the singleton type of the Con.)
1256     return value;
1257   }
1258 

1928       }
1929     }
1930 
1931     // Don't do this for integer types. There is only potential profit if
1932     // the element type t is lower than _type; that is, for int types, if _type is
1933     // more restrictive than t.  This only happens here if one is short and the other
1934     // char (both 16 bits), and in those cases we've made an intentional decision
1935     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1936     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1937     //
1938     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1939     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
1940     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1941     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
1942     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1943     // In fact, that could have been the original type of p1, and p1 could have
1944     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1945     // expression (LShiftL quux 3) independently optimized to the constant 8.
1946     if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1947         && (_type->isa_vect() == NULL)
1948         && t->isa_inlinetype() == NULL
1949         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1950       // t might actually be lower than _type, if _type is a unique
1951       // concrete subclass of abstract class t.
1952       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
1953         const Type* jt = t->join_speculative(_type);
1954         // In any case, do not allow the join, per se, to empty out the type.
1955         if (jt->empty() && !t->empty()) {
1956           // This can happen if a interface-typed array narrows to a class type.
1957           jt = _type;
1958         }
1959 #ifdef ASSERT
1960         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1961           // The pointers in the autobox arrays are always non-null
1962           Node* base = adr->in(AddPNode::Base);
1963           if ((base != NULL) && base->is_DecodeN()) {
1964             // Get LoadN node which loads IntegerCache.cache field
1965             base = base->in(1);
1966           }
1967           if ((base != NULL) && base->is_Con()) {
1968             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1969             if ((base_type != NULL) && base_type->is_autobox_cache()) {
1970               // It could be narrow oop
1971               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1972             }
1973           }
1974         }
1975 #endif
1976         return jt;
1977       }
1978     }
1979   } else if (tp->base() == Type::InstPtr) {
1980     assert( off != Type::OffsetBot ||
1981             // arrays can be cast to Objects
1982             !tp->isa_instptr() ||
1983             tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
1984             // Default value load
1985             tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
1986             // unsafe field access may not have a constant offset
1987             C->has_unsafe_access(),
1988             "Field accesses must be precise" );
1989     // For oop loads, we expect the _type to be precise.
1990 

1991     const TypeInstPtr* tinst = tp->is_instptr();
1992     BasicType bt = memory_type();
1993 
1994     // Optimize loads from constant fields.
1995     ciObject* const_oop = tinst->const_oop();
1996     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1997       ciType* mirror_type = const_oop->as_instance()->java_mirror_type();
1998       if (mirror_type != NULL) {
1999         const Type* const_oop = NULL;
2000         ciInlineKlass* vk = mirror_type->is_inlinetype() ? mirror_type->as_inline_klass() : NULL;
2001         // Fold default value loads
2002         if (vk != NULL && off == vk->default_value_offset()) {
2003           const_oop = TypeInstPtr::make(vk->default_instance());
2004         }
2005         // Fold class mirror loads
2006         if (off == java_lang_Class::primary_mirror_offset()) {
2007           const_oop = (vk == NULL) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->ref_instance());
2008         } else if (off == java_lang_Class::secondary_mirror_offset()) {
2009           const_oop = (vk == NULL) ? TypePtr::NULL_PTR : TypeInstPtr::make(vk->val_instance());
2010         }
2011         if (const_oop != NULL) {
2012           return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
2013         }
2014       }
2015       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2016       if (con_type != NULL) {
2017         return con_type;
2018       }
2019     }
2020   } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2021     assert(off != Type::OffsetBot ||
2022             !tp->isa_instklassptr() ||
2023            // arrays can be cast to Objects
2024            tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2025            // also allow array-loading from the primary supertype
2026            // array during subtype checks
2027            Opcode() == Op_LoadKlass,
2028            "Field accesses must be precise");
2029     // For klass/static loads, we expect the _type to be precise
2030   } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
2031     if (adr->is_Load() && off == 0) {
2032       /* With mirrors being an indirect in the Klass*
2033        * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2034        * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2035        *
2036        * So check the type and klass of the node before the LoadP.
2037        */
2038       Node* adr2 = adr->in(MemNode::Address);
2039       const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2040       if (tkls != NULL) {
2041         if (tkls->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
2042           ciKlass* klass = tkls->exact_klass();
2043           assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2044           assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2045           return TypeInstPtr::make(klass->java_mirror());
2046         }
2047       }
2048     } else {
2049       // Check for a load of the default value offset from the InlineKlassFixedBlock:
2050       // LoadI(LoadP(inline_klass, adr_inlineklass_fixed_block_offset), default_value_offset_offset)
2051       intptr_t offset = 0;
2052       Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2053       if (base != NULL && base->is_Load() && offset == in_bytes(InlineKlass::default_value_offset_offset())) {
2054         const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
2055         if (tkls != NULL && tkls->is_loaded() && tkls->klass_is_exact() && tkls->exact_klass()->is_inlinetype() &&
2056             tkls->offset() == in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())) {
2057           assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
2058           assert(Opcode() == Op_LoadI, "must load an int from fixed block");
2059           return TypeInt::make(tkls->exact_klass()->as_inline_klass()->default_value_offset());
2060         }
2061       }
2062     }
2063   }
2064 
2065   const TypeKlassPtr *tkls = tp->isa_klassptr();
2066   if (tkls != NULL && !StressReflectiveCode) {
2067     if (tkls->is_loaded() && tkls->klass_is_exact()) {
2068       ciKlass* klass = tkls->exact_klass();
2069       // We are loading a field from a Klass metaobject whose identity
2070       // is known at compile time (the type is "exact" or "precise").
2071       // Check for fields we know are maintained as constants by the VM.
2072       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2073         // The field is Klass::_super_check_offset.  Return its (constant) value.
2074         // (Folds up type checking code.)
2075         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2076         return TypeInt::make(klass->super_check_offset());
2077       }
2078       // Compute index into primary_supers array
2079       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2080       // Check for overflowing; use unsigned compare to handle the negative case.

2146   if (ReduceFieldZeroing || is_instance || is_boxed_value) {
2147     Node* value = can_see_stored_value(mem,phase);
2148     if (value != NULL && value->is_Con()) {
2149       assert(value->bottom_type()->higher_equal(_type),"sanity");
2150       return value->bottom_type();
2151     }
2152   }
2153 
2154   bool is_vect = (_type->isa_vect() != NULL);
2155   if (is_instance && !is_vect) {
2156     // If we have an instance type and our memory input is the
2157     // programs's initial memory state, there is no matching store,
2158     // so just return a zero of the appropriate type -
2159     // except if it is vectorized - then we have no zero constant.
2160     Node *mem = in(MemNode::Memory);
2161     if (mem->is_Parm() && mem->in(0)->is_Start()) {
2162       assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2163       return Type::get_zero_type(_type->basic_type());
2164     }
2165   }

2166   Node* alloc = is_new_object_mark_load(phase);
2167   if (alloc != NULL) {
2168     if (EnableValhalla) {
2169       // The mark word may contain property bits (inline, flat, null-free)
2170       Node* klass_node = alloc->in(AllocateNode::KlassNode);
2171       const TypeKlassPtr* tkls = phase->type(klass_node)->is_klassptr();
2172       if (tkls->is_loaded() && tkls->klass_is_exact()) {
2173         return TypeX::make(tkls->exact_klass()->prototype_header().value());
2174       }
2175     } else {
2176       return TypeX::make(markWord::prototype().value());
2177     }
2178   }
2179 
2180   return _type;
2181 }
2182 
2183 //------------------------------match_edge-------------------------------------
2184 // Do we Match on this edge index or not?  Match only the address.
2185 uint LoadNode::match_edge(uint idx) const {
2186   return idx == MemNode::Address;
2187 }
2188 
2189 //--------------------------LoadBNode::Ideal--------------------------------------
2190 //
2191 //  If the previous store is to the same address as this load,
2192 //  and the value stored was larger than a byte, replace this load
2193 //  with the value stored truncated to a byte.  If no truncation is
2194 //  needed, the replacement is done in LoadNode::Identity().
2195 //
2196 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2197   Node* mem = in(MemNode::Memory);

2308   return LoadNode::Ideal(phase, can_reshape);
2309 }
2310 
2311 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2312   Node* mem = in(MemNode::Memory);
2313   Node* value = can_see_stored_value(mem,phase);
2314   if (value != NULL && value->is_Con() &&
2315       !value->bottom_type()->higher_equal(_type)) {
2316     // If the input to the store does not fit with the load's result type,
2317     // it must be truncated. We can't delay until Ideal call since
2318     // a singleton Value is needed for split_thru_phi optimization.
2319     int con = value->get_int();
2320     return TypeInt::make((con << 16) >> 16);
2321   }
2322   return LoadNode::Value(phase);
2323 }
2324 
2325 //=============================================================================
2326 //----------------------------LoadKlassNode::make------------------------------
2327 // Polymorphic factory method:
2328 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* at,
2329                           const TypeKlassPtr* tk) {
2330   // sanity check the alias category against the created node type
2331   const TypePtr *adr_type = adr->bottom_type()->isa_ptr();
2332   assert(adr_type != NULL, "expecting TypeKlassPtr");
2333 #ifdef _LP64
2334   if (adr_type->is_ptr_to_narrowklass()) {
2335     assert(UseCompressedClassPointers, "no compressed klasses");
2336     Node* load_klass = gvn.transform(new LoadNKlassNode(ctl, mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2337     return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2338   }
2339 #endif
2340   assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2341   return new LoadKlassNode(ctl, mem, adr, at, tk, MemNode::unordered);
2342 }
2343 
2344 //------------------------------Value------------------------------------------
2345 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {
2346   return klass_value_common(phase);
2347 }
2348 
2349 // In most cases, LoadKlassNode does not have the control input set. If the control

2356   // Either input is TOP ==> the result is TOP
2357   const Type *t1 = phase->type( in(MemNode::Memory) );
2358   if (t1 == Type::TOP)  return Type::TOP;
2359   Node *adr = in(MemNode::Address);
2360   const Type *t2 = phase->type( adr );
2361   if (t2 == Type::TOP)  return Type::TOP;
2362   const TypePtr *tp = t2->is_ptr();
2363   if (TypePtr::above_centerline(tp->ptr()) ||
2364       tp->ptr() == TypePtr::Null)  return Type::TOP;
2365 
2366   // Return a more precise klass, if possible
2367   const TypeInstPtr *tinst = tp->isa_instptr();
2368   if (tinst != NULL) {
2369     ciInstanceKlass* ik = tinst->instance_klass();
2370     int offset = tinst->offset();
2371     if (ik == phase->C->env()->Class_klass()
2372         && (offset == java_lang_Class::klass_offset() ||
2373             offset == java_lang_Class::array_klass_offset())) {
2374       // We are loading a special hidden field from a Class mirror object,
2375       // the field which points to the VM's Klass metaobject.
2376       bool null_free = false;
2377       ciType* t = tinst->java_mirror_type(&null_free);
2378       // java_mirror_type returns non-null for compile-time Class constants.
2379       if (t != NULL) {
2380         // constant oop => constant klass
2381         if (offset == java_lang_Class::array_klass_offset()) {
2382           if (t->is_void()) {
2383             // We cannot create a void array.  Since void is a primitive type return null
2384             // klass.  Users of this result need to do a null check on the returned klass.
2385             return TypePtr::NULL_PTR;
2386           }
2387           return TypeKlassPtr::make(ciArrayKlass::make(t, null_free));
2388         }
2389         if (!t->is_klass()) {
2390           // a primitive Class (e.g., int.class) has NULL for a klass field
2391           return TypePtr::NULL_PTR;
2392         }
2393         // (Folds up the 1st indirection in aClassConstant.getModifiers().)
2394         return TypeKlassPtr::make(t->as_klass());
2395       }
2396       // non-constant mirror, so we can't tell what's going on
2397     }
2398     if (!tinst->is_loaded())
2399       return _type;             // Bail out if not loaded
2400     if (offset == oopDesc::klass_offset_in_bytes()) {
2401       return tinst->as_klass_type(true);
2402     }
2403   }
2404 
2405   // Check for loading klass from an array
2406   const TypeAryPtr* tary = tp->isa_aryptr();
2407   if (tary != NULL && tary->elem() != Type::BOTTOM &&
2408       tary->offset() == oopDesc::klass_offset_in_bytes()) {
2409     return tary->as_klass_type(true);
2410   }
2411 
2412   // Check for loading klass from an array klass
2413   const TypeKlassPtr *tkls = tp->isa_klassptr();
2414   if (tkls != NULL && !StressReflectiveCode) {
2415     if (!tkls->is_loaded())
2416      return _type;             // Bail out if not loaded
2417     if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2418         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2419       // // Always returning precise element type is incorrect,
2420       // // e.g., element type could be object and array may contain strings
2421       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2422 
2423       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2424       // according to the element type's subclassing.
2425       return tkls->is_aryklassptr()->elem();
2426     }

2612 //=============================================================================
2613 //---------------------------StoreNode::make-----------------------------------
2614 // Polymorphic factory method:
2615 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2616   assert((mo == unordered || mo == release), "unexpected");
2617   Compile* C = gvn.C;
2618   assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2619          ctl != NULL, "raw memory operations should have control edge");
2620 
2621   switch (bt) {
2622   case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2623   case T_BYTE:    return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2624   case T_INT:     return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2625   case T_CHAR:
2626   case T_SHORT:   return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2627   case T_LONG:    return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2628   case T_FLOAT:   return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2629   case T_DOUBLE:  return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2630   case T_METADATA:
2631   case T_ADDRESS:
2632   case T_PRIMITIVE_OBJECT:
2633   case T_OBJECT:
2634 #ifdef _LP64
2635     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2636       val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2637       return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2638     } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2639                (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2640                 adr->bottom_type()->isa_rawptr())) {
2641       val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2642       return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2643     }
2644 #endif
2645     {
2646       return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2647     }
2648   default:
2649     ShouldNotReachHere();
2650     return (StoreNode*)NULL;
2651   }
2652 }

2663 
2664   // Since they are not commoned, do not hash them:
2665   return NO_HASH;
2666 }
2667 
2668 //------------------------------Ideal------------------------------------------
2669 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
2670 // When a store immediately follows a relevant allocation/initialization,
2671 // try to capture it into the initialization, or hoist it above.
2672 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2673   Node* p = MemNode::Ideal_common(phase, can_reshape);
2674   if (p)  return (p == NodeSentinel) ? NULL : p;
2675 
2676   Node* mem     = in(MemNode::Memory);
2677   Node* address = in(MemNode::Address);
2678   Node* value   = in(MemNode::ValueIn);
2679   // Back-to-back stores to same address?  Fold em up.  Generally
2680   // unsafe if I have intervening uses...  Also disallowed for StoreCM
2681   // since they must follow each StoreP operation.  Redundant StoreCMs
2682   // are eliminated just before matching in final_graph_reshape.
2683   if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
2684     Node* st = mem;
2685     // If Store 'st' has more than one use, we cannot fold 'st' away.
2686     // For example, 'st' might be the final state at a conditional
2687     // return.  Or, 'st' might be used by some node which is live at
2688     // the same time 'st' is live, which might be unschedulable.  So,
2689     // require exactly ONE user until such time as we clone 'mem' for
2690     // each of 'mem's uses (thus making the exactly-1-user-rule hold
2691     // true).
2692     while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2693       // Looking at a dead closed cycle of memory?
2694       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2695       assert(Opcode() == st->Opcode() ||
2696              st->Opcode() == Op_StoreVector ||
2697              Opcode() == Op_StoreVector ||
2698              st->Opcode() == Op_StoreVectorScatter ||
2699              Opcode() == Op_StoreVectorScatter ||
2700              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2701              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2702              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2703              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
2704              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2705              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2706 
2707       if (st->in(MemNode::Address)->eqv_uncast(address) &&
2708           st->as_Store()->memory_size() <= this->memory_size()) {
2709         Node* use = st->raw_out(0);
2710         if (phase->is_IterGVN()) {
2711           phase->is_IterGVN()->rehash_node_delayed(use);
2712         }
2713         // It's OK to do this in the parser, since DU info is always accurate,
2714         // and the parser always refers to nodes via SafePointNode maps.
2715         use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
2716         return this;
2717       }
2718       st = st->in(MemNode::Memory);
2719     }
2720   }
2721 
2722 
2723   // Capture an unaliased, unconditional, simple store into an initializer.

2780   // Load then Store?  Then the Store is useless
2781   if (val->is_Load() &&
2782       val->in(MemNode::Address)->eqv_uncast(adr) &&
2783       val->in(MemNode::Memory )->eqv_uncast(mem) &&
2784       val->as_Load()->store_Opcode() == Opcode()) {
2785     result = mem;
2786   }
2787 
2788   // Two stores in a row of the same value?
2789   if (result == this &&
2790       mem->is_Store() &&
2791       mem->in(MemNode::Address)->eqv_uncast(adr) &&
2792       mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2793       mem->Opcode() == Opcode()) {
2794     result = mem;
2795   }
2796 
2797   // Store of zero anywhere into a freshly-allocated object?
2798   // Then the store is useless.
2799   // (It must already have been captured by the InitializeNode.)
2800   if (result == this && ReduceFieldZeroing) {

2801     // a newly allocated object is already all-zeroes everywhere
2802     if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
2803         (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
2804       assert(!phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == NULL, "storing null to inline type array is forbidden");
2805       result = mem;
2806     }
2807 
2808     if (result == this && phase->type(val)->is_zero_type()) {
2809       // the store may also apply to zero-bits in an earlier object
2810       Node* prev_mem = find_previous_store(phase);
2811       // Steps (a), (b):  Walk past independent stores to find an exact match.
2812       if (prev_mem != NULL) {
2813         Node* prev_val = can_see_stored_value(prev_mem, phase);
2814         if (prev_val != NULL && prev_val == val) {
2815           // prev_val and val might differ by a cast; it would be good
2816           // to keep the more informative of the two.
2817           result = mem;
2818         }
2819       }
2820     }
2821   }
2822 
2823   PhaseIterGVN* igvn = phase->is_IterGVN();
2824   if (result != this && igvn != NULL) {
2825     MemBarNode* trailing = trailing_membar();
2826     if (trailing != NULL) {
2827 #ifdef ASSERT
2828       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();

2973 Node* StoreCMNode::Identity(PhaseGVN* phase) {
2974   // No need to card mark when storing a null ptr
2975   Node* my_store = in(MemNode::OopStore);
2976   if (my_store->is_Store()) {
2977     const Type *t1 = phase->type( my_store->in(MemNode::ValueIn) );
2978     if( t1 == TypePtr::NULL_PTR ) {
2979       return in(MemNode::Memory);
2980     }
2981   }
2982   return this;
2983 }
2984 
2985 //=============================================================================
2986 //------------------------------Ideal---------------------------------------
2987 Node *StoreCMNode::Ideal(PhaseGVN *phase, bool can_reshape){
2988   Node* progress = StoreNode::Ideal(phase, can_reshape);
2989   if (progress != NULL) return progress;
2990 
2991   Node* my_store = in(MemNode::OopStore);
2992   if (my_store->is_MergeMem()) {
2993     if (oop_alias_idx() != phase->C->get_alias_index(TypeAryPtr::INLINES) ||
2994         phase->C->flattened_accesses_share_alias()) {
2995       // The alias that was recorded is no longer accurate enough.
2996       Node* mem = my_store->as_MergeMem()->memory_at(oop_alias_idx());
2997       set_req_X(MemNode::OopStore, mem, phase);
2998       return this;
2999     }
3000   }
3001 
3002   return NULL;
3003 }
3004 
3005 //------------------------------Value-----------------------------------------
3006 const Type* StoreCMNode::Value(PhaseGVN* phase) const {
3007   // Either input is TOP ==> the result is TOP (checked in StoreNode::Value).
3008   // If extra input is TOP ==> the result is TOP
3009   const Type* t = phase->type(in(MemNode::OopStore));
3010   if (t == Type::TOP) {
3011     return Type::TOP;
3012   }
3013   return StoreNode::Value(phase);
3014 }
3015 
3016 
3017 //=============================================================================
3018 //----------------------------------SCMemProjNode------------------------------
3019 const Type* SCMemProjNode::Value(PhaseGVN* phase) const

3138 // Clearing a short array is faster with stores
3139 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3140   // Already know this is a large node, do not try to ideal it
3141   if (_is_large) return NULL;
3142 
3143   const int unit = BytesPerLong;
3144   const TypeX* t = phase->type(in(2))->isa_intptr_t();
3145   if (!t)  return NULL;
3146   if (!t->is_con())  return NULL;
3147   intptr_t raw_count = t->get_con();
3148   intptr_t size = raw_count;
3149   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
3150   // Clearing nothing uses the Identity call.
3151   // Negative clears are possible on dead ClearArrays
3152   // (see jck test stmt114.stmt11402.val).
3153   if (size <= 0 || size % unit != 0)  return NULL;
3154   intptr_t count = size / unit;
3155   // Length too long; communicate this to matchers and assemblers.
3156   // Assemblers are responsible to produce fast hardware clears for it.
3157   if (size > InitArrayShortSize) {
3158     return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
3159   } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
3160     return NULL;
3161   }
3162   if (!IdealizeClearArrayNode) return NULL;
3163   Node *mem = in(1);
3164   if( phase->type(mem)==Type::TOP ) return NULL;
3165   Node *adr = in(3);
3166   const Type* at = phase->type(adr);
3167   if( at==Type::TOP ) return NULL;
3168   const TypePtr* atp = at->isa_ptr();
3169   // adjust atp to be the correct array element address type
3170   if (atp == NULL)  atp = TypePtr::BOTTOM;
3171   else              atp = atp->add_offset(Type::OffsetBot);
3172   // Get base for derived pointer purposes
3173   if( adr->Opcode() != Op_AddP ) Unimplemented();
3174   Node *base = adr->in(1);
3175 
3176   Node *val = in(4);
3177   Node *off  = phase->MakeConX(BytesPerLong);
3178   mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3179   count--;
3180   while( count-- ) {
3181     mem = phase->transform(mem);
3182     adr = phase->transform(new AddPNode(base,adr,off));
3183     mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3184   }
3185   return mem;
3186 }
3187 
3188 //----------------------------step_through----------------------------------
3189 // Return allocation input memory edge if it is different instance
3190 // or itself if it is the one we are looking for.
3191 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
3192   Node* n = *np;
3193   assert(n->is_ClearArray(), "sanity");
3194   intptr_t offset;
3195   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3196   // This method is called only before Allocate nodes are expanded
3197   // during macro nodes expansion. Before that ClearArray nodes are
3198   // only generated in PhaseMacroExpand::generate_arraycopy() (before
3199   // Allocate nodes are expanded) which follows allocations.
3200   assert(alloc != NULL, "should have allocation");
3201   if (alloc->_idx == instance_id) {
3202     // Can not bypass initialization of the instance we are looking for.
3203     return false;
3204   }
3205   // Otherwise skip it.
3206   InitializeNode* init = alloc->initialization();
3207   if (init != NULL)
3208     *np = init->in(TypeFunc::Memory);
3209   else
3210     *np = alloc->in(TypeFunc::Memory);
3211   return true;
3212 }
3213 
3214 //----------------------------clear_memory-------------------------------------
3215 // Generate code to initialize object storage to zero.
3216 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3217                                    Node* val,
3218                                    Node* raw_val,
3219                                    intptr_t start_offset,
3220                                    Node* end_offset,
3221                                    PhaseGVN* phase) {
3222   intptr_t offset = start_offset;
3223 
3224   int unit = BytesPerLong;
3225   if ((offset % unit) != 0) {
3226     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3227     adr = phase->transform(adr);
3228     const TypePtr* atp = TypeRawPtr::BOTTOM;
3229     if (val != NULL) {
3230       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3231       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3232     } else {
3233       assert(raw_val == NULL, "val may not be null");
3234       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3235     }
3236     mem = phase->transform(mem);
3237     offset += BytesPerInt;
3238   }
3239   assert((offset % unit) == 0, "");
3240 
3241   // Initialize the remaining stuff, if any, with a ClearArray.
3242   return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3243 }
3244 
3245 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3246                                    Node* raw_val,
3247                                    Node* start_offset,
3248                                    Node* end_offset,
3249                                    PhaseGVN* phase) {
3250   if (start_offset == end_offset) {
3251     // nothing to do
3252     return mem;
3253   }
3254 
3255   int unit = BytesPerLong;
3256   Node* zbase = start_offset;
3257   Node* zend  = end_offset;
3258 
3259   // Scale to the unit required by the CPU:
3260   if (!Matcher::init_array_count_is_in_bytes) {
3261     Node* shift = phase->intcon(exact_log2(unit));
3262     zbase = phase->transform(new URShiftXNode(zbase, shift) );
3263     zend  = phase->transform(new URShiftXNode(zend,  shift) );
3264   }
3265 
3266   // Bulk clear double-words
3267   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3268   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3269   if (raw_val == NULL) {
3270     raw_val = phase->MakeConX(0);
3271   }
3272   mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3273   return phase->transform(mem);
3274 }
3275 
3276 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3277                                    Node* val,
3278                                    Node* raw_val,
3279                                    intptr_t start_offset,
3280                                    intptr_t end_offset,
3281                                    PhaseGVN* phase) {
3282   if (start_offset == end_offset) {
3283     // nothing to do
3284     return mem;
3285   }
3286 
3287   assert((end_offset % BytesPerInt) == 0, "odd end offset");
3288   intptr_t done_offset = end_offset;
3289   if ((done_offset % BytesPerLong) != 0) {
3290     done_offset -= BytesPerInt;
3291   }
3292   if (done_offset > start_offset) {
3293     mem = clear_memory(ctl, mem, dest, val, raw_val,
3294                        start_offset, phase->MakeConX(done_offset), phase);
3295   }
3296   if (done_offset < end_offset) { // emit the final 32-bit store
3297     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3298     adr = phase->transform(adr);
3299     const TypePtr* atp = TypeRawPtr::BOTTOM;
3300     if (val != NULL) {
3301       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3302       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3303     } else {
3304       assert(raw_val == NULL, "val may not be null");
3305       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3306     }
3307     mem = phase->transform(mem);
3308     done_offset += BytesPerInt;
3309   }
3310   assert(done_offset == end_offset, "");
3311   return mem;
3312 }
3313 
3314 //=============================================================================
3315 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3316   : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3317     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3318 #ifdef ASSERT
3319   , _pair_idx(0)
3320 #endif
3321 {
3322   init_class_id(Class_MemBar);
3323   Node* top = C->top();
3324   init_req(TypeFunc::I_O,top);
3325   init_req(TypeFunc::FramePtr,top);
3326   init_req(TypeFunc::ReturnAdr,top);

3432       PhaseIterGVN* igvn = phase->is_IterGVN();
3433       remove(igvn);
3434       // Must return either the original node (now dead) or a new node
3435       // (Do not return a top here, since that would break the uniqueness of top.)
3436       return new ConINode(TypeInt::ZERO);
3437     }
3438   }
3439   return progress ? this : NULL;
3440 }
3441 
3442 //------------------------------Value------------------------------------------
3443 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3444   if( !in(0) ) return Type::TOP;
3445   if( phase->type(in(0)) == Type::TOP )
3446     return Type::TOP;
3447   return TypeTuple::MEMBAR;
3448 }
3449 
3450 //------------------------------match------------------------------------------
3451 // Construct projections for memory.
3452 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
3453   switch (proj->_con) {
3454   case TypeFunc::Control:
3455   case TypeFunc::Memory:
3456     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3457   }
3458   ShouldNotReachHere();
3459   return NULL;
3460 }
3461 
3462 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3463   trailing->_kind = TrailingStore;
3464   leading->_kind = LeadingStore;
3465 #ifdef ASSERT
3466   trailing->_pair_idx = leading->_idx;
3467   leading->_pair_idx = leading->_idx;
3468 #endif
3469 }
3470 
3471 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3472   trailing->_kind = TrailingLoadStore;

3739   return (req() > RawStores);
3740 }
3741 
3742 void InitializeNode::set_complete(PhaseGVN* phase) {
3743   assert(!is_complete(), "caller responsibility");
3744   _is_complete = Complete;
3745 
3746   // After this node is complete, it contains a bunch of
3747   // raw-memory initializations.  There is no need for
3748   // it to have anything to do with non-raw memory effects.
3749   // Therefore, tell all non-raw users to re-optimize themselves,
3750   // after skipping the memory effects of this initialization.
3751   PhaseIterGVN* igvn = phase->is_IterGVN();
3752   if (igvn)  igvn->add_users_to_worklist(this);
3753 }
3754 
3755 // convenience function
3756 // return false if the init contains any stores already
3757 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3758   InitializeNode* init = initialization();
3759   if (init == NULL || init->is_complete()) {
3760     return false;
3761   }
3762   init->remove_extra_zeroes();
3763   // for now, if this allocation has already collected any inits, bail:
3764   if (init->is_non_zero())  return false;
3765   init->set_complete(phase);
3766   return true;
3767 }
3768 
3769 void InitializeNode::remove_extra_zeroes() {
3770   if (req() == RawStores)  return;
3771   Node* zmem = zero_memory();
3772   uint fill = RawStores;
3773   for (uint i = fill; i < req(); i++) {
3774     Node* n = in(i);
3775     if (n->is_top() || n == zmem)  continue;  // skip
3776     if (fill < i)  set_req(fill, n);          // compact
3777     ++fill;
3778   }
3779   // delete any empty spaces created:
3780   while (fill < req()) {
3781     del_req(fill);

3919             // store node that we'd like to capture. We need to check
3920             // the uses of the MergeMemNode.
3921             mems.push(n);
3922           }
3923         } else if (n->is_Mem()) {
3924           Node* other_adr = n->in(MemNode::Address);
3925           if (other_adr == adr) {
3926             failed = true;
3927             break;
3928           } else {
3929             const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
3930             if (other_t_adr != NULL) {
3931               int other_alias_idx = phase->C->get_alias_index(other_t_adr);
3932               if (other_alias_idx == alias_idx) {
3933                 // A load from the same memory slice as the store right
3934                 // after the InitializeNode. We check the control of the
3935                 // object/array that is loaded from. If it's the same as
3936                 // the store control then we cannot capture the store.
3937                 assert(!n->is_Store(), "2 stores to same slice on same control?");
3938                 Node* base = other_adr;
3939                 if (base->is_Phi()) {
3940                   // In rare case, base may be a PhiNode and it may read
3941                   // the same memory slice between InitializeNode and store.
3942                   failed = true;
3943                   break;
3944                 }
3945                 assert(base->is_AddP(), "should be addp but is %s", base->Name());
3946                 base = base->in(AddPNode::Base);
3947                 if (base != NULL) {
3948                   base = base->uncast();
3949                   if (base->is_Proj() && base->in(0) == alloc) {
3950                     failed = true;
3951                     break;
3952                   }
3953                 }
3954               }
3955             }
3956           }
3957         } else {
3958           failed = true;
3959           break;
3960         }
3961       }
3962     }
3963   }
3964   if (failed) {

4509         //   z's_done      12  16  16  16    12  16    12
4510         //   z's_needed    12  16  16  16    16  16    16
4511         //   zsize          0   0   0   0     4   0     4
4512         if (next_full_store < 0) {
4513           // Conservative tack:  Zero to end of current word.
4514           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4515         } else {
4516           // Zero to beginning of next fully initialized word.
4517           // Or, don't zero at all, if we are already in that word.
4518           assert(next_full_store >= zeroes_needed, "must go forward");
4519           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4520           zeroes_needed = next_full_store;
4521         }
4522       }
4523 
4524       if (zeroes_needed > zeroes_done) {
4525         intptr_t zsize = zeroes_needed - zeroes_done;
4526         // Do some incremental zeroing on rawmem, in parallel with inits.
4527         zeroes_done = align_down(zeroes_done, BytesPerInt);
4528         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4529                                               allocation()->in(AllocateNode::DefaultValue),
4530                                               allocation()->in(AllocateNode::RawDefaultValue),
4531                                               zeroes_done, zeroes_needed,
4532                                               phase);
4533         zeroes_done = zeroes_needed;
4534         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4535           do_zeroing = false;   // leave the hole, next time
4536       }
4537     }
4538 
4539     // Collect the store and move on:
4540     phase->replace_input_of(st, MemNode::Memory, inits);
4541     inits = st;                 // put it on the linearized chain
4542     set_req(i, zmem);           // unhook from previous position
4543 
4544     if (zeroes_done == st_off)
4545       zeroes_done = next_init_off;
4546 
4547     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4548 
4549     #ifdef ASSERT
4550     // Various order invariants.  Weaker than stores_are_sane because

4570   remove_extra_zeroes();        // clear out all the zmems left over
4571   add_req(inits);
4572 
4573   if (!(UseTLAB && ZeroTLAB)) {
4574     // If anything remains to be zeroed, zero it all now.
4575     zeroes_done = align_down(zeroes_done, BytesPerInt);
4576     // if it is the last unused 4 bytes of an instance, forget about it
4577     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4578     if (zeroes_done + BytesPerLong >= size_limit) {
4579       AllocateNode* alloc = allocation();
4580       assert(alloc != NULL, "must be present");
4581       if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4582         Node* klass_node = alloc->in(AllocateNode::KlassNode);
4583         ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
4584         if (zeroes_done == k->layout_helper())
4585           zeroes_done = size_limit;
4586       }
4587     }
4588     if (zeroes_done < size_limit) {
4589       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4590                                             allocation()->in(AllocateNode::DefaultValue),
4591                                             allocation()->in(AllocateNode::RawDefaultValue),
4592                                             zeroes_done, size_in_bytes, phase);
4593     }
4594   }
4595 
4596   set_complete(phase);
4597   return rawmem;
4598 }
4599 
4600 
4601 #ifdef ASSERT
4602 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4603   if (is_complete())
4604     return true;                // stores could be anything at this point
4605   assert(allocation() != NULL, "must be present");
4606   intptr_t last_off = allocation()->minimum_header_size();
4607   for (uint i = InitializeNode::RawStores; i < req(); i++) {
4608     Node* st = in(i);
4609     intptr_t st_off = get_store_offset(st, phase);
4610     if (st_off < 0)  continue;  // ignore dead garbage
4611     if (last_off > st_off) {
< prev index next >