< prev index next >

src/hotspot/share/opto/memnode.cpp

Print this page

   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 

  26 #include "classfile/javaClasses.hpp"

  27 #include "compiler/compileLog.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/c2/barrierSetC2.hpp"
  30 #include "gc/shared/tlab_globals.hpp"
  31 #include "memory/allocation.inline.hpp"
  32 #include "memory/resourceArea.hpp"

  33 #include "oops/objArrayKlass.hpp"
  34 #include "opto/addnode.hpp"
  35 #include "opto/arraycopynode.hpp"
  36 #include "opto/cfgnode.hpp"
  37 #include "opto/compile.hpp"
  38 #include "opto/connode.hpp"
  39 #include "opto/convertnode.hpp"

  40 #include "opto/loopnode.hpp"
  41 #include "opto/machnode.hpp"
  42 #include "opto/matcher.hpp"
  43 #include "opto/memnode.hpp"
  44 #include "opto/mempointer.hpp"
  45 #include "opto/mulnode.hpp"
  46 #include "opto/narrowptrnode.hpp"
  47 #include "opto/phaseX.hpp"
  48 #include "opto/regalloc.hpp"
  49 #include "opto/regmask.hpp"
  50 #include "opto/rootnode.hpp"
  51 #include "opto/traceMergeStoresTag.hpp"
  52 #include "opto/vectornode.hpp"
  53 #include "utilities/align.hpp"
  54 #include "utilities/copy.hpp"

  55 #include "utilities/macros.hpp"
  56 #include "utilities/powerOfTwo.hpp"
  57 #include "utilities/vmError.hpp"
  58 
  59 // Portions of code courtesy of Clifford Click
  60 
  61 // Optimization - Graph Style
  62 
  63 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
  64 
  65 //=============================================================================
  66 uint MemNode::size_of() const { return sizeof(*this); }
  67 
  68 const TypePtr *MemNode::adr_type() const {
  69   Node* adr = in(Address);
  70   if (adr == nullptr)  return nullptr; // node is dead
  71   const TypePtr* cross_check = nullptr;
  72   DEBUG_ONLY(cross_check = _adr_type);
  73   return calculate_adr_type(adr->bottom_type(), cross_check);
  74 }

 123       st->print(", idx=Bot;");
 124     else if (atp->index() == Compile::AliasIdxTop)
 125       st->print(", idx=Top;");
 126     else if (atp->index() == Compile::AliasIdxRaw)
 127       st->print(", idx=Raw;");
 128     else {
 129       ciField* field = atp->field();
 130       if (field) {
 131         st->print(", name=");
 132         field->print_name_on(st);
 133       }
 134       st->print(", idx=%d;", atp->index());
 135     }
 136   }
 137 }
 138 
 139 extern void print_alias_types();
 140 
 141 #endif
 142 
 143 Node *MemNode::optimize_simple_memory_chain(Node *mchain, const TypeOopPtr *t_oop, Node *load, PhaseGVN *phase) {
 144   assert((t_oop != nullptr), "sanity");
 145   bool is_instance = t_oop->is_known_instance_field();
 146   bool is_boxed_value_load = t_oop->is_ptr_to_boxed_value() &&
 147                              (load != nullptr) && load->is_Load() &&
 148                              (phase->is_IterGVN() != nullptr);
 149   if (!(is_instance || is_boxed_value_load))
 150     return mchain;  // don't try to optimize non-instance types

































































































 151   uint instance_id = t_oop->instance_id();
 152   Node *start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
 153   Node *prev = nullptr;
 154   Node *result = mchain;
 155   while (prev != result) {
 156     prev = result;
 157     if (result == start_mem)
 158       break;  // hit one of our sentinels



 159     // skip over a call which does not affect this memory slice
 160     if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
 161       Node *proj_in = result->in(0);
 162       if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
 163         break;  // hit one of our sentinels

 164       } else if (proj_in->is_Call()) {
 165         // ArrayCopyNodes processed here as well
 166         CallNode *call = proj_in->as_Call();
 167         if (!call->may_modify(t_oop, phase)) { // returns false for instances


 168           result = call->in(TypeFunc::Memory);
 169         }
 170       } else if (proj_in->is_Initialize()) {
 171         AllocateNode* alloc = proj_in->as_Initialize()->allocation();
 172         // Stop if this is the initialization for the object instance which
 173         // contains this memory slice, otherwise skip over it.
 174         if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
 175           break;
 176         }
 177         if (is_instance) {
 178           result = proj_in->in(TypeFunc::Memory);
 179         } else if (is_boxed_value_load) {
 180           Node* klass = alloc->in(AllocateNode::KlassNode);
 181           const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
 182           if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
 183             result = proj_in->in(TypeFunc::Memory); // not related allocation




 184           }
 185         }
 186       } else if (proj_in->is_MemBar()) {
 187         ArrayCopyNode* ac = nullptr;
 188         if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
 189           break;
 190         }
 191         result = proj_in->in(TypeFunc::Memory);





 192       } else if (proj_in->is_top()) {
 193         break; // dead code
 194       } else {
 195         assert(false, "unexpected projection");
 196       }
 197     } else if (result->is_ClearArray()) {
 198       if (!is_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
 199         // Can not bypass initialization of the instance
 200         // we are looking for.
 201         break;
 202       }
 203       // Otherwise skip it (the call updated 'result' value).
 204     } else if (result->is_MergeMem()) {
 205       result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
 206     }
 207   }
 208   return result;
 209 }
 210 
 211 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
 212   const TypeOopPtr* t_oop = t_adr->isa_oopptr();
 213   if (t_oop == nullptr)
 214     return mchain;  // don't try to optimize non-oop types
 215   Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
 216   bool is_instance = t_oop->is_known_instance_field();
 217   PhaseIterGVN *igvn = phase->is_IterGVN();
 218   if (is_instance && igvn != nullptr && result->is_Phi()) {
 219     PhiNode *mphi = result->as_Phi();
 220     assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
 221     const TypePtr *t = mphi->adr_type();
 222     bool do_split = false;
 223     // In the following cases, Load memory input can be further optimized based on
 224     // its precise address type
 225     if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
 226       do_split = true;
 227     } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
 228       const TypeOopPtr* mem_t =
 229         t->is_oopptr()->cast_to_exactness(true)
 230         ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
 231         ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
 232       if (t_oop->isa_aryptr()) {
 233         mem_t = mem_t->is_aryptr()
 234                      ->cast_to_stable(t_oop->is_aryptr()->is_stable())
 235                      ->cast_to_size(t_oop->is_aryptr()->size())


 236                      ->with_offset(t_oop->is_aryptr()->offset())
 237                      ->is_aryptr();
 238       }
 239       do_split = mem_t == t_oop;
 240     }
 241     if (do_split) {
 242       // clone the Phi with our address type
 243       result = mphi->split_out_instance(t_adr, igvn);
 244     } else {
 245       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 246     }
 247   }
 248   return result;
 249 }
 250 
 251 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 252   uint alias_idx = phase->C->get_alias_index(tp);
 253   Node *mem = mmem;
 254 #ifdef ASSERT
 255   {
 256     // Check that current type is consistent with the alias index used during graph construction
 257     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 258     bool consistent =  adr_check == nullptr || adr_check->empty() ||
 259                        phase->C->must_alias(adr_check, alias_idx );
 260     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 261     if( !consistent && adr_check != nullptr && !adr_check->empty() &&
 262                tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 263         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 264         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 265           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 266           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 267       // don't assert if it is dead code.
 268       consistent = true;
 269     }
 270     if( !consistent ) {
 271       st->print("alias_idx==%d, adr_check==", alias_idx);
 272       if( adr_check == nullptr ) {
 273         st->print("null");
 274       } else {
 275         adr_check->dump();
 276       }
 277       st->cr();
 278       print_alias_types();
 279       assert(consistent, "adr_check must match alias idx");
 280     }
 281   }
 282 #endif

 954          "use LoadKlassNode instead");
 955   assert(!(adr_type->isa_aryptr() &&
 956            adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
 957          "use LoadRangeNode instead");
 958   // Check control edge of raw loads
 959   assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
 960           // oop will be recorded in oop map if load crosses safepoint
 961           rt->isa_oopptr() || is_immutable_value(adr),
 962           "raw memory operations should have control edge");
 963   LoadNode* load = nullptr;
 964   switch (bt) {
 965   case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 966   case T_BYTE:    load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 967   case T_INT:     load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 968   case T_CHAR:    load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 969   case T_SHORT:   load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
 970   case T_LONG:    load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
 971   case T_FLOAT:   load = new LoadFNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
 972   case T_DOUBLE:  load = new LoadDNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency, require_atomic_access); break;
 973   case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(),  mo, control_dependency); break;

 974   case T_OBJECT:
 975   case T_NARROWOOP:
 976 #ifdef _LP64
 977     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
 978       load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
 979     } else
 980 #endif
 981     {
 982       assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
 983       load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
 984     }
 985     break;
 986   default:
 987     ShouldNotReachHere();
 988     break;
 989   }
 990   assert(load != nullptr, "LoadNode should have been created");
 991   if (unaligned) {
 992     load->set_unaligned_access();
 993   }
 994   if (mismatched) {
 995     load->set_mismatched_access();
 996   }
 997   if (unsafe) {
 998     load->set_unsafe_access();
 999   }
1000   load->set_barrier_data(barrier_data);
1001   if (load->Opcode() == Op_LoadN) {
1002     Node* ld = gvn.transform(load);
1003     return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1004   }
1005 
1006   return load;
1007 }
1008 
1009 //------------------------------hash-------------------------------------------
1010 uint LoadNode::hash() const {
1011   // unroll addition of interesting fields
1012   return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1013 }
1014 
1015 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1016   if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1017     bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1018     bool is_stable_ary = FoldStableValues &&
1019                          (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1020                          tp->isa_aryptr()->is_stable();
1021 
1022     return (eliminate_boxing && non_volatile) || is_stable_ary;
1023   }
1024 
1025   return false;
1026 }
1027 
1028 LoadNode* LoadNode::pin_array_access_node() const {
1029   const TypePtr* adr_type = this->adr_type();
1030   if (adr_type != nullptr && adr_type->isa_aryptr()) {
1031     return clone_pinned();
1032   }
1033   return nullptr;
1034 }
1035 
1036 // Is the value loaded previously stored by an arraycopy? If so return
1037 // a load node that reads from the source array so we may be able to
1038 // optimize out the ArrayCopy node later.
1039 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1040   Node* ld_adr = in(MemNode::Address);
1041   intptr_t ld_off = 0;
1042   AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);

1058     if (ac->as_ArrayCopy()->is_clonebasic()) {
1059       assert(ld_alloc != nullptr, "need an alloc");
1060       assert(addp->is_AddP(), "address must be addp");
1061       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1062       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1063       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1064       addp->set_req(AddPNode::Base, src);
1065       addp->set_req(AddPNode::Address, src);
1066     } else {
1067       assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1068              ac->as_ArrayCopy()->is_copyof_validated() ||
1069              ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1070       assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1071       addp->set_req(AddPNode::Base, src);
1072       addp->set_req(AddPNode::Address, src);
1073 
1074       const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1075       BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1076       if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1077 
1078       uint header = arrayOopDesc::base_offset_in_bytes(ary_elem);
1079       uint shift  = exact_log2(type2aelembytes(ary_elem));
1080 
1081       Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1082 #ifdef _LP64
1083       diff = phase->transform(new ConvI2LNode(diff));
1084 #endif
1085       diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1086 
1087       Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1088       addp->set_req(AddPNode::Offset, offset);
1089     }
1090     addp = phase->transform(addp);
1091 #ifdef ASSERT
1092     const TypePtr* adr_type = phase->type(addp)->is_ptr();
1093     ld->_adr_type = adr_type;
1094 #endif
1095     ld->set_req(MemNode::Address, addp);
1096     ld->set_req(0, ctl);
1097     ld->set_req(MemNode::Memory, mem);
1098     return ld;
1099   }
1100   return nullptr;
1101 }
1102 










1103 
1104 //---------------------------can_see_stored_value------------------------------
1105 // This routine exists to make sure this set of tests is done the same
1106 // everywhere.  We need to make a coordinated change: first LoadNode::Ideal
1107 // will change the graph shape in a way which makes memory alive twice at the
1108 // same time (uses the Oracle model of aliasing), then some
1109 // LoadXNode::Identity will fold things back to the equivalence-class model
1110 // of aliasing.
1111 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1112   Node* ld_adr = in(MemNode::Address);
1113   intptr_t ld_off = 0;
1114   Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);









1115   Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1116   const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1117   Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1118   // This is more general than load from boxing objects.
1119   if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1120     uint alias_idx = atp->index();
1121     Node* result = nullptr;
1122     Node* current = st;
1123     // Skip through chains of MemBarNodes checking the MergeMems for
1124     // new states for the slice of this load.  Stop once any other
1125     // kind of node is encountered.  Loads from final memory can skip
1126     // through any kind of MemBar but normal loads shouldn't skip
1127     // through MemBarAcquire since the could allow them to move out of
1128     // a synchronized region. It is not safe to step over MemBarCPUOrder,
1129     // because alias info above them may be inaccurate (e.g., due to
1130     // mixed/mismatched unsafe accesses).
1131     bool is_final_mem = !atp->is_rewritable();
1132     while (current->is_Proj()) {
1133       int opc = current->in(0)->Opcode();
1134       if ((is_final_mem && (opc == Op_MemBarAcquire ||

1178         // Same base, same offset.
1179         // Possible improvement for arrays: check index value instead of absolute offset.
1180 
1181         // At this point we have proven something like this setup:
1182         //   B = << base >>
1183         //   L =  LoadQ(AddP(Check/CastPP(B), #Off))
1184         //   S = StoreQ(AddP(             B , #Off), V)
1185         // (Actually, we haven't yet proven the Q's are the same.)
1186         // In other words, we are loading from a casted version of
1187         // the same pointer-and-offset that we stored to.
1188         // Casted version may carry a dependency and it is respected.
1189         // Thus, we are able to replace L by V.
1190       }
1191       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1192       if (store_Opcode() != st->Opcode()) {
1193         return nullptr;
1194       }
1195       // LoadVector/StoreVector needs additional check to ensure the types match.
1196       if (st->is_StoreVector()) {
1197         const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
1198         const TypeVect* out_vt = as_LoadVector()->vect_type();
1199         if (in_vt != out_vt) {
1200           return nullptr;
1201         }
1202       }
1203       return st->in(MemNode::ValueIn);
1204     }
1205 
1206     // A load from a freshly-created object always returns zero.
1207     // (This can happen after LoadNode::Ideal resets the load's memory input
1208     // to find_captured_store, which returned InitializeNode::zero_memory.)
1209     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1210         (st->in(0) == ld_alloc) &&
1211         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1212       // return a zero value for the load's basic type
1213       // (This is one of the few places where a generic PhaseTransform
1214       // can create new nodes.  Think of it as lazily manifesting
1215       // virtually pre-existing constants.)







1216       if (value_basic_type() != T_VOID) {
1217         if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1218           // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1219           // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1220           // by the ArrayCopyNode.
1221           return phase->zerocon(value_basic_type());
1222         }
1223       } else {
1224         // TODO: materialize all-zero vector constant
1225         assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1226       }
1227     }
1228 
1229     // A load from an initialization barrier can match a captured store.
1230     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1231       InitializeNode* init = st->in(0)->as_Initialize();
1232       AllocateNode* alloc = init->allocation();
1233       if ((alloc != nullptr) && (alloc == ld_alloc)) {
1234         // examine a captured store value
1235         st = init->find_captured_store(ld_off, memory_size(), phase);

1248       base = bs->step_over_gc_barrier(base);
1249       if (base != nullptr && base->is_Proj() &&
1250           base->as_Proj()->_con == TypeFunc::Parms &&
1251           base->in(0)->is_CallStaticJava() &&
1252           base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1253         return base->in(0)->in(TypeFunc::Parms);
1254       }
1255     }
1256 
1257     break;
1258   }
1259 
1260   return nullptr;
1261 }
1262 
1263 //----------------------is_instance_field_load_with_local_phi------------------
1264 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1265   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1266       in(Address)->is_AddP() ) {
1267     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1268     // Only instances and boxed values.
1269     if( t_oop != nullptr &&
1270         (t_oop->is_ptr_to_boxed_value() ||
1271          t_oop->is_known_instance_field()) &&
1272         t_oop->offset() != Type::OffsetBot &&
1273         t_oop->offset() != Type::OffsetTop) {
1274       return true;
1275     }
1276   }
1277   return false;
1278 }
1279 
1280 //------------------------------Identity---------------------------------------
1281 // Loads are identity if previous store is to same address
1282 Node* LoadNode::Identity(PhaseGVN* phase) {
1283   // If the previous store-maker is the right kind of Store, and the store is
1284   // to the same address, then we are equal to the value stored.
1285   Node* mem = in(Memory);
1286   Node* value = can_see_stored_value(mem, phase);
1287   if( value ) {
1288     // byte, short & char stores truncate naturally.
1289     // A load has to load the truncated value which requires
1290     // some sort of masking operation and that requires an

1299     // usually runs first, producing the singleton type of the Con.)
1300     if (!has_pinned_control_dependency() || value->is_Con()) {
1301       return value;
1302     } else {
1303       return this;
1304     }
1305   }
1306 
1307   if (has_pinned_control_dependency()) {
1308     return this;
1309   }
1310   // Search for an existing data phi which was generated before for the same
1311   // instance's field to avoid infinite generation of phis in a loop.
1312   Node *region = mem->in(0);
1313   if (is_instance_field_load_with_local_phi(region)) {
1314     const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1315     int this_index  = phase->C->get_alias_index(addr_t);
1316     int this_offset = addr_t->offset();
1317     int this_iid    = addr_t->instance_id();
1318     if (!addr_t->is_known_instance() &&
1319          addr_t->is_ptr_to_boxed_value()) {
1320       // Use _idx of address base (could be Phi node) for boxed values.
1321       intptr_t   ignore = 0;
1322       Node*      base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1323       if (base == nullptr) {
1324         return this;
1325       }
1326       this_iid = base->_idx;
1327     }
1328     const Type* this_type = bottom_type();
1329     for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1330       Node* phi = region->fast_out(i);
1331       if (phi->is_Phi() && phi != mem &&
1332           phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1333         return phi;
1334       }
1335     }
1336   }
1337 
1338   return this;
1339 }
1340 

1856   bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
1857          phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
1858 
1859   // Skip up past a SafePoint control.  Cannot do this for Stores because
1860   // pointer stores & cardmarks must stay on the same side of a SafePoint.
1861   if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
1862       phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw  &&
1863       !addr_mark &&
1864       (depends_only_on_test() || has_unknown_control_dependency())) {
1865     ctrl = ctrl->in(0);
1866     set_req(MemNode::Control,ctrl);
1867     progress = true;
1868   }
1869 
1870   intptr_t ignore = 0;
1871   Node*    base   = AddPNode::Ideal_base_and_offset(address, phase, ignore);
1872   if (base != nullptr
1873       && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
1874     // Check for useless control edge in some common special cases
1875     if (in(MemNode::Control) != nullptr

1876         && can_remove_control()
1877         && phase->type(base)->higher_equal(TypePtr::NOTNULL)
1878         && all_controls_dominate(base, phase->C->start())) {
1879       // A method-invariant, non-null address (constant or 'this' argument).
1880       set_req(MemNode::Control, nullptr);
1881       progress = true;
1882     }
1883   }
1884 
1885   Node* mem = in(MemNode::Memory);
1886   const TypePtr *addr_t = phase->type(address)->isa_ptr();
1887 
1888   if (can_reshape && (addr_t != nullptr)) {
1889     // try to optimize our memory input
1890     Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
1891     if (opt_mem != mem) {
1892       set_req_X(MemNode::Memory, opt_mem, phase);
1893       if (phase->type( opt_mem ) == Type::TOP) return nullptr;
1894       return this;
1895     }

1952   // fold up, do so.
1953   Node* prev_mem = find_previous_store(phase);
1954   if (prev_mem != nullptr) {
1955     Node* value = can_see_arraycopy_value(prev_mem, phase);
1956     if (value != nullptr) {
1957       return value;
1958     }
1959   }
1960   // Steps (a), (b):  Walk past independent stores to find an exact match.
1961   if (prev_mem != nullptr && prev_mem != in(MemNode::Memory)) {
1962     // (c) See if we can fold up on the spot, but don't fold up here.
1963     // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
1964     // just return a prior value, which is done by Identity calls.
1965     if (can_see_stored_value(prev_mem, phase)) {
1966       // Make ready for step (d):
1967       set_req_X(MemNode::Memory, prev_mem, phase);
1968       return this;
1969     }
1970   }
1971 
1972   return progress ? this : nullptr;







1973 }
1974 
1975 // Helper to recognize certain Klass fields which are invariant across
1976 // some group of array types (e.g., int[] or all T[] where T < Object).
1977 const Type*
1978 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
1979                                  ciKlass* klass) const {
1980   assert(!UseCompactObjectHeaders || tkls->offset() != in_bytes(Klass::prototype_header_offset()),
1981          "must not happen");
1982   if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
1983     // The field is Klass::_access_flags.  Return its (constant) value.
1984     assert(Opcode() == Op_LoadUS, "must load an unsigned short from _access_flags");
1985     return TypeInt::make(klass->access_flags());
1986   }
1987   if (tkls->offset() == in_bytes(Klass::misc_flags_offset())) {
1988     // The field is Klass::_misc_flags.  Return its (constant) value.
1989     assert(Opcode() == Op_LoadUB, "must load an unsigned byte from _misc_flags");
1990     return TypeInt::make(klass->misc_flags());
1991   }
1992   if (tkls->offset() == in_bytes(Klass::layout_helper_offset())) {

2052       }
2053     }
2054 
2055     // Don't do this for integer types. There is only potential profit if
2056     // the element type t is lower than _type; that is, for int types, if _type is
2057     // more restrictive than t.  This only happens here if one is short and the other
2058     // char (both 16 bits), and in those cases we've made an intentional decision
2059     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2060     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2061     //
2062     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2063     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
2064     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2065     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
2066     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2067     // In fact, that could have been the original type of p1, and p1 could have
2068     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2069     // expression (LShiftL quux 3) independently optimized to the constant 8.
2070     if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2071         && (_type->isa_vect() == nullptr)

2072         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2073       // t might actually be lower than _type, if _type is a unique
2074       // concrete subclass of abstract class t.
2075       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
2076         const Type* jt = t->join_speculative(_type);
2077         // In any case, do not allow the join, per se, to empty out the type.
2078         if (jt->empty() && !t->empty()) {
2079           // This can happen if a interface-typed array narrows to a class type.
2080           jt = _type;
2081         }
2082 #ifdef ASSERT
2083         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2084           // The pointers in the autobox arrays are always non-null
2085           Node* base = adr->in(AddPNode::Base);
2086           if ((base != nullptr) && base->is_DecodeN()) {
2087             // Get LoadN node which loads IntegerCache.cache field
2088             base = base->in(1);
2089           }
2090           if ((base != nullptr) && base->is_Con()) {
2091             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2092             if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2093               // It could be narrow oop
2094               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2095             }
2096           }
2097         }
2098 #endif
2099         return jt;
2100       }
2101     }
2102   } else if (tp->base() == Type::InstPtr) {
2103     assert( off != Type::OffsetBot ||
2104             // arrays can be cast to Objects
2105             !tp->isa_instptr() ||
2106             tp->is_instptr()->instance_klass()->is_java_lang_Object() ||


2107             // unsafe field access may not have a constant offset
2108             C->has_unsafe_access(),
2109             "Field accesses must be precise" );
2110     // For oop loads, we expect the _type to be precise.
2111 
2112     // Optimize loads from constant fields.
2113     const TypeInstPtr* tinst = tp->is_instptr();



2114     ciObject* const_oop = tinst->const_oop();
2115     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2116       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), value_basic_type());
2117       if (con_type != nullptr) {
2118         return con_type;
2119       }
2120     }
2121   } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2122     assert(off != Type::OffsetBot ||
2123             !tp->isa_instklassptr() ||
2124            // arrays can be cast to Objects
2125            tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2126            // also allow array-loading from the primary supertype
2127            // array during subtype checks
2128            Opcode() == Op_LoadKlass,
2129            "Field accesses must be precise");
2130     // For klass/static loads, we expect the _type to be precise
2131   } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2132     /* With mirrors being an indirect in the Klass*
2133      * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2134      * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2135      *
2136      * So check the type and klass of the node before the LoadP.

2143         assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2144         assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2145         return TypeInstPtr::make(klass->java_mirror());
2146       }
2147     }
2148   }
2149 
2150   const TypeKlassPtr *tkls = tp->isa_klassptr();
2151   if (tkls != nullptr) {
2152     if (tkls->is_loaded() && tkls->klass_is_exact()) {
2153       ciKlass* klass = tkls->exact_klass();
2154       // We are loading a field from a Klass metaobject whose identity
2155       // is known at compile time (the type is "exact" or "precise").
2156       // Check for fields we know are maintained as constants by the VM.
2157       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2158         // The field is Klass::_super_check_offset.  Return its (constant) value.
2159         // (Folds up type checking code.)
2160         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2161         return TypeInt::make(klass->super_check_offset());
2162       }
2163       if (UseCompactObjectHeaders) {
2164         if (tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2165           // The field is Klass::_prototype_header. Return its (constant) value.
2166           assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2167           return TypeX::make(klass->prototype_header());
2168         }






2169       }
2170       // Compute index into primary_supers array
2171       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2172       // Check for overflowing; use unsigned compare to handle the negative case.
2173       if( depth < ciKlass::primary_super_limit() ) {
2174         // The field is an element of Klass::_primary_supers.  Return its (constant) value.
2175         // (Folds up type checking code.)
2176         assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2177         ciKlass *ss = klass->super_of_depth(depth);
2178         return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2179       }
2180       const Type* aift = load_array_final_field(tkls, klass);
2181       if (aift != nullptr)  return aift;
2182     }
2183 
2184     // We can still check if we are loading from the primary_supers array at a
2185     // shallow enough depth.  Even though the klass is not exact, entries less
2186     // than or equal to its super depth are correct.
2187     if (tkls->is_loaded()) {
2188       ciKlass* klass = nullptr;

2219         !tkls->is_instklassptr()->might_be_an_array() // not the supertype of all T[] (java.lang.Object) or has an interface that is not Serializable or Cloneable
2220     ) {
2221       assert(Opcode() == Op_LoadI, "must load an int from _layout_helper");
2222       jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2223       // The key property of this type is that it folds up tests
2224       // for array-ness, since it proves that the layout_helper is positive.
2225       // Thus, a generic value like the basic object layout helper works fine.
2226       return TypeInt::make(min_size, max_jint, Type::WidenMin);
2227     }
2228   }
2229 
2230   bool is_vect = (_type->isa_vect() != nullptr);
2231   if (is_instance && !is_vect) {
2232     // If we have an instance type and our memory input is the
2233     // programs's initial memory state, there is no matching store,
2234     // so just return a zero of the appropriate type -
2235     // except if it is vectorized - then we have no zero constant.
2236     Node *mem = in(MemNode::Memory);
2237     if (mem->is_Parm() && mem->in(0)->is_Start()) {
2238       assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");












2239       return Type::get_zero_type(_type->basic_type());
2240     }
2241   }
2242 
2243   if (!UseCompactObjectHeaders) {
2244     Node* alloc = is_new_object_mark_load();
2245     if (alloc != nullptr) {
2246       return TypeX::make(markWord::prototype().value());









2247     }
2248   }
2249 
2250   return _type;
2251 }
2252 
2253 //------------------------------match_edge-------------------------------------
2254 // Do we Match on this edge index or not?  Match only the address.
2255 uint LoadNode::match_edge(uint idx) const {
2256   return idx == MemNode::Address;
2257 }
2258 
2259 //--------------------------LoadBNode::Ideal--------------------------------------
2260 //
2261 //  If the previous store is to the same address as this load,
2262 //  and the value stored was larger than a byte, replace this load
2263 //  with the value stored truncated to a byte.  If no truncation is
2264 //  needed, the replacement is done in LoadNode::Identity().
2265 //
2266 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {

2375     }
2376   }
2377   // Identity call will handle the case where truncation is not needed.
2378   return LoadNode::Ideal(phase, can_reshape);
2379 }
2380 
2381 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2382   Node* mem = in(MemNode::Memory);
2383   Node* value = can_see_stored_value(mem,phase);
2384   if (value != nullptr && value->is_Con() &&
2385       !value->bottom_type()->higher_equal(_type)) {
2386     // If the input to the store does not fit with the load's result type,
2387     // it must be truncated. We can't delay until Ideal call since
2388     // a singleton Value is needed for split_thru_phi optimization.
2389     int con = value->get_int();
2390     return TypeInt::make((con << 16) >> 16);
2391   }
2392   return LoadNode::Value(phase);
2393 }
2394 













2395 //=============================================================================
2396 //----------------------------LoadKlassNode::make------------------------------
2397 // Polymorphic factory method:
2398 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2399   // sanity check the alias category against the created node type
2400   const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2401   assert(adr_type != nullptr, "expecting TypeKlassPtr");
2402 #ifdef _LP64
2403   if (adr_type->is_ptr_to_narrowklass()) {
2404     assert(UseCompressedClassPointers, "no compressed klasses");
2405     Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2406     return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2407   }
2408 #endif
2409   assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2410   return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2411 }
2412 
2413 //------------------------------Value------------------------------------------
2414 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {

2448           }
2449           return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2450         }
2451         if (!t->is_klass()) {
2452           // a primitive Class (e.g., int.class) has null for a klass field
2453           return TypePtr::NULL_PTR;
2454         }
2455         // Fold up the load of the hidden field
2456         return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2457       }
2458       // non-constant mirror, so we can't tell what's going on
2459     }
2460     if (!tinst->is_loaded())
2461       return _type;             // Bail out if not loaded
2462     if (offset == oopDesc::klass_offset_in_bytes()) {
2463       return tinst->as_klass_type(true);
2464     }
2465   }
2466 
2467   // Check for loading klass from an array
2468   const TypeAryPtr *tary = tp->isa_aryptr();
2469   if (tary != nullptr &&
2470       tary->offset() == oopDesc::klass_offset_in_bytes()) {
2471     return tary->as_klass_type(true);


2472   }
2473 
2474   // Check for loading klass from an array klass
2475   const TypeKlassPtr *tkls = tp->isa_klassptr();
2476   if (tkls != nullptr && !StressReflectiveCode) {
2477     if (!tkls->is_loaded())
2478      return _type;             // Bail out if not loaded
2479     if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2480         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2481       // // Always returning precise element type is incorrect,
2482       // // e.g., element type could be object and array may contain strings
2483       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2484 
2485       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2486       // according to the element type's subclassing.
2487       return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2488     }
2489     if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2490         tkls->offset() == in_bytes(Klass::super_offset())) {
2491       ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();

2531     base = bs->step_over_gc_barrier(base);
2532   }
2533 
2534   // We can fetch the klass directly through an AllocateNode.
2535   // This works even if the klass is not constant (clone or newArray).
2536   if (offset == oopDesc::klass_offset_in_bytes()) {
2537     Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2538     if (allocated_klass != nullptr) {
2539       return allocated_klass;
2540     }
2541   }
2542 
2543   // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2544   // See inline_native_Class_query for occurrences of these patterns.
2545   // Java Example:  x.getClass().isAssignableFrom(y)
2546   //
2547   // This improves reflective code, often making the Class
2548   // mirror go completely dead.  (Current exception:  Class
2549   // mirrors may appear in debug info, but we could clean them out by
2550   // introducing a new debug info operator for Klass.java_mirror).




2551 
2552   if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2553       && offset == java_lang_Class::klass_offset()) {
2554     if (base->is_Load()) {
2555       Node* base2 = base->in(MemNode::Address);
2556       if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2557         Node* adr2 = base2->in(MemNode::Address);
2558         const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2559         if (tkls != nullptr && !tkls->empty()
2560             && (tkls->isa_instklassptr() || tkls->isa_aryklassptr())
2561             && adr2->is_AddP()
2562            ) {
2563           int mirror_field = in_bytes(Klass::java_mirror_offset());
2564           if (tkls->offset() == mirror_field) {
2565             return adr2->in(AddPNode::Base);
2566           }
2567         }
2568       }
2569     }
2570   }
2571 
2572   return this;
2573 }
2574 
2575 LoadNode* LoadNode::clone_pinned() const {
2576   LoadNode* ld = clone()->as_Load();
2577   ld->_control_dependency = UnknownControl;
2578   return ld;
2579 }
2580 
2581 
2582 //------------------------------Value------------------------------------------

2687 //---------------------------StoreNode::make-----------------------------------
2688 // Polymorphic factory method:
2689 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2690   assert((mo == unordered || mo == release), "unexpected");
2691   Compile* C = gvn.C;
2692   assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2693          ctl != nullptr, "raw memory operations should have control edge");
2694 
2695   switch (bt) {
2696   case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2697   case T_BYTE:    return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2698   case T_INT:     return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2699   case T_CHAR:
2700   case T_SHORT:   return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2701   case T_LONG:    return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2702   case T_FLOAT:   return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2703   case T_DOUBLE:  return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2704   case T_METADATA:
2705   case T_ADDRESS:
2706   case T_OBJECT:

2707 #ifdef _LP64
2708     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2709       val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2710       return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2711     } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2712                (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2713                 adr->bottom_type()->isa_rawptr())) {
2714       val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2715       return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2716     }
2717 #endif
2718     {
2719       return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2720     }
2721   default:
2722     ShouldNotReachHere();
2723     return (StoreNode*)nullptr;
2724   }
2725 }
2726 
2727 //--------------------------bottom_type----------------------------------------
2728 const Type *StoreNode::bottom_type() const {
2729   return Type::MEMORY;
2730 }
2731 
2732 //------------------------------hash-------------------------------------------
2733 uint StoreNode::hash() const {
2734   // unroll addition of interesting fields
2735   //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2736 
2737   // Since they are not commoned, do not hash them:
2738   return NO_HASH;
2739 }
2740 
2741 // Link together multiple stores (B/S/C/I) into a longer one.
2742 //

3364   }
3365   ss.print_cr("[TraceMergeStores]: with");
3366   merged_input_value->dump("\n", false, &ss);
3367   merged_store->dump("\n", false, &ss);
3368   tty->print("%s", ss.as_string());
3369 }
3370 #endif
3371 
3372 //------------------------------Ideal------------------------------------------
3373 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3374 // When a store immediately follows a relevant allocation/initialization,
3375 // try to capture it into the initialization, or hoist it above.
3376 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3377   Node* p = MemNode::Ideal_common(phase, can_reshape);
3378   if (p)  return (p == NodeSentinel) ? nullptr : p;
3379 
3380   Node* mem     = in(MemNode::Memory);
3381   Node* address = in(MemNode::Address);
3382   Node* value   = in(MemNode::ValueIn);
3383   // Back-to-back stores to same address?  Fold em up.  Generally
3384   // unsafe if I have intervening uses.
3385   {
3386     Node* st = mem;
3387     // If Store 'st' has more than one use, we cannot fold 'st' away.
3388     // For example, 'st' might be the final state at a conditional
3389     // return.  Or, 'st' might be used by some node which is live at
3390     // the same time 'st' is live, which might be unschedulable.  So,
3391     // require exactly ONE user until such time as we clone 'mem' for
3392     // each of 'mem's uses (thus making the exactly-1-user-rule hold
3393     // true).
3394     while (st->is_Store() && st->outcnt() == 1) {
3395       // Looking at a dead closed cycle of memory?
3396       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3397       assert(Opcode() == st->Opcode() ||
3398              st->Opcode() == Op_StoreVector ||
3399              Opcode() == Op_StoreVector ||
3400              st->Opcode() == Op_StoreVectorScatter ||
3401              Opcode() == Op_StoreVectorScatter ||
3402              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3403              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3404              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy


3405              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3406              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3407 
3408       if (st->in(MemNode::Address)->eqv_uncast(address) &&
3409           st->as_Store()->memory_size() <= this->memory_size()) {
3410         Node* use = st->raw_out(0);
3411         if (phase->is_IterGVN()) {
3412           phase->is_IterGVN()->rehash_node_delayed(use);
3413         }
3414         // It's OK to do this in the parser, since DU info is always accurate,
3415         // and the parser always refers to nodes via SafePointNode maps.
3416         use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3417         return this;
3418       }
3419       st = st->in(MemNode::Memory);
3420     }
3421   }
3422 
3423 
3424   // Capture an unaliased, unconditional, simple store into an initializer.

3522       const StoreVectorNode* store_vector = as_StoreVector();
3523       const StoreVectorNode* mem_vector = mem->as_StoreVector();
3524       const Node* store_indices = store_vector->indices();
3525       const Node* mem_indices = mem_vector->indices();
3526       const Node* store_mask = store_vector->mask();
3527       const Node* mem_mask = mem_vector->mask();
3528       // Ensure types, indices, and masks match
3529       if (store_vector->vect_type() == mem_vector->vect_type() &&
3530           ((store_indices == nullptr) == (mem_indices == nullptr) &&
3531            (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3532           ((store_mask == nullptr) == (mem_mask == nullptr) &&
3533            (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3534         result = mem;
3535       }
3536     }
3537   }
3538 
3539   // Store of zero anywhere into a freshly-allocated object?
3540   // Then the store is useless.
3541   // (It must already have been captured by the InitializeNode.)
3542   if (result == this &&
3543       ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
3544     // a newly allocated object is already all-zeroes everywhere
3545     if (mem->is_Proj() && mem->in(0)->is_Allocate()) {

3546       result = mem;
3547     }
3548 
3549     if (result == this) {
3550       // the store may also apply to zero-bits in an earlier object
3551       Node* prev_mem = find_previous_store(phase);
3552       // Steps (a), (b):  Walk past independent stores to find an exact match.
3553       if (prev_mem != nullptr) {
3554         Node* prev_val = can_see_stored_value(prev_mem, phase);
3555         if (prev_val != nullptr && prev_val == val) {
3556           // prev_val and val might differ by a cast; it would be good
3557           // to keep the more informative of the two.
3558           result = mem;
3559         }
3560       }
3561     }
3562   }
3563 
3564   PhaseIterGVN* igvn = phase->is_IterGVN();
3565   if (result != this && igvn != nullptr) {
3566     MemBarNode* trailing = trailing_membar();
3567     if (trailing != nullptr) {
3568 #ifdef ASSERT
3569       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();

4033 // Clearing a short array is faster with stores
4034 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4035   // Already know this is a large node, do not try to ideal it
4036   if (_is_large) return nullptr;
4037 
4038   const int unit = BytesPerLong;
4039   const TypeX* t = phase->type(in(2))->isa_intptr_t();
4040   if (!t)  return nullptr;
4041   if (!t->is_con())  return nullptr;
4042   intptr_t raw_count = t->get_con();
4043   intptr_t size = raw_count;
4044   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4045   // Clearing nothing uses the Identity call.
4046   // Negative clears are possible on dead ClearArrays
4047   // (see jck test stmt114.stmt11402.val).
4048   if (size <= 0 || size % unit != 0)  return nullptr;
4049   intptr_t count = size / unit;
4050   // Length too long; communicate this to matchers and assemblers.
4051   // Assemblers are responsible to produce fast hardware clears for it.
4052   if (size > InitArrayShortSize) {
4053     return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
4054   } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4055     return nullptr;
4056   }
4057   if (!IdealizeClearArrayNode) return nullptr;
4058   Node *mem = in(1);
4059   if( phase->type(mem)==Type::TOP ) return nullptr;
4060   Node *adr = in(3);
4061   const Type* at = phase->type(adr);
4062   if( at==Type::TOP ) return nullptr;
4063   const TypePtr* atp = at->isa_ptr();
4064   // adjust atp to be the correct array element address type
4065   if (atp == nullptr)  atp = TypePtr::BOTTOM;
4066   else              atp = atp->add_offset(Type::OffsetBot);
4067   // Get base for derived pointer purposes
4068   if( adr->Opcode() != Op_AddP ) Unimplemented();
4069   Node *base = adr->in(1);
4070 
4071   Node *zero = phase->makecon(TypeLong::ZERO);
4072   Node *off  = phase->MakeConX(BytesPerLong);
4073   mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4074   count--;
4075   while( count-- ) {
4076     mem = phase->transform(mem);
4077     adr = phase->transform(new AddPNode(base,adr,off));
4078     mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
4079   }
4080   return mem;
4081 }
4082 
4083 //----------------------------step_through----------------------------------
4084 // Return allocation input memory edge if it is different instance
4085 // or itself if it is the one we are looking for.
4086 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4087   Node* n = *np;
4088   assert(n->is_ClearArray(), "sanity");
4089   intptr_t offset;
4090   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4091   // This method is called only before Allocate nodes are expanded
4092   // during macro nodes expansion. Before that ClearArray nodes are
4093   // only generated in PhaseMacroExpand::generate_arraycopy() (before
4094   // Allocate nodes are expanded) which follows allocations.
4095   assert(alloc != nullptr, "should have allocation");
4096   if (alloc->_idx == instance_id) {
4097     // Can not bypass initialization of the instance we are looking for.
4098     return false;
4099   }
4100   // Otherwise skip it.
4101   InitializeNode* init = alloc->initialization();
4102   if (init != nullptr)
4103     *np = init->in(TypeFunc::Memory);
4104   else
4105     *np = alloc->in(TypeFunc::Memory);
4106   return true;
4107 }
4108 
4109 //----------------------------clear_memory-------------------------------------
4110 // Generate code to initialize object storage to zero.
4111 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


4112                                    intptr_t start_offset,
4113                                    Node* end_offset,
4114                                    PhaseGVN* phase) {
4115   intptr_t offset = start_offset;
4116 
4117   int unit = BytesPerLong;
4118   if ((offset % unit) != 0) {
4119     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
4120     adr = phase->transform(adr);
4121     const TypePtr* atp = TypeRawPtr::BOTTOM;
4122     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






4123     mem = phase->transform(mem);
4124     offset += BytesPerInt;
4125   }
4126   assert((offset % unit) == 0, "");
4127 
4128   // Initialize the remaining stuff, if any, with a ClearArray.
4129   return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
4130 }
4131 
4132 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,

4133                                    Node* start_offset,
4134                                    Node* end_offset,
4135                                    PhaseGVN* phase) {
4136   if (start_offset == end_offset) {
4137     // nothing to do
4138     return mem;
4139   }
4140 
4141   int unit = BytesPerLong;
4142   Node* zbase = start_offset;
4143   Node* zend  = end_offset;
4144 
4145   // Scale to the unit required by the CPU:
4146   if (!Matcher::init_array_count_is_in_bytes) {
4147     Node* shift = phase->intcon(exact_log2(unit));
4148     zbase = phase->transform(new URShiftXNode(zbase, shift) );
4149     zend  = phase->transform(new URShiftXNode(zend,  shift) );
4150   }
4151 
4152   // Bulk clear double-words
4153   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4154   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
4155   mem = new ClearArrayNode(ctl, mem, zsize, adr, false);



4156   return phase->transform(mem);
4157 }
4158 
4159 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,


4160                                    intptr_t start_offset,
4161                                    intptr_t end_offset,
4162                                    PhaseGVN* phase) {
4163   if (start_offset == end_offset) {
4164     // nothing to do
4165     return mem;
4166   }
4167 
4168   assert((end_offset % BytesPerInt) == 0, "odd end offset");
4169   intptr_t done_offset = end_offset;
4170   if ((done_offset % BytesPerLong) != 0) {
4171     done_offset -= BytesPerInt;
4172   }
4173   if (done_offset > start_offset) {
4174     mem = clear_memory(ctl, mem, dest,
4175                        start_offset, phase->MakeConX(done_offset), phase);
4176   }
4177   if (done_offset < end_offset) { // emit the final 32-bit store
4178     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
4179     adr = phase->transform(adr);
4180     const TypePtr* atp = TypeRawPtr::BOTTOM;
4181     mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);






4182     mem = phase->transform(mem);
4183     done_offset += BytesPerInt;
4184   }
4185   assert(done_offset == end_offset, "");
4186   return mem;
4187 }
4188 
4189 //=============================================================================
4190 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4191   : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4192     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4193 #ifdef ASSERT
4194   , _pair_idx(0)
4195 #endif
4196 {
4197   init_class_id(Class_MemBar);
4198   Node* top = C->top();
4199   init_req(TypeFunc::I_O,top);
4200   init_req(TypeFunc::FramePtr,top);
4201   init_req(TypeFunc::ReturnAdr,top);

4304       PhaseIterGVN* igvn = phase->is_IterGVN();
4305       remove(igvn);
4306       // Must return either the original node (now dead) or a new node
4307       // (Do not return a top here, since that would break the uniqueness of top.)
4308       return new ConINode(TypeInt::ZERO);
4309     }
4310   }
4311   return progress ? this : nullptr;
4312 }
4313 
4314 //------------------------------Value------------------------------------------
4315 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4316   if( !in(0) ) return Type::TOP;
4317   if( phase->type(in(0)) == Type::TOP )
4318     return Type::TOP;
4319   return TypeTuple::MEMBAR;
4320 }
4321 
4322 //------------------------------match------------------------------------------
4323 // Construct projections for memory.
4324 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
4325   switch (proj->_con) {
4326   case TypeFunc::Control:
4327   case TypeFunc::Memory:
4328     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4329   }
4330   ShouldNotReachHere();
4331   return nullptr;
4332 }
4333 
4334 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4335   trailing->_kind = TrailingStore;
4336   leading->_kind = LeadingStore;
4337 #ifdef ASSERT
4338   trailing->_pair_idx = leading->_idx;
4339   leading->_pair_idx = leading->_idx;
4340 #endif
4341 }
4342 
4343 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4344   trailing->_kind = TrailingLoadStore;

4591   return (req() > RawStores);
4592 }
4593 
4594 void InitializeNode::set_complete(PhaseGVN* phase) {
4595   assert(!is_complete(), "caller responsibility");
4596   _is_complete = Complete;
4597 
4598   // After this node is complete, it contains a bunch of
4599   // raw-memory initializations.  There is no need for
4600   // it to have anything to do with non-raw memory effects.
4601   // Therefore, tell all non-raw users to re-optimize themselves,
4602   // after skipping the memory effects of this initialization.
4603   PhaseIterGVN* igvn = phase->is_IterGVN();
4604   if (igvn)  igvn->add_users_to_worklist(this);
4605 }
4606 
4607 // convenience function
4608 // return false if the init contains any stores already
4609 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4610   InitializeNode* init = initialization();
4611   if (init == nullptr || init->is_complete())  return false;


4612   init->remove_extra_zeroes();
4613   // for now, if this allocation has already collected any inits, bail:
4614   if (init->is_non_zero())  return false;
4615   init->set_complete(phase);
4616   return true;
4617 }
4618 
4619 void InitializeNode::remove_extra_zeroes() {
4620   if (req() == RawStores)  return;
4621   Node* zmem = zero_memory();
4622   uint fill = RawStores;
4623   for (uint i = fill; i < req(); i++) {
4624     Node* n = in(i);
4625     if (n->is_top() || n == zmem)  continue;  // skip
4626     if (fill < i)  set_req(fill, n);          // compact
4627     ++fill;
4628   }
4629   // delete any empty spaces created:
4630   while (fill < req()) {
4631     del_req(fill);

4775             // store node that we'd like to capture. We need to check
4776             // the uses of the MergeMemNode.
4777             mems.push(n);
4778           }
4779         } else if (n->is_Mem()) {
4780           Node* other_adr = n->in(MemNode::Address);
4781           if (other_adr == adr) {
4782             failed = true;
4783             break;
4784           } else {
4785             const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
4786             if (other_t_adr != nullptr) {
4787               int other_alias_idx = phase->C->get_alias_index(other_t_adr);
4788               if (other_alias_idx == alias_idx) {
4789                 // A load from the same memory slice as the store right
4790                 // after the InitializeNode. We check the control of the
4791                 // object/array that is loaded from. If it's the same as
4792                 // the store control then we cannot capture the store.
4793                 assert(!n->is_Store(), "2 stores to same slice on same control?");
4794                 Node* base = other_adr;






4795                 assert(base->is_AddP(), "should be addp but is %s", base->Name());
4796                 base = base->in(AddPNode::Base);
4797                 if (base != nullptr) {
4798                   base = base->uncast();
4799                   if (base->is_Proj() && base->in(0) == alloc) {
4800                     failed = true;
4801                     break;
4802                   }
4803                 }
4804               }
4805             }
4806           }
4807         } else {
4808           failed = true;
4809           break;
4810         }
4811       }
4812     }
4813   }
4814   if (failed) {

5361         //   z's_done      12  16  16  16    12  16    12
5362         //   z's_needed    12  16  16  16    16  16    16
5363         //   zsize          0   0   0   0     4   0     4
5364         if (next_full_store < 0) {
5365           // Conservative tack:  Zero to end of current word.
5366           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5367         } else {
5368           // Zero to beginning of next fully initialized word.
5369           // Or, don't zero at all, if we are already in that word.
5370           assert(next_full_store >= zeroes_needed, "must go forward");
5371           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5372           zeroes_needed = next_full_store;
5373         }
5374       }
5375 
5376       if (zeroes_needed > zeroes_done) {
5377         intptr_t zsize = zeroes_needed - zeroes_done;
5378         // Do some incremental zeroing on rawmem, in parallel with inits.
5379         zeroes_done = align_down(zeroes_done, BytesPerInt);
5380         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


5381                                               zeroes_done, zeroes_needed,
5382                                               phase);
5383         zeroes_done = zeroes_needed;
5384         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5385           do_zeroing = false;   // leave the hole, next time
5386       }
5387     }
5388 
5389     // Collect the store and move on:
5390     phase->replace_input_of(st, MemNode::Memory, inits);
5391     inits = st;                 // put it on the linearized chain
5392     set_req(i, zmem);           // unhook from previous position
5393 
5394     if (zeroes_done == st_off)
5395       zeroes_done = next_init_off;
5396 
5397     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5398 
5399     #ifdef ASSERT
5400     // Various order invariants.  Weaker than stores_are_sane because

5420   remove_extra_zeroes();        // clear out all the zmems left over
5421   add_req(inits);
5422 
5423   if (!(UseTLAB && ZeroTLAB)) {
5424     // If anything remains to be zeroed, zero it all now.
5425     zeroes_done = align_down(zeroes_done, BytesPerInt);
5426     // if it is the last unused 4 bytes of an instance, forget about it
5427     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5428     if (zeroes_done + BytesPerLong >= size_limit) {
5429       AllocateNode* alloc = allocation();
5430       assert(alloc != nullptr, "must be present");
5431       if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5432         Node* klass_node = alloc->in(AllocateNode::KlassNode);
5433         ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5434         if (zeroes_done == k->layout_helper())
5435           zeroes_done = size_limit;
5436       }
5437     }
5438     if (zeroes_done < size_limit) {
5439       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,


5440                                             zeroes_done, size_in_bytes, phase);
5441     }
5442   }
5443 
5444   set_complete(phase);
5445   return rawmem;
5446 }
5447 
5448 
5449 #ifdef ASSERT
5450 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5451   if (is_complete())
5452     return true;                // stores could be anything at this point
5453   assert(allocation() != nullptr, "must be present");
5454   intptr_t last_off = allocation()->minimum_header_size();
5455   for (uint i = InitializeNode::RawStores; i < req(); i++) {
5456     Node* st = in(i);
5457     intptr_t st_off = get_store_offset(st, phase);
5458     if (st_off < 0)  continue;  // ignore dead garbage
5459     if (last_off > st_off) {

   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "ci/ciFlatArrayKlass.hpp"
  27 #include "classfile/javaClasses.hpp"
  28 #include "classfile/systemDictionary.hpp"
  29 #include "compiler/compileLog.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/c2/barrierSetC2.hpp"
  32 #include "gc/shared/tlab_globals.hpp"
  33 #include "memory/allocation.inline.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "oops/flatArrayKlass.hpp"
  36 #include "oops/objArrayKlass.hpp"
  37 #include "opto/addnode.hpp"
  38 #include "opto/arraycopynode.hpp"
  39 #include "opto/cfgnode.hpp"
  40 #include "opto/compile.hpp"
  41 #include "opto/connode.hpp"
  42 #include "opto/convertnode.hpp"
  43 #include "opto/inlinetypenode.hpp"
  44 #include "opto/loopnode.hpp"
  45 #include "opto/machnode.hpp"
  46 #include "opto/matcher.hpp"
  47 #include "opto/memnode.hpp"
  48 #include "opto/mempointer.hpp"
  49 #include "opto/mulnode.hpp"
  50 #include "opto/narrowptrnode.hpp"
  51 #include "opto/phaseX.hpp"
  52 #include "opto/regalloc.hpp"
  53 #include "opto/regmask.hpp"
  54 #include "opto/rootnode.hpp"
  55 #include "opto/traceMergeStoresTag.hpp"
  56 #include "opto/vectornode.hpp"
  57 #include "utilities/align.hpp"
  58 #include "utilities/copy.hpp"
  59 #include "utilities/globalDefinitions.hpp"
  60 #include "utilities/macros.hpp"
  61 #include "utilities/powerOfTwo.hpp"
  62 #include "utilities/vmError.hpp"
  63 
  64 // Portions of code courtesy of Clifford Click
  65 
  66 // Optimization - Graph Style
  67 
  68 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
  69 
  70 //=============================================================================
  71 uint MemNode::size_of() const { return sizeof(*this); }
  72 
  73 const TypePtr *MemNode::adr_type() const {
  74   Node* adr = in(Address);
  75   if (adr == nullptr)  return nullptr; // node is dead
  76   const TypePtr* cross_check = nullptr;
  77   DEBUG_ONLY(cross_check = _adr_type);
  78   return calculate_adr_type(adr->bottom_type(), cross_check);
  79 }

 128       st->print(", idx=Bot;");
 129     else if (atp->index() == Compile::AliasIdxTop)
 130       st->print(", idx=Top;");
 131     else if (atp->index() == Compile::AliasIdxRaw)
 132       st->print(", idx=Raw;");
 133     else {
 134       ciField* field = atp->field();
 135       if (field) {
 136         st->print(", name=");
 137         field->print_name_on(st);
 138       }
 139       st->print(", idx=%d;", atp->index());
 140     }
 141   }
 142 }
 143 
 144 extern void print_alias_types();
 145 
 146 #endif
 147 
 148 // Find the memory output corresponding to the fall-through path of a call
 149 static Node* find_call_fallthrough_mem_output(CallNode* call) {
 150   ResourceMark rm;
 151   CallProjections* projs = call->extract_projections(false, false);
 152   Node* res = projs->fallthrough_memproj;
 153   assert(res != nullptr, "must have a fallthrough mem output");
 154   return res;
 155 }
 156 
 157 // Try to find a better memory input for a load from a strict final field
 158 static Node* try_optimize_strict_final_load_memory(PhaseGVN* phase, Node* adr, ProjNode*& base_local) {
 159   intptr_t offset = 0;
 160   Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
 161   if (base == nullptr) {
 162     return nullptr;
 163   }
 164 
 165   Node* base_uncasted = base->uncast();
 166   if (base_uncasted->is_Proj()) {
 167     MultiNode* multi = base_uncasted->in(0)->as_Multi();
 168     if (multi->is_Allocate()) {
 169       base_local = base_uncasted->as_Proj();
 170       return nullptr;
 171     } else if (multi->is_Call()) {
 172       // The oop is returned from a call, the memory can be the fallthrough output of the call
 173       return find_call_fallthrough_mem_output(multi->as_Call());
 174     } else if (multi->is_Start()) {
 175       // The oop is a parameter
 176       if (phase->C->method()->is_object_constructor() && base_uncasted->as_Proj()->_con == TypeFunc::Parms) {
 177         // The receiver of a constructor is similar to the result of an AllocateNode
 178         base_local = base_uncasted->as_Proj();
 179         return nullptr;
 180       } else {
 181         // Use the start memory otherwise
 182         return multi->proj_out(TypeFunc::Memory);
 183       }
 184     }
 185   }
 186 
 187   return nullptr;
 188 }
 189 
 190 // Whether a call can modify a strict final field, given that the object is allocated inside the
 191 // current compilation unit, or is the first parameter when the compilation root is a constructor.
 192 // This is equivalent to asking whether 'call' is a constructor invocation and the class declaring
 193 // the target method is a subclass of the class declaring 'field'.
 194 static bool call_can_modify_local_object(ciField* field, CallNode* call) {
 195   if (!call->is_CallJava()) {
 196     return false;
 197   }
 198 
 199   ciMethod* target = call->as_CallJava()->method();
 200   if (target == nullptr || !target->is_object_constructor()) {
 201     return false;
 202   }
 203 
 204   // If 'field' is declared in a class that is a subclass of the one declaring the constructor,
 205   // then the field is set inside the constructor, else the field must be set before the
 206   // constructor invocation. E.g. A field Super.x will be set during the execution of Sub::<init>,
 207   // while a field Sub.y must be set before Super::<init> is invoked.
 208   // We can try to be more heroic and decide if the receiver of the constructor invocation is the
 209   // object from which we are loading from. This, however, may be problematic as deciding if 2
 210   // nodes are definitely different may not be trivial, especially if the graph is not canonical.
 211   // As a result, it is made more conservative for now.
 212   assert(call->req() > TypeFunc::Parms, "constructor must have at least 1 argument");
 213   return target->holder()->is_subclass_of(field->holder());
 214 }
 215 
 216 Node* MemNode::optimize_simple_memory_chain(Node* mchain, const TypeOopPtr* t_oop, Node* load, PhaseGVN* phase) {
 217   assert(t_oop != nullptr, "sanity");
 218   bool is_known_instance = t_oop->is_known_instance_field();
 219   bool is_strict_final_load = false;
 220 
 221   // After macro expansion, an allocation may become a call, changing the memory input to the
 222   // memory output of that call would be illegal. As a result, disallow this transformation after
 223   // macro expansion.
 224   if (phase->is_IterGVN() && phase->C->allow_macro_nodes() && load != nullptr && load->is_Load() && !load->as_Load()->is_mismatched_access()) {
 225     is_strict_final_load = t_oop->is_ptr_to_strict_final_field();
 226 #ifdef ASSERT
 227     if ((t_oop->is_inlinetypeptr() && t_oop->inline_klass()->contains_field_offset(t_oop->offset())) || t_oop->is_ptr_to_boxed_value()) {
 228       assert(is_strict_final_load, "sanity check for basic cases");
 229     }
 230 #endif // ASSERT
 231   }
 232 
 233   if (!is_known_instance && !is_strict_final_load) {
 234     return mchain;
 235   }
 236 
 237   Node* result = mchain;
 238   ProjNode* base_local = nullptr;
 239 
 240   ciField* field = nullptr;
 241   if (is_strict_final_load) {
 242     field = phase->C->alias_type(t_oop)->field();
 243     assert(field != nullptr, "must point to a field");
 244 
 245     Node* adr = load->in(MemNode::Address);
 246     assert(phase->type(adr) == t_oop, "inconsistent type");
 247     Node* tmp = try_optimize_strict_final_load_memory(phase, adr, base_local);
 248     if (tmp != nullptr) {
 249       result = tmp;
 250     }
 251   }
 252 
 253   uint instance_id = t_oop->instance_id();
 254   Node* start_mem = phase->C->start()->proj_out_or_null(TypeFunc::Memory);
 255   Node* prev = nullptr;

 256   while (prev != result) {
 257     prev = result;
 258     if (result == start_mem) {
 259       // start_mem is the earliest memory possible
 260       break;
 261     }
 262 
 263     // skip over a call which does not affect this memory slice
 264     if (result->is_Proj() && result->as_Proj()->_con == TypeFunc::Memory) {
 265       Node* proj_in = result->in(0);
 266       if (proj_in->is_Allocate() && proj_in->_idx == instance_id) {
 267         // This is the allocation that creates the object from which we are loading from
 268         break;
 269       } else if (proj_in->is_Call()) {
 270         // ArrayCopyNodes processed here as well
 271         CallNode* call = proj_in->as_Call();
 272         if (!call->may_modify(t_oop, phase)) {
 273           result = call->in(TypeFunc::Memory);
 274         } else if (is_strict_final_load && base_local != nullptr && !call_can_modify_local_object(field, call)) {
 275           result = call->in(TypeFunc::Memory);
 276         }
 277       } else if (proj_in->is_Initialize()) {
 278         AllocateNode* alloc = proj_in->as_Initialize()->allocation();
 279         // Stop if this is the initialization for the object instance which
 280         // contains this memory slice, otherwise skip over it.
 281         if ((alloc == nullptr) || (alloc->_idx == instance_id)) {
 282           break;
 283         }
 284         if (is_known_instance) {
 285           result = proj_in->in(TypeFunc::Memory);
 286         } else if (is_strict_final_load) {
 287           Node* klass = alloc->in(AllocateNode::KlassNode);
 288           const TypeKlassPtr* tklass = phase->type(klass)->is_klassptr();
 289           if (tklass->klass_is_exact() && !tklass->exact_klass()->equals(t_oop->is_instptr()->exact_klass())) {
 290             // Allocation of another type, must be another object
 291             result = proj_in->in(TypeFunc::Memory);
 292           } else if (base_local != nullptr && (base_local->is_Parm() || base_local->in(0) != alloc)) {
 293             // Allocation of another object
 294             result = proj_in->in(TypeFunc::Memory);
 295           }
 296         }
 297       } else if (proj_in->is_MemBar()) {
 298         ArrayCopyNode* ac = nullptr;
 299         if (ArrayCopyNode::may_modify(t_oop, proj_in->as_MemBar(), phase, ac)) {
 300           break;
 301         }
 302         result = proj_in->in(TypeFunc::Memory);
 303       } else if (proj_in->is_LoadFlat() || proj_in->is_StoreFlat()) {
 304         if (is_strict_final_load) {
 305           // LoadFlat and StoreFlat cannot happen to strict final fields
 306           result = proj_in->in(TypeFunc::Memory);
 307         }
 308       } else if (proj_in->is_top()) {
 309         break; // dead code
 310       } else {
 311         assert(false, "unexpected projection of %s", proj_in->Name());
 312       }
 313     } else if (result->is_ClearArray()) {
 314       if (!is_known_instance || !ClearArrayNode::step_through(&result, instance_id, phase)) {
 315         // Can not bypass initialization of the instance
 316         // we are looking for.
 317         break;
 318       }
 319       // Otherwise skip it (the call updated 'result' value).
 320     } else if (result->is_MergeMem()) {
 321       result = step_through_mergemem(phase, result->as_MergeMem(), t_oop, nullptr, tty);
 322     }
 323   }
 324   return result;
 325 }
 326 
 327 Node *MemNode::optimize_memory_chain(Node *mchain, const TypePtr *t_adr, Node *load, PhaseGVN *phase) {
 328   const TypeOopPtr* t_oop = t_adr->isa_oopptr();
 329   if (t_oop == nullptr)
 330     return mchain;  // don't try to optimize non-oop types
 331   Node* result = optimize_simple_memory_chain(mchain, t_oop, load, phase);
 332   bool is_instance = t_oop->is_known_instance_field();
 333   PhaseIterGVN *igvn = phase->is_IterGVN();
 334   if (is_instance && igvn != nullptr && result->is_Phi()) {
 335     PhiNode *mphi = result->as_Phi();
 336     assert(mphi->bottom_type() == Type::MEMORY, "memory phi required");
 337     const TypePtr *t = mphi->adr_type();
 338     bool do_split = false;
 339     // In the following cases, Load memory input can be further optimized based on
 340     // its precise address type
 341     if (t == TypePtr::BOTTOM || t == TypeRawPtr::BOTTOM ) {
 342       do_split = true;
 343     } else if (t->isa_oopptr() && !t->is_oopptr()->is_known_instance()) {
 344       const TypeOopPtr* mem_t =
 345         t->is_oopptr()->cast_to_exactness(true)
 346         ->is_oopptr()->cast_to_ptr_type(t_oop->ptr())
 347         ->is_oopptr()->cast_to_instance_id(t_oop->instance_id());
 348       if (t_oop->isa_aryptr()) {
 349         mem_t = mem_t->is_aryptr()
 350                      ->cast_to_stable(t_oop->is_aryptr()->is_stable())
 351                      ->cast_to_size(t_oop->is_aryptr()->size())
 352                      ->cast_to_not_flat(t_oop->is_aryptr()->is_not_flat())
 353                      ->cast_to_not_null_free(t_oop->is_aryptr()->is_not_null_free())
 354                      ->with_offset(t_oop->is_aryptr()->offset())
 355                      ->is_aryptr();
 356       }
 357       do_split = mem_t == t_oop;
 358     }
 359     if (do_split) {
 360       // clone the Phi with our address type
 361       result = mphi->split_out_instance(t_adr, igvn);
 362     } else {
 363       assert(phase->C->get_alias_index(t) == phase->C->get_alias_index(t_adr), "correct memory chain");
 364     }
 365   }
 366   return result;
 367 }
 368 
 369 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st) {
 370   uint alias_idx = phase->C->get_alias_index(tp);
 371   Node *mem = mmem;
 372 #ifdef ASSERT
 373   {
 374     // Check that current type is consistent with the alias index used during graph construction
 375     assert(alias_idx >= Compile::AliasIdxRaw, "must not be a bad alias_idx");
 376     bool consistent =  adr_check == nullptr || adr_check->empty() ||
 377                        phase->C->must_alias(adr_check, alias_idx );
 378     // Sometimes dead array references collapse to a[-1], a[-2], or a[-3]
 379     if( !consistent && adr_check != nullptr && !adr_check->empty() &&
 380         tp->isa_aryptr() &&        tp->offset() == Type::OffsetBot &&
 381         adr_check->isa_aryptr() && adr_check->offset() != Type::OffsetBot &&
 382         ( adr_check->offset() == arrayOopDesc::length_offset_in_bytes() ||
 383           adr_check->offset() == oopDesc::klass_offset_in_bytes() ||
 384           adr_check->offset() == oopDesc::mark_offset_in_bytes() ) ) {
 385       // don't assert if it is dead code.
 386       consistent = true;
 387     }
 388     if( !consistent ) {
 389       st->print("alias_idx==%d, adr_check==", alias_idx);
 390       if( adr_check == nullptr ) {
 391         st->print("null");
 392       } else {
 393         adr_check->dump();
 394       }
 395       st->cr();
 396       print_alias_types();
 397       assert(consistent, "adr_check must match alias idx");
 398     }
 399   }
 400 #endif

1072          "use LoadKlassNode instead");
1073   assert(!(adr_type->isa_aryptr() &&
1074            adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
1075          "use LoadRangeNode instead");
1076   // Check control edge of raw loads
1077   assert( ctl != nullptr || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
1078           // oop will be recorded in oop map if load crosses safepoint
1079           rt->isa_oopptr() || is_immutable_value(adr),
1080           "raw memory operations should have control edge");
1081   LoadNode* load = nullptr;
1082   switch (bt) {
1083   case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
1084   case T_BYTE:    load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
1085   case T_INT:     load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
1086   case T_CHAR:    load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
1087   case T_SHORT:   load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(),  mo, control_dependency); break;
1088   case T_LONG:    load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency, require_atomic_access); break;
1089   case T_FLOAT:   load = new LoadFNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency); break;
1090   case T_DOUBLE:  load = new LoadDNode (ctl, mem, adr, adr_type, rt,            mo, control_dependency, require_atomic_access); break;
1091   case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(),  mo, control_dependency); break;
1092   case T_ARRAY:
1093   case T_OBJECT:
1094   case T_NARROWOOP:
1095 #ifdef _LP64
1096     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
1097       load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
1098     } else
1099 #endif
1100     {
1101       assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
1102       load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
1103     }
1104     break;
1105   default:
1106     assert(false, "unexpected basic type %s", type2name(bt));
1107     break;
1108   }
1109   assert(load != nullptr, "LoadNode should have been created");
1110   if (unaligned) {
1111     load->set_unaligned_access();
1112   }
1113   if (mismatched) {
1114     load->set_mismatched_access();
1115   }
1116   if (unsafe) {
1117     load->set_unsafe_access();
1118   }
1119   load->set_barrier_data(barrier_data);
1120   if (load->Opcode() == Op_LoadN) {
1121     Node* ld = gvn.transform(load);
1122     return new DecodeNNode(ld, ld->bottom_type()->make_ptr());
1123   }
1124 
1125   return load;
1126 }
1127 
1128 //------------------------------hash-------------------------------------------
1129 uint LoadNode::hash() const {
1130   // unroll addition of interesting fields
1131   return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address);
1132 }
1133 
1134 static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp, bool eliminate_boxing) {
1135   if ((atp != nullptr) && (atp->index() >= Compile::AliasIdxRaw)) {
1136     bool non_volatile = (atp->field() != nullptr) && !atp->field()->is_volatile();
1137     bool is_stable_ary = FoldStableValues &&
1138                          (tp != nullptr) && (tp->isa_aryptr() != nullptr) &&
1139                          tp->isa_aryptr()->is_stable();
1140 
1141     return (eliminate_boxing && non_volatile) || is_stable_ary || tp->is_inlinetypeptr();
1142   }
1143 
1144   return false;
1145 }
1146 
1147 LoadNode* LoadNode::pin_array_access_node() const {
1148   const TypePtr* adr_type = this->adr_type();
1149   if (adr_type != nullptr && adr_type->isa_aryptr()) {
1150     return clone_pinned();
1151   }
1152   return nullptr;
1153 }
1154 
1155 // Is the value loaded previously stored by an arraycopy? If so return
1156 // a load node that reads from the source array so we may be able to
1157 // optimize out the ArrayCopy node later.
1158 Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
1159   Node* ld_adr = in(MemNode::Address);
1160   intptr_t ld_off = 0;
1161   AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);

1177     if (ac->as_ArrayCopy()->is_clonebasic()) {
1178       assert(ld_alloc != nullptr, "need an alloc");
1179       assert(addp->is_AddP(), "address must be addp");
1180       BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1181       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Base)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1182       assert(bs->step_over_gc_barrier(addp->in(AddPNode::Address)) == bs->step_over_gc_barrier(ac->in(ArrayCopyNode::Dest)), "strange pattern");
1183       addp->set_req(AddPNode::Base, src);
1184       addp->set_req(AddPNode::Address, src);
1185     } else {
1186       assert(ac->as_ArrayCopy()->is_arraycopy_validated() ||
1187              ac->as_ArrayCopy()->is_copyof_validated() ||
1188              ac->as_ArrayCopy()->is_copyofrange_validated(), "only supported cases");
1189       assert(addp->in(AddPNode::Base) == addp->in(AddPNode::Address), "should be");
1190       addp->set_req(AddPNode::Base, src);
1191       addp->set_req(AddPNode::Address, src);
1192 
1193       const TypeAryPtr* ary_t = phase->type(in(MemNode::Address))->isa_aryptr();
1194       BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type();
1195       if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT;
1196 
1197       uint shift  = ary_t->is_flat() ? ary_t->flat_log_elem_size() : exact_log2(type2aelembytes(ary_elem));

1198 
1199       Node* diff = phase->transform(new SubINode(ac->in(ArrayCopyNode::SrcPos), ac->in(ArrayCopyNode::DestPos)));
1200 #ifdef _LP64
1201       diff = phase->transform(new ConvI2LNode(diff));
1202 #endif
1203       diff = phase->transform(new LShiftXNode(diff, phase->intcon(shift)));
1204 
1205       Node* offset = phase->transform(new AddXNode(addp->in(AddPNode::Offset), diff));
1206       addp->set_req(AddPNode::Offset, offset);
1207     }
1208     addp = phase->transform(addp);
1209 #ifdef ASSERT
1210     const TypePtr* adr_type = phase->type(addp)->is_ptr();
1211     ld->_adr_type = adr_type;
1212 #endif
1213     ld->set_req(MemNode::Address, addp);
1214     ld->set_req(0, ctl);
1215     ld->set_req(MemNode::Memory, mem);
1216     return ld;
1217   }
1218   return nullptr;
1219 }
1220 
1221 static Node* see_through_inline_type(PhaseValues* phase, const MemNode* load, Node* base, int offset) {
1222   if (!load->is_mismatched_access() && base != nullptr && base->is_InlineType() && offset > oopDesc::klass_offset_in_bytes()) {
1223     InlineTypeNode* vt = base->as_InlineType();
1224     Node* value = vt->field_value_by_offset(offset, true);
1225     assert(value != nullptr, "must see some value");
1226     return value;
1227   }
1228 
1229   return nullptr;
1230 }
1231 
1232 //---------------------------can_see_stored_value------------------------------
1233 // This routine exists to make sure this set of tests is done the same
1234 // everywhere.  We need to make a coordinated change: first LoadNode::Ideal
1235 // will change the graph shape in a way which makes memory alive twice at the
1236 // same time (uses the Oracle model of aliasing), then some
1237 // LoadXNode::Identity will fold things back to the equivalence-class model
1238 // of aliasing.
1239 Node* MemNode::can_see_stored_value(Node* st, PhaseValues* phase) const {
1240   Node* ld_adr = in(MemNode::Address);
1241   intptr_t ld_off = 0;
1242   Node* ld_base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ld_off);
1243   // Try to see through an InlineTypeNode
1244   // LoadN is special because the input is not compressed
1245   if (Opcode() != Op_LoadN) {
1246     Node* value = see_through_inline_type(phase, this, ld_base, ld_off);
1247     if (value != nullptr) {
1248       return value;
1249     }
1250   }
1251 
1252   Node* ld_alloc = AllocateNode::Ideal_allocation(ld_base);
1253   const TypeInstPtr* tp = phase->type(ld_adr)->isa_instptr();
1254   Compile::AliasType* atp = (tp != nullptr) ? phase->C->alias_type(tp) : nullptr;
1255   // This is more general than load from boxing objects.
1256   if (skip_through_membars(atp, tp, phase->C->eliminate_boxing())) {
1257     uint alias_idx = atp->index();
1258     Node* result = nullptr;
1259     Node* current = st;
1260     // Skip through chains of MemBarNodes checking the MergeMems for
1261     // new states for the slice of this load.  Stop once any other
1262     // kind of node is encountered.  Loads from final memory can skip
1263     // through any kind of MemBar but normal loads shouldn't skip
1264     // through MemBarAcquire since the could allow them to move out of
1265     // a synchronized region. It is not safe to step over MemBarCPUOrder,
1266     // because alias info above them may be inaccurate (e.g., due to
1267     // mixed/mismatched unsafe accesses).
1268     bool is_final_mem = !atp->is_rewritable();
1269     while (current->is_Proj()) {
1270       int opc = current->in(0)->Opcode();
1271       if ((is_final_mem && (opc == Op_MemBarAcquire ||

1315         // Same base, same offset.
1316         // Possible improvement for arrays: check index value instead of absolute offset.
1317 
1318         // At this point we have proven something like this setup:
1319         //   B = << base >>
1320         //   L =  LoadQ(AddP(Check/CastPP(B), #Off))
1321         //   S = StoreQ(AddP(             B , #Off), V)
1322         // (Actually, we haven't yet proven the Q's are the same.)
1323         // In other words, we are loading from a casted version of
1324         // the same pointer-and-offset that we stored to.
1325         // Casted version may carry a dependency and it is respected.
1326         // Thus, we are able to replace L by V.
1327       }
1328       // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1329       if (store_Opcode() != st->Opcode()) {
1330         return nullptr;
1331       }
1332       // LoadVector/StoreVector needs additional check to ensure the types match.
1333       if (st->is_StoreVector()) {
1334         const TypeVect*  in_vt = st->as_StoreVector()->vect_type();
1335         const TypeVect* out_vt = is_Load() ? as_LoadVector()->vect_type() : as_StoreVector()->vect_type();
1336         if (in_vt != out_vt) {
1337           return nullptr;
1338         }
1339       }
1340       return st->in(MemNode::ValueIn);
1341     }
1342 
1343     // A load from a freshly-created object always returns zero.
1344     // (This can happen after LoadNode::Ideal resets the load's memory input
1345     // to find_captured_store, which returned InitializeNode::zero_memory.)
1346     if (st->is_Proj() && st->in(0)->is_Allocate() &&
1347         (st->in(0) == ld_alloc) &&
1348         (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1349       // return a zero value for the load's basic type
1350       // (This is one of the few places where a generic PhaseTransform
1351       // can create new nodes.  Think of it as lazily manifesting
1352       // virtually pre-existing constants.)
1353       Node* init_value = ld_alloc->in(AllocateNode::InitValue);
1354       if (init_value != nullptr) {
1355         // TODO 8350865 Scalar replacement does not work well for flat arrays.
1356         // Is this correct for non-all-zero init values? Don't we need field_value_by_offset?
1357         return init_value;
1358       }
1359       assert(ld_alloc->in(AllocateNode::RawInitValue) == nullptr, "init value may not be null");
1360       if (value_basic_type() != T_VOID) {
1361         if (ReduceBulkZeroing || find_array_copy_clone(ld_alloc, in(MemNode::Memory)) == nullptr) {
1362           // If ReduceBulkZeroing is disabled, we need to check if the allocation does not belong to an
1363           // ArrayCopyNode clone. If it does, then we cannot assume zero since the initialization is done
1364           // by the ArrayCopyNode.
1365           return phase->zerocon(value_basic_type());
1366         }
1367       } else {
1368         // TODO: materialize all-zero vector constant
1369         assert(!isa_Load() || as_Load()->type()->isa_vect(), "");
1370       }
1371     }
1372 
1373     // A load from an initialization barrier can match a captured store.
1374     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1375       InitializeNode* init = st->in(0)->as_Initialize();
1376       AllocateNode* alloc = init->allocation();
1377       if ((alloc != nullptr) && (alloc == ld_alloc)) {
1378         // examine a captured store value
1379         st = init->find_captured_store(ld_off, memory_size(), phase);

1392       base = bs->step_over_gc_barrier(base);
1393       if (base != nullptr && base->is_Proj() &&
1394           base->as_Proj()->_con == TypeFunc::Parms &&
1395           base->in(0)->is_CallStaticJava() &&
1396           base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1397         return base->in(0)->in(TypeFunc::Parms);
1398       }
1399     }
1400 
1401     break;
1402   }
1403 
1404   return nullptr;
1405 }
1406 
1407 //----------------------is_instance_field_load_with_local_phi------------------
1408 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1409   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1410       in(Address)->is_AddP() ) {
1411     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1412     // Only known instances and immutable fields
1413     if( t_oop != nullptr &&
1414         (t_oop->is_ptr_to_strict_final_field() ||
1415          t_oop->is_known_instance_field()) &&
1416         t_oop->offset() != Type::OffsetBot &&
1417         t_oop->offset() != Type::OffsetTop) {
1418       return true;
1419     }
1420   }
1421   return false;
1422 }
1423 
1424 //------------------------------Identity---------------------------------------
1425 // Loads are identity if previous store is to same address
1426 Node* LoadNode::Identity(PhaseGVN* phase) {
1427   // If the previous store-maker is the right kind of Store, and the store is
1428   // to the same address, then we are equal to the value stored.
1429   Node* mem = in(Memory);
1430   Node* value = can_see_stored_value(mem, phase);
1431   if( value ) {
1432     // byte, short & char stores truncate naturally.
1433     // A load has to load the truncated value which requires
1434     // some sort of masking operation and that requires an

1443     // usually runs first, producing the singleton type of the Con.)
1444     if (!has_pinned_control_dependency() || value->is_Con()) {
1445       return value;
1446     } else {
1447       return this;
1448     }
1449   }
1450 
1451   if (has_pinned_control_dependency()) {
1452     return this;
1453   }
1454   // Search for an existing data phi which was generated before for the same
1455   // instance's field to avoid infinite generation of phis in a loop.
1456   Node *region = mem->in(0);
1457   if (is_instance_field_load_with_local_phi(region)) {
1458     const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1459     int this_index  = phase->C->get_alias_index(addr_t);
1460     int this_offset = addr_t->offset();
1461     int this_iid    = addr_t->instance_id();
1462     if (!addr_t->is_known_instance() &&
1463          addr_t->is_ptr_to_strict_final_field()) {
1464       // Use _idx of address base (could be Phi node) for immutable fields in unknown instances
1465       intptr_t   ignore = 0;
1466       Node*      base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1467       if (base == nullptr) {
1468         return this;
1469       }
1470       this_iid = base->_idx;
1471     }
1472     const Type* this_type = bottom_type();
1473     for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
1474       Node* phi = region->fast_out(i);
1475       if (phi->is_Phi() && phi != mem &&
1476           phi->as_Phi()->is_same_inst_field(this_type, (int)mem->_idx, this_iid, this_index, this_offset)) {
1477         return phi;
1478       }
1479     }
1480   }
1481 
1482   return this;
1483 }
1484 

2000   bool addr_mark = ((phase->type(address)->isa_oopptr() || phase->type(address)->isa_narrowoop()) &&
2001          phase->type(address)->is_ptr()->offset() == oopDesc::mark_offset_in_bytes());
2002 
2003   // Skip up past a SafePoint control.  Cannot do this for Stores because
2004   // pointer stores & cardmarks must stay on the same side of a SafePoint.
2005   if( ctrl != nullptr && ctrl->Opcode() == Op_SafePoint &&
2006       phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw  &&
2007       !addr_mark &&
2008       (depends_only_on_test() || has_unknown_control_dependency())) {
2009     ctrl = ctrl->in(0);
2010     set_req(MemNode::Control,ctrl);
2011     progress = true;
2012   }
2013 
2014   intptr_t ignore = 0;
2015   Node*    base   = AddPNode::Ideal_base_and_offset(address, phase, ignore);
2016   if (base != nullptr
2017       && phase->C->get_alias_index(phase->type(address)->is_ptr()) != Compile::AliasIdxRaw) {
2018     // Check for useless control edge in some common special cases
2019     if (in(MemNode::Control) != nullptr
2020         && !(phase->type(address)->is_inlinetypeptr() && is_mismatched_access())
2021         && can_remove_control()
2022         && phase->type(base)->higher_equal(TypePtr::NOTNULL)
2023         && all_controls_dominate(base, phase->C->start())) {
2024       // A method-invariant, non-null address (constant or 'this' argument).
2025       set_req(MemNode::Control, nullptr);
2026       progress = true;
2027     }
2028   }
2029 
2030   Node* mem = in(MemNode::Memory);
2031   const TypePtr *addr_t = phase->type(address)->isa_ptr();
2032 
2033   if (can_reshape && (addr_t != nullptr)) {
2034     // try to optimize our memory input
2035     Node* opt_mem = MemNode::optimize_memory_chain(mem, addr_t, this, phase);
2036     if (opt_mem != mem) {
2037       set_req_X(MemNode::Memory, opt_mem, phase);
2038       if (phase->type( opt_mem ) == Type::TOP) return nullptr;
2039       return this;
2040     }

2097   // fold up, do so.
2098   Node* prev_mem = find_previous_store(phase);
2099   if (prev_mem != nullptr) {
2100     Node* value = can_see_arraycopy_value(prev_mem, phase);
2101     if (value != nullptr) {
2102       return value;
2103     }
2104   }
2105   // Steps (a), (b):  Walk past independent stores to find an exact match.
2106   if (prev_mem != nullptr && prev_mem != in(MemNode::Memory)) {
2107     // (c) See if we can fold up on the spot, but don't fold up here.
2108     // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
2109     // just return a prior value, which is done by Identity calls.
2110     if (can_see_stored_value(prev_mem, phase)) {
2111       // Make ready for step (d):
2112       set_req_X(MemNode::Memory, prev_mem, phase);
2113       return this;
2114     }
2115   }
2116 
2117   if (progress) {
2118     return this;
2119   }
2120 
2121   if (!can_reshape) {
2122     phase->record_for_igvn(this);
2123   }
2124   return nullptr;
2125 }
2126 
2127 // Helper to recognize certain Klass fields which are invariant across
2128 // some group of array types (e.g., int[] or all T[] where T < Object).
2129 const Type*
2130 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
2131                                  ciKlass* klass) const {
2132   assert(!UseCompactObjectHeaders || tkls->offset() != in_bytes(Klass::prototype_header_offset()),
2133          "must not happen");
2134   if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
2135     // The field is Klass::_access_flags.  Return its (constant) value.
2136     assert(Opcode() == Op_LoadUS, "must load an unsigned short from _access_flags");
2137     return TypeInt::make(klass->access_flags());
2138   }
2139   if (tkls->offset() == in_bytes(Klass::misc_flags_offset())) {
2140     // The field is Klass::_misc_flags.  Return its (constant) value.
2141     assert(Opcode() == Op_LoadUB, "must load an unsigned byte from _misc_flags");
2142     return TypeInt::make(klass->misc_flags());
2143   }
2144   if (tkls->offset() == in_bytes(Klass::layout_helper_offset())) {

2204       }
2205     }
2206 
2207     // Don't do this for integer types. There is only potential profit if
2208     // the element type t is lower than _type; that is, for int types, if _type is
2209     // more restrictive than t.  This only happens here if one is short and the other
2210     // char (both 16 bits), and in those cases we've made an intentional decision
2211     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
2212     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
2213     //
2214     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
2215     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
2216     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
2217     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
2218     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
2219     // In fact, that could have been the original type of p1, and p1 could have
2220     // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
2221     // expression (LShiftL quux 3) independently optimized to the constant 8.
2222     if ((t->isa_int() == nullptr) && (t->isa_long() == nullptr)
2223         && (_type->isa_vect() == nullptr)
2224         && !ary->is_flat()
2225         && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
2226       // t might actually be lower than _type, if _type is a unique
2227       // concrete subclass of abstract class t.
2228       if (off_beyond_header || off == Type::OffsetBot) {  // is the offset beyond the header?
2229         const Type* jt = t->join_speculative(_type);
2230         // In any case, do not allow the join, per se, to empty out the type.
2231         if (jt->empty() && !t->empty()) {
2232           // This can happen if a interface-typed array narrows to a class type.
2233           jt = _type;
2234         }
2235 #ifdef ASSERT
2236         if (phase->C->eliminate_boxing() && adr->is_AddP()) {
2237           // The pointers in the autobox arrays are always non-null
2238           Node* base = adr->in(AddPNode::Base);
2239           if ((base != nullptr) && base->is_DecodeN()) {
2240             // Get LoadN node which loads IntegerCache.cache field
2241             base = base->in(1);
2242           }
2243           if ((base != nullptr) && base->is_Con()) {
2244             const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
2245             if ((base_type != nullptr) && base_type->is_autobox_cache()) {
2246               // It could be narrow oop
2247               assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
2248             }
2249           }
2250         }
2251 #endif
2252         return jt;
2253       }
2254     }
2255   } else if (tp->base() == Type::InstPtr) {
2256     assert( off != Type::OffsetBot ||
2257             // arrays can be cast to Objects
2258             !tp->isa_instptr() ||
2259             tp->is_instptr()->instance_klass()->is_java_lang_Object() ||
2260             // Default value load
2261             tp->is_instptr()->instance_klass() == ciEnv::current()->Class_klass() ||
2262             // unsafe field access may not have a constant offset
2263             C->has_unsafe_access(),
2264             "Field accesses must be precise" );
2265     // For oop loads, we expect the _type to be precise.
2266 

2267     const TypeInstPtr* tinst = tp->is_instptr();
2268     BasicType bt = value_basic_type();
2269 
2270     // Optimize loads from constant fields.
2271     ciObject* const_oop = tinst->const_oop();
2272     if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != nullptr && const_oop->is_instance()) {
2273       const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
2274       if (con_type != nullptr) {
2275         return con_type;
2276       }
2277     }
2278   } else if (tp->base() == Type::KlassPtr || tp->base() == Type::InstKlassPtr || tp->base() == Type::AryKlassPtr) {
2279     assert(off != Type::OffsetBot ||
2280             !tp->isa_instklassptr() ||
2281            // arrays can be cast to Objects
2282            tp->isa_instklassptr()->instance_klass()->is_java_lang_Object() ||
2283            // also allow array-loading from the primary supertype
2284            // array during subtype checks
2285            Opcode() == Op_LoadKlass,
2286            "Field accesses must be precise");
2287     // For klass/static loads, we expect the _type to be precise
2288   } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
2289     /* With mirrors being an indirect in the Klass*
2290      * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
2291      * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
2292      *
2293      * So check the type and klass of the node before the LoadP.

2300         assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2301         assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
2302         return TypeInstPtr::make(klass->java_mirror());
2303       }
2304     }
2305   }
2306 
2307   const TypeKlassPtr *tkls = tp->isa_klassptr();
2308   if (tkls != nullptr) {
2309     if (tkls->is_loaded() && tkls->klass_is_exact()) {
2310       ciKlass* klass = tkls->exact_klass();
2311       // We are loading a field from a Klass metaobject whose identity
2312       // is known at compile time (the type is "exact" or "precise").
2313       // Check for fields we know are maintained as constants by the VM.
2314       if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
2315         // The field is Klass::_super_check_offset.  Return its (constant) value.
2316         // (Folds up type checking code.)
2317         assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
2318         return TypeInt::make(klass->super_check_offset());
2319       }
2320       if (tkls->offset() == in_bytes(ObjArrayKlass::next_refined_array_klass_offset()) && klass->is_obj_array_klass()) {
2321         // Fold loads from LibraryCallKit::load_default_refined_array_klass
2322         return tkls->is_aryklassptr()->refined_array_klass_ptr();
2323       }
2324       if (klass->is_flat_array_klass() && tkls->offset() == in_bytes(FlatArrayKlass::layout_kind_offset())) {
2325         assert(Opcode() == Op_LoadI, "must load an int from _layout_kind");
2326         return TypeInt::make(static_cast<jint>(klass->as_flat_array_klass()->layout_kind()));
2327       }
2328       if (UseCompactObjectHeaders && tkls->offset() == in_bytes(Klass::prototype_header_offset())) {
2329         // The field is Klass::_prototype_header. Return its (constant) value.
2330         assert(this->Opcode() == Op_LoadX, "must load a proper type from _prototype_header");
2331         return TypeX::make(klass->prototype_header());
2332       }
2333       // Compute index into primary_supers array
2334       juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
2335       // Check for overflowing; use unsigned compare to handle the negative case.
2336       if( depth < ciKlass::primary_super_limit() ) {
2337         // The field is an element of Klass::_primary_supers.  Return its (constant) value.
2338         // (Folds up type checking code.)
2339         assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
2340         ciKlass *ss = klass->super_of_depth(depth);
2341         return ss ? TypeKlassPtr::make(ss, Type::trust_interfaces) : TypePtr::NULL_PTR;
2342       }
2343       const Type* aift = load_array_final_field(tkls, klass);
2344       if (aift != nullptr)  return aift;
2345     }
2346 
2347     // We can still check if we are loading from the primary_supers array at a
2348     // shallow enough depth.  Even though the klass is not exact, entries less
2349     // than or equal to its super depth are correct.
2350     if (tkls->is_loaded()) {
2351       ciKlass* klass = nullptr;

2382         !tkls->is_instklassptr()->might_be_an_array() // not the supertype of all T[] (java.lang.Object) or has an interface that is not Serializable or Cloneable
2383     ) {
2384       assert(Opcode() == Op_LoadI, "must load an int from _layout_helper");
2385       jint min_size = Klass::instance_layout_helper(oopDesc::header_size(), false);
2386       // The key property of this type is that it folds up tests
2387       // for array-ness, since it proves that the layout_helper is positive.
2388       // Thus, a generic value like the basic object layout helper works fine.
2389       return TypeInt::make(min_size, max_jint, Type::WidenMin);
2390     }
2391   }
2392 
2393   bool is_vect = (_type->isa_vect() != nullptr);
2394   if (is_instance && !is_vect) {
2395     // If we have an instance type and our memory input is the
2396     // programs's initial memory state, there is no matching store,
2397     // so just return a zero of the appropriate type -
2398     // except if it is vectorized - then we have no zero constant.
2399     Node *mem = in(MemNode::Memory);
2400     if (mem->is_Parm() && mem->in(0)->is_Start()) {
2401       assert(mem->as_Parm()->_con == TypeFunc::Memory, "must be memory Parm");
2402       // TODO 8350865 Scalar replacement does not work well for flat arrays.
2403       // Escape Analysis assumes that arrays are always zeroed during allocation which is not true for null-free arrays
2404       // ConnectionGraph::split_unique_types will re-wire the memory of loads from such arrays around the allocation
2405       // TestArrays::test6 and test152 and TestBasicFunctionality::test20 are affected by this.
2406       if (tp->isa_aryptr() && tp->is_aryptr()->is_flat() && tp->is_aryptr()->is_null_free()) {
2407         intptr_t offset = 0;
2408         Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
2409         AllocateNode* alloc = AllocateNode::Ideal_allocation(base);
2410         if (alloc != nullptr && alloc->is_AllocateArray() && alloc->in(AllocateNode::InitValue) != nullptr) {
2411           return _type;
2412         }
2413       }
2414       return Type::get_zero_type(_type->basic_type());
2415     }
2416   }

2417   if (!UseCompactObjectHeaders) {
2418     Node* alloc = is_new_object_mark_load();
2419     if (alloc != nullptr) {
2420       if (EnableValhalla) {
2421         // The mark word may contain property bits (inline, flat, null-free)
2422         Node* klass_node = alloc->in(AllocateNode::KlassNode);
2423         const TypeKlassPtr* tkls = phase->type(klass_node)->isa_klassptr();
2424         if (tkls != nullptr && tkls->is_loaded() && tkls->klass_is_exact()) {
2425           return TypeX::make(tkls->exact_klass()->prototype_header());
2426         }
2427       } else {
2428         return TypeX::make(markWord::prototype().value());
2429       }
2430     }
2431   }
2432 
2433   return _type;
2434 }
2435 
2436 //------------------------------match_edge-------------------------------------
2437 // Do we Match on this edge index or not?  Match only the address.
2438 uint LoadNode::match_edge(uint idx) const {
2439   return idx == MemNode::Address;
2440 }
2441 
2442 //--------------------------LoadBNode::Ideal--------------------------------------
2443 //
2444 //  If the previous store is to the same address as this load,
2445 //  and the value stored was larger than a byte, replace this load
2446 //  with the value stored truncated to a byte.  If no truncation is
2447 //  needed, the replacement is done in LoadNode::Identity().
2448 //
2449 Node* LoadBNode::Ideal(PhaseGVN* phase, bool can_reshape) {

2558     }
2559   }
2560   // Identity call will handle the case where truncation is not needed.
2561   return LoadNode::Ideal(phase, can_reshape);
2562 }
2563 
2564 const Type* LoadSNode::Value(PhaseGVN* phase) const {
2565   Node* mem = in(MemNode::Memory);
2566   Node* value = can_see_stored_value(mem,phase);
2567   if (value != nullptr && value->is_Con() &&
2568       !value->bottom_type()->higher_equal(_type)) {
2569     // If the input to the store does not fit with the load's result type,
2570     // it must be truncated. We can't delay until Ideal call since
2571     // a singleton Value is needed for split_thru_phi optimization.
2572     int con = value->get_int();
2573     return TypeInt::make((con << 16) >> 16);
2574   }
2575   return LoadNode::Value(phase);
2576 }
2577 
2578 Node* LoadNNode::Ideal(PhaseGVN* phase, bool can_reshape) {
2579   // Loading from an InlineType, find the input and make an EncodeP
2580   Node* addr = in(Address);
2581   intptr_t offset;
2582   Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
2583   Node* value = see_through_inline_type(phase, this, base, offset);
2584   if (value != nullptr) {
2585     return new EncodePNode(value, type());
2586   }
2587 
2588   return LoadNode::Ideal(phase, can_reshape);
2589 }
2590 
2591 //=============================================================================
2592 //----------------------------LoadKlassNode::make------------------------------
2593 // Polymorphic factory method:
2594 Node* LoadKlassNode::make(PhaseGVN& gvn, Node* mem, Node* adr, const TypePtr* at, const TypeKlassPtr* tk) {
2595   // sanity check the alias category against the created node type
2596   const TypePtr* adr_type = adr->bottom_type()->isa_ptr();
2597   assert(adr_type != nullptr, "expecting TypeKlassPtr");
2598 #ifdef _LP64
2599   if (adr_type->is_ptr_to_narrowklass()) {
2600     assert(UseCompressedClassPointers, "no compressed klasses");
2601     Node* load_klass = gvn.transform(new LoadNKlassNode(mem, adr, at, tk->make_narrowklass(), MemNode::unordered));
2602     return new DecodeNKlassNode(load_klass, load_klass->bottom_type()->make_ptr());
2603   }
2604 #endif
2605   assert(!adr_type->is_ptr_to_narrowklass() && !adr_type->is_ptr_to_narrowoop(), "should have got back a narrow oop");
2606   return new LoadKlassNode(mem, adr, at, tk, MemNode::unordered);
2607 }
2608 
2609 //------------------------------Value------------------------------------------
2610 const Type* LoadKlassNode::Value(PhaseGVN* phase) const {

2644           }
2645           return TypeKlassPtr::make(ciArrayKlass::make(t), Type::trust_interfaces);
2646         }
2647         if (!t->is_klass()) {
2648           // a primitive Class (e.g., int.class) has null for a klass field
2649           return TypePtr::NULL_PTR;
2650         }
2651         // Fold up the load of the hidden field
2652         return TypeKlassPtr::make(t->as_klass(), Type::trust_interfaces);
2653       }
2654       // non-constant mirror, so we can't tell what's going on
2655     }
2656     if (!tinst->is_loaded())
2657       return _type;             // Bail out if not loaded
2658     if (offset == oopDesc::klass_offset_in_bytes()) {
2659       return tinst->as_klass_type(true);
2660     }
2661   }
2662 
2663   // Check for loading klass from an array
2664   const TypeAryPtr* tary = tp->isa_aryptr();
2665   if (tary != nullptr &&
2666       tary->offset() == oopDesc::klass_offset_in_bytes()) {
2667     const TypeAryKlassPtr* res = tary->as_klass_type(true)->is_aryklassptr();
2668     // The klass of an array object must be a refined array klass
2669     return res->refined_array_klass_ptr();
2670   }
2671 
2672   // Check for loading klass from an array klass
2673   const TypeKlassPtr *tkls = tp->isa_klassptr();
2674   if (tkls != nullptr && !StressReflectiveCode) {
2675     if (!tkls->is_loaded())
2676      return _type;             // Bail out if not loaded
2677     if (tkls->isa_aryklassptr() && tkls->is_aryklassptr()->elem()->isa_klassptr() &&
2678         tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2679       // // Always returning precise element type is incorrect,
2680       // // e.g., element type could be object and array may contain strings
2681       // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2682 
2683       // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2684       // according to the element type's subclassing.
2685       return tkls->is_aryklassptr()->elem()->isa_klassptr()->cast_to_exactness(tkls->klass_is_exact());
2686     }
2687     if (tkls->isa_instklassptr() != nullptr && tkls->klass_is_exact() &&
2688         tkls->offset() == in_bytes(Klass::super_offset())) {
2689       ciKlass* sup = tkls->is_instklassptr()->instance_klass()->super();

2729     base = bs->step_over_gc_barrier(base);
2730   }
2731 
2732   // We can fetch the klass directly through an AllocateNode.
2733   // This works even if the klass is not constant (clone or newArray).
2734   if (offset == oopDesc::klass_offset_in_bytes()) {
2735     Node* allocated_klass = AllocateNode::Ideal_klass(base, phase);
2736     if (allocated_klass != nullptr) {
2737       return allocated_klass;
2738     }
2739   }
2740 
2741   // Simplify k.java_mirror.as_klass to plain k, where k is a Klass*.
2742   // See inline_native_Class_query for occurrences of these patterns.
2743   // Java Example:  x.getClass().isAssignableFrom(y)
2744   //
2745   // This improves reflective code, often making the Class
2746   // mirror go completely dead.  (Current exception:  Class
2747   // mirrors may appear in debug info, but we could clean them out by
2748   // introducing a new debug info operator for Klass.java_mirror).
2749   //
2750   // This optimization does not apply to arrays because if k is not a
2751   // constant, it was obtained via load_klass which returns the VM type
2752   // and '.java_mirror.as_klass' should return the Java type instead.
2753 
2754   if (toop->isa_instptr() && toop->is_instptr()->instance_klass() == phase->C->env()->Class_klass()
2755       && offset == java_lang_Class::klass_offset()) {
2756     if (base->is_Load()) {
2757       Node* base2 = base->in(MemNode::Address);
2758       if (base2->is_Load()) { /* direct load of a load which is the OopHandle */
2759         Node* adr2 = base2->in(MemNode::Address);
2760         const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
2761         if (tkls != nullptr && !tkls->empty()
2762             && ((tkls->isa_instklassptr() && !tkls->is_instklassptr()->might_be_an_array()))
2763             && adr2->is_AddP()) {

2764           int mirror_field = in_bytes(Klass::java_mirror_offset());
2765           if (tkls->offset() == mirror_field) {
2766             return adr2->in(AddPNode::Base);
2767           }
2768         }
2769       }
2770     }
2771   }
2772 
2773   return this;
2774 }
2775 
2776 LoadNode* LoadNode::clone_pinned() const {
2777   LoadNode* ld = clone()->as_Load();
2778   ld->_control_dependency = UnknownControl;
2779   return ld;
2780 }
2781 
2782 
2783 //------------------------------Value------------------------------------------

2888 //---------------------------StoreNode::make-----------------------------------
2889 // Polymorphic factory method:
2890 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo, bool require_atomic_access) {
2891   assert((mo == unordered || mo == release), "unexpected");
2892   Compile* C = gvn.C;
2893   assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2894          ctl != nullptr, "raw memory operations should have control edge");
2895 
2896   switch (bt) {
2897   case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2898   case T_BYTE:    return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2899   case T_INT:     return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2900   case T_CHAR:
2901   case T_SHORT:   return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2902   case T_LONG:    return new StoreLNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2903   case T_FLOAT:   return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2904   case T_DOUBLE:  return new StoreDNode(ctl, mem, adr, adr_type, val, mo, require_atomic_access);
2905   case T_METADATA:
2906   case T_ADDRESS:
2907   case T_OBJECT:
2908   case T_ARRAY:
2909 #ifdef _LP64
2910     if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2911       val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2912       return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2913     } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2914                (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2915                 adr->bottom_type()->isa_rawptr())) {
2916       val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2917       return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2918     }
2919 #endif
2920     {
2921       return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2922     }
2923   default:
2924     assert(false, "unexpected basic type %s", type2name(bt));
2925     return (StoreNode*)nullptr;
2926   }
2927 }
2928 
2929 //--------------------------bottom_type----------------------------------------
2930 const Type *StoreNode::bottom_type() const {
2931   return Type::MEMORY;
2932 }
2933 
2934 //------------------------------hash-------------------------------------------
2935 uint StoreNode::hash() const {
2936   // unroll addition of interesting fields
2937   //return (uintptr_t)in(Control) + (uintptr_t)in(Memory) + (uintptr_t)in(Address) + (uintptr_t)in(ValueIn);
2938 
2939   // Since they are not commoned, do not hash them:
2940   return NO_HASH;
2941 }
2942 
2943 // Link together multiple stores (B/S/C/I) into a longer one.
2944 //

3566   }
3567   ss.print_cr("[TraceMergeStores]: with");
3568   merged_input_value->dump("\n", false, &ss);
3569   merged_store->dump("\n", false, &ss);
3570   tty->print("%s", ss.as_string());
3571 }
3572 #endif
3573 
3574 //------------------------------Ideal------------------------------------------
3575 // Change back-to-back Store(, p, x) -> Store(m, p, y) to Store(m, p, x).
3576 // When a store immediately follows a relevant allocation/initialization,
3577 // try to capture it into the initialization, or hoist it above.
3578 Node *StoreNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3579   Node* p = MemNode::Ideal_common(phase, can_reshape);
3580   if (p)  return (p == NodeSentinel) ? nullptr : p;
3581 
3582   Node* mem     = in(MemNode::Memory);
3583   Node* address = in(MemNode::Address);
3584   Node* value   = in(MemNode::ValueIn);
3585   // Back-to-back stores to same address?  Fold em up.  Generally
3586   // unsafe if I have intervening uses...
3587   if (phase->C->get_adr_type(phase->C->get_alias_index(adr_type())) != TypeAryPtr::INLINES) {
3588     Node* st = mem;
3589     // If Store 'st' has more than one use, we cannot fold 'st' away.
3590     // For example, 'st' might be the final state at a conditional
3591     // return.  Or, 'st' might be used by some node which is live at
3592     // the same time 'st' is live, which might be unschedulable.  So,
3593     // require exactly ONE user until such time as we clone 'mem' for
3594     // each of 'mem's uses (thus making the exactly-1-user-rule hold
3595     // true).
3596     while (st->is_Store() && st->outcnt() == 1) {
3597       // Looking at a dead closed cycle of memory?
3598       assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
3599       assert(Opcode() == st->Opcode() ||
3600              st->Opcode() == Op_StoreVector ||
3601              Opcode() == Op_StoreVector ||
3602              st->Opcode() == Op_StoreVectorScatter ||
3603              Opcode() == Op_StoreVectorScatter ||
3604              phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
3605              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
3606              (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
3607              (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
3608              (st->adr_type()->isa_aryptr() && st->adr_type()->is_aryptr()->is_flat()) || // TODO 8343835
3609              (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
3610              "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
3611 
3612       if (st->in(MemNode::Address)->eqv_uncast(address) &&
3613           st->as_Store()->memory_size() <= this->memory_size()) {
3614         Node* use = st->raw_out(0);
3615         if (phase->is_IterGVN()) {
3616           phase->is_IterGVN()->rehash_node_delayed(use);
3617         }
3618         // It's OK to do this in the parser, since DU info is always accurate,
3619         // and the parser always refers to nodes via SafePointNode maps.
3620         use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase);
3621         return this;
3622       }
3623       st = st->in(MemNode::Memory);
3624     }
3625   }
3626 
3627 
3628   // Capture an unaliased, unconditional, simple store into an initializer.

3726       const StoreVectorNode* store_vector = as_StoreVector();
3727       const StoreVectorNode* mem_vector = mem->as_StoreVector();
3728       const Node* store_indices = store_vector->indices();
3729       const Node* mem_indices = mem_vector->indices();
3730       const Node* store_mask = store_vector->mask();
3731       const Node* mem_mask = mem_vector->mask();
3732       // Ensure types, indices, and masks match
3733       if (store_vector->vect_type() == mem_vector->vect_type() &&
3734           ((store_indices == nullptr) == (mem_indices == nullptr) &&
3735            (store_indices == nullptr || store_indices->eqv_uncast(mem_indices))) &&
3736           ((store_mask == nullptr) == (mem_mask == nullptr) &&
3737            (store_mask == nullptr || store_mask->eqv_uncast(mem_mask)))) {
3738         result = mem;
3739       }
3740     }
3741   }
3742 
3743   // Store of zero anywhere into a freshly-allocated object?
3744   // Then the store is useless.
3745   // (It must already have been captured by the InitializeNode.)
3746   if (result == this && ReduceFieldZeroing) {

3747     // a newly allocated object is already all-zeroes everywhere
3748     if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
3749         (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::InitValue) == val)) {
3750       result = mem;
3751     }
3752 
3753     if (result == this && phase->type(val)->is_zero_type()) {
3754       // the store may also apply to zero-bits in an earlier object
3755       Node* prev_mem = find_previous_store(phase);
3756       // Steps (a), (b):  Walk past independent stores to find an exact match.
3757       if (prev_mem != nullptr) {
3758         Node* prev_val = can_see_stored_value(prev_mem, phase);
3759         if (prev_val != nullptr && prev_val == val) {
3760           // prev_val and val might differ by a cast; it would be good
3761           // to keep the more informative of the two.
3762           result = mem;
3763         }
3764       }
3765     }
3766   }
3767 
3768   PhaseIterGVN* igvn = phase->is_IterGVN();
3769   if (result != this && igvn != nullptr) {
3770     MemBarNode* trailing = trailing_membar();
3771     if (trailing != nullptr) {
3772 #ifdef ASSERT
3773       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();

4237 // Clearing a short array is faster with stores
4238 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
4239   // Already know this is a large node, do not try to ideal it
4240   if (_is_large) return nullptr;
4241 
4242   const int unit = BytesPerLong;
4243   const TypeX* t = phase->type(in(2))->isa_intptr_t();
4244   if (!t)  return nullptr;
4245   if (!t->is_con())  return nullptr;
4246   intptr_t raw_count = t->get_con();
4247   intptr_t size = raw_count;
4248   if (!Matcher::init_array_count_is_in_bytes) size *= unit;
4249   // Clearing nothing uses the Identity call.
4250   // Negative clears are possible on dead ClearArrays
4251   // (see jck test stmt114.stmt11402.val).
4252   if (size <= 0 || size % unit != 0)  return nullptr;
4253   intptr_t count = size / unit;
4254   // Length too long; communicate this to matchers and assemblers.
4255   // Assemblers are responsible to produce fast hardware clears for it.
4256   if (size > InitArrayShortSize) {
4257     return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
4258   } else if (size > 2 && Matcher::match_rule_supported_vector(Op_ClearArray, 4, T_LONG)) {
4259     return nullptr;
4260   }
4261   if (!IdealizeClearArrayNode) return nullptr;
4262   Node *mem = in(1);
4263   if( phase->type(mem)==Type::TOP ) return nullptr;
4264   Node *adr = in(3);
4265   const Type* at = phase->type(adr);
4266   if( at==Type::TOP ) return nullptr;
4267   const TypePtr* atp = at->isa_ptr();
4268   // adjust atp to be the correct array element address type
4269   if (atp == nullptr)  atp = TypePtr::BOTTOM;
4270   else              atp = atp->add_offset(Type::OffsetBot);
4271   // Get base for derived pointer purposes
4272   if( adr->Opcode() != Op_AddP ) Unimplemented();
4273   Node *base = adr->in(1);
4274 
4275   Node *val = in(4);
4276   Node *off  = phase->MakeConX(BytesPerLong);
4277   mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4278   count--;
4279   while( count-- ) {
4280     mem = phase->transform(mem);
4281     adr = phase->transform(new AddPNode(base,adr,off));
4282     mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
4283   }
4284   return mem;
4285 }
4286 
4287 //----------------------------step_through----------------------------------
4288 // Return allocation input memory edge if it is different instance
4289 // or itself if it is the one we are looking for.
4290 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseValues* phase) {
4291   Node* n = *np;
4292   assert(n->is_ClearArray(), "sanity");
4293   intptr_t offset;
4294   AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
4295   // This method is called only before Allocate nodes are expanded
4296   // during macro nodes expansion. Before that ClearArray nodes are
4297   // only generated in PhaseMacroExpand::generate_arraycopy() (before
4298   // Allocate nodes are expanded) which follows allocations.
4299   assert(alloc != nullptr, "should have allocation");
4300   if (alloc->_idx == instance_id) {
4301     // Can not bypass initialization of the instance we are looking for.
4302     return false;
4303   }
4304   // Otherwise skip it.
4305   InitializeNode* init = alloc->initialization();
4306   if (init != nullptr)
4307     *np = init->in(TypeFunc::Memory);
4308   else
4309     *np = alloc->in(TypeFunc::Memory);
4310   return true;
4311 }
4312 
4313 //----------------------------clear_memory-------------------------------------
4314 // Generate code to initialize object storage to zero.
4315 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4316                                    Node* val,
4317                                    Node* raw_val,
4318                                    intptr_t start_offset,
4319                                    Node* end_offset,
4320                                    PhaseGVN* phase) {
4321   intptr_t offset = start_offset;
4322 
4323   int unit = BytesPerLong;
4324   if ((offset % unit) != 0) {
4325     Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
4326     adr = phase->transform(adr);
4327     const TypePtr* atp = TypeRawPtr::BOTTOM;
4328     if (val != nullptr) {
4329       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4330       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4331     } else {
4332       assert(raw_val == nullptr, "val may not be null");
4333       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4334     }
4335     mem = phase->transform(mem);
4336     offset += BytesPerInt;
4337   }
4338   assert((offset % unit) == 0, "");
4339 
4340   // Initialize the remaining stuff, if any, with a ClearArray.
4341   return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
4342 }
4343 
4344 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4345                                    Node* raw_val,
4346                                    Node* start_offset,
4347                                    Node* end_offset,
4348                                    PhaseGVN* phase) {
4349   if (start_offset == end_offset) {
4350     // nothing to do
4351     return mem;
4352   }
4353 
4354   int unit = BytesPerLong;
4355   Node* zbase = start_offset;
4356   Node* zend  = end_offset;
4357 
4358   // Scale to the unit required by the CPU:
4359   if (!Matcher::init_array_count_is_in_bytes) {
4360     Node* shift = phase->intcon(exact_log2(unit));
4361     zbase = phase->transform(new URShiftXNode(zbase, shift) );
4362     zend  = phase->transform(new URShiftXNode(zend,  shift) );
4363   }
4364 
4365   // Bulk clear double-words
4366   Node* zsize = phase->transform(new SubXNode(zend, zbase) );
4367   Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
4368   if (raw_val == nullptr) {
4369     raw_val = phase->MakeConX(0);
4370   }
4371   mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
4372   return phase->transform(mem);
4373 }
4374 
4375 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
4376                                    Node* val,
4377                                    Node* raw_val,
4378                                    intptr_t start_offset,
4379                                    intptr_t end_offset,
4380                                    PhaseGVN* phase) {
4381   if (start_offset == end_offset) {
4382     // nothing to do
4383     return mem;
4384   }
4385 
4386   assert((end_offset % BytesPerInt) == 0, "odd end offset");
4387   intptr_t done_offset = end_offset;
4388   if ((done_offset % BytesPerLong) != 0) {
4389     done_offset -= BytesPerInt;
4390   }
4391   if (done_offset > start_offset) {
4392     mem = clear_memory(ctl, mem, dest, val, raw_val,
4393                        start_offset, phase->MakeConX(done_offset), phase);
4394   }
4395   if (done_offset < end_offset) { // emit the final 32-bit store
4396     Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
4397     adr = phase->transform(adr);
4398     const TypePtr* atp = TypeRawPtr::BOTTOM;
4399     if (val != nullptr) {
4400       assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
4401       mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
4402     } else {
4403       assert(raw_val == nullptr, "val may not be null");
4404       mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
4405     }
4406     mem = phase->transform(mem);
4407     done_offset += BytesPerInt;
4408   }
4409   assert(done_offset == end_offset, "");
4410   return mem;
4411 }
4412 
4413 //=============================================================================
4414 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
4415   : MultiNode(TypeFunc::Parms + (precedent == nullptr? 0: 1)),
4416     _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
4417 #ifdef ASSERT
4418   , _pair_idx(0)
4419 #endif
4420 {
4421   init_class_id(Class_MemBar);
4422   Node* top = C->top();
4423   init_req(TypeFunc::I_O,top);
4424   init_req(TypeFunc::FramePtr,top);
4425   init_req(TypeFunc::ReturnAdr,top);

4528       PhaseIterGVN* igvn = phase->is_IterGVN();
4529       remove(igvn);
4530       // Must return either the original node (now dead) or a new node
4531       // (Do not return a top here, since that would break the uniqueness of top.)
4532       return new ConINode(TypeInt::ZERO);
4533     }
4534   }
4535   return progress ? this : nullptr;
4536 }
4537 
4538 //------------------------------Value------------------------------------------
4539 const Type* MemBarNode::Value(PhaseGVN* phase) const {
4540   if( !in(0) ) return Type::TOP;
4541   if( phase->type(in(0)) == Type::TOP )
4542     return Type::TOP;
4543   return TypeTuple::MEMBAR;
4544 }
4545 
4546 //------------------------------match------------------------------------------
4547 // Construct projections for memory.
4548 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
4549   switch (proj->_con) {
4550   case TypeFunc::Control:
4551   case TypeFunc::Memory:
4552     return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
4553   }
4554   ShouldNotReachHere();
4555   return nullptr;
4556 }
4557 
4558 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4559   trailing->_kind = TrailingStore;
4560   leading->_kind = LeadingStore;
4561 #ifdef ASSERT
4562   trailing->_pair_idx = leading->_idx;
4563   leading->_pair_idx = leading->_idx;
4564 #endif
4565 }
4566 
4567 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
4568   trailing->_kind = TrailingLoadStore;

4815   return (req() > RawStores);
4816 }
4817 
4818 void InitializeNode::set_complete(PhaseGVN* phase) {
4819   assert(!is_complete(), "caller responsibility");
4820   _is_complete = Complete;
4821 
4822   // After this node is complete, it contains a bunch of
4823   // raw-memory initializations.  There is no need for
4824   // it to have anything to do with non-raw memory effects.
4825   // Therefore, tell all non-raw users to re-optimize themselves,
4826   // after skipping the memory effects of this initialization.
4827   PhaseIterGVN* igvn = phase->is_IterGVN();
4828   if (igvn)  igvn->add_users_to_worklist(this);
4829 }
4830 
4831 // convenience function
4832 // return false if the init contains any stores already
4833 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
4834   InitializeNode* init = initialization();
4835   if (init == nullptr || init->is_complete()) {
4836     return false;
4837   }
4838   init->remove_extra_zeroes();
4839   // for now, if this allocation has already collected any inits, bail:
4840   if (init->is_non_zero())  return false;
4841   init->set_complete(phase);
4842   return true;
4843 }
4844 
4845 void InitializeNode::remove_extra_zeroes() {
4846   if (req() == RawStores)  return;
4847   Node* zmem = zero_memory();
4848   uint fill = RawStores;
4849   for (uint i = fill; i < req(); i++) {
4850     Node* n = in(i);
4851     if (n->is_top() || n == zmem)  continue;  // skip
4852     if (fill < i)  set_req(fill, n);          // compact
4853     ++fill;
4854   }
4855   // delete any empty spaces created:
4856   while (fill < req()) {
4857     del_req(fill);

5001             // store node that we'd like to capture. We need to check
5002             // the uses of the MergeMemNode.
5003             mems.push(n);
5004           }
5005         } else if (n->is_Mem()) {
5006           Node* other_adr = n->in(MemNode::Address);
5007           if (other_adr == adr) {
5008             failed = true;
5009             break;
5010           } else {
5011             const TypePtr* other_t_adr = phase->type(other_adr)->isa_ptr();
5012             if (other_t_adr != nullptr) {
5013               int other_alias_idx = phase->C->get_alias_index(other_t_adr);
5014               if (other_alias_idx == alias_idx) {
5015                 // A load from the same memory slice as the store right
5016                 // after the InitializeNode. We check the control of the
5017                 // object/array that is loaded from. If it's the same as
5018                 // the store control then we cannot capture the store.
5019                 assert(!n->is_Store(), "2 stores to same slice on same control?");
5020                 Node* base = other_adr;
5021                 if (base->is_Phi()) {
5022                   // In rare case, base may be a PhiNode and it may read
5023                   // the same memory slice between InitializeNode and store.
5024                   failed = true;
5025                   break;
5026                 }
5027                 assert(base->is_AddP(), "should be addp but is %s", base->Name());
5028                 base = base->in(AddPNode::Base);
5029                 if (base != nullptr) {
5030                   base = base->uncast();
5031                   if (base->is_Proj() && base->in(0) == alloc) {
5032                     failed = true;
5033                     break;
5034                   }
5035                 }
5036               }
5037             }
5038           }
5039         } else {
5040           failed = true;
5041           break;
5042         }
5043       }
5044     }
5045   }
5046   if (failed) {

5593         //   z's_done      12  16  16  16    12  16    12
5594         //   z's_needed    12  16  16  16    16  16    16
5595         //   zsize          0   0   0   0     4   0     4
5596         if (next_full_store < 0) {
5597           // Conservative tack:  Zero to end of current word.
5598           zeroes_needed = align_up(zeroes_needed, BytesPerInt);
5599         } else {
5600           // Zero to beginning of next fully initialized word.
5601           // Or, don't zero at all, if we are already in that word.
5602           assert(next_full_store >= zeroes_needed, "must go forward");
5603           assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
5604           zeroes_needed = next_full_store;
5605         }
5606       }
5607 
5608       if (zeroes_needed > zeroes_done) {
5609         intptr_t zsize = zeroes_needed - zeroes_done;
5610         // Do some incremental zeroing on rawmem, in parallel with inits.
5611         zeroes_done = align_down(zeroes_done, BytesPerInt);
5612         rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5613                                               allocation()->in(AllocateNode::InitValue),
5614                                               allocation()->in(AllocateNode::RawInitValue),
5615                                               zeroes_done, zeroes_needed,
5616                                               phase);
5617         zeroes_done = zeroes_needed;
5618         if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
5619           do_zeroing = false;   // leave the hole, next time
5620       }
5621     }
5622 
5623     // Collect the store and move on:
5624     phase->replace_input_of(st, MemNode::Memory, inits);
5625     inits = st;                 // put it on the linearized chain
5626     set_req(i, zmem);           // unhook from previous position
5627 
5628     if (zeroes_done == st_off)
5629       zeroes_done = next_init_off;
5630 
5631     assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
5632 
5633     #ifdef ASSERT
5634     // Various order invariants.  Weaker than stores_are_sane because

5654   remove_extra_zeroes();        // clear out all the zmems left over
5655   add_req(inits);
5656 
5657   if (!(UseTLAB && ZeroTLAB)) {
5658     // If anything remains to be zeroed, zero it all now.
5659     zeroes_done = align_down(zeroes_done, BytesPerInt);
5660     // if it is the last unused 4 bytes of an instance, forget about it
5661     intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
5662     if (zeroes_done + BytesPerLong >= size_limit) {
5663       AllocateNode* alloc = allocation();
5664       assert(alloc != nullptr, "must be present");
5665       if (alloc != nullptr && alloc->Opcode() == Op_Allocate) {
5666         Node* klass_node = alloc->in(AllocateNode::KlassNode);
5667         ciKlass* k = phase->type(klass_node)->is_instklassptr()->instance_klass();
5668         if (zeroes_done == k->layout_helper())
5669           zeroes_done = size_limit;
5670       }
5671     }
5672     if (zeroes_done < size_limit) {
5673       rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
5674                                             allocation()->in(AllocateNode::InitValue),
5675                                             allocation()->in(AllocateNode::RawInitValue),
5676                                             zeroes_done, size_in_bytes, phase);
5677     }
5678   }
5679 
5680   set_complete(phase);
5681   return rawmem;
5682 }
5683 
5684 
5685 #ifdef ASSERT
5686 bool InitializeNode::stores_are_sane(PhaseValues* phase) {
5687   if (is_complete())
5688     return true;                // stores could be anything at this point
5689   assert(allocation() != nullptr, "must be present");
5690   intptr_t last_off = allocation()->minimum_header_size();
5691   for (uint i = InitializeNode::RawStores; i < req(); i++) {
5692     Node* st = in(i);
5693     intptr_t st_off = get_store_offset(st, phase);
5694     if (st_off < 0)  continue;  // ignore dead garbage
5695     if (last_off > st_off) {
< prev index next >