< prev index next >

src/share/vm/opto/memnode.cpp

Print this page




  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/systemDictionary.hpp"
  27 #include "compiler/compileLog.hpp"
  28 #include "memory/allocation.inline.hpp"
  29 #include "oops/objArrayKlass.hpp"
  30 #include "opto/addnode.hpp"
  31 #include "opto/cfgnode.hpp"
  32 #include "opto/compile.hpp"
  33 #include "opto/connode.hpp"
  34 #include "opto/loopnode.hpp"
  35 #include "opto/machnode.hpp"
  36 #include "opto/matcher.hpp"
  37 #include "opto/memnode.hpp"
  38 #include "opto/mulnode.hpp"
  39 #include "opto/phaseX.hpp"
  40 #include "opto/regmask.hpp"





  41 
  42 // Portions of code courtesy of Clifford Click
  43 
  44 // Optimization - Graph Style
  45 
  46 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
  47 
  48 //=============================================================================
  49 uint MemNode::size_of() const { return sizeof(*this); }
  50 
  51 const TypePtr *MemNode::adr_type() const {
  52   Node* adr = in(Address);
  53   const TypePtr* cross_check = NULL;
  54   DEBUG_ONLY(cross_check = _adr_type);
  55   return calculate_adr_type(adr->bottom_type(), cross_check);
  56 }
  57 
  58 bool MemNode::check_if_adr_maybe_raw(Node* adr) {
  59   if (adr != NULL) {
  60     if (adr->bottom_type()->base() == Type::RawPtr || adr->bottom_type()->base() == Type::AnyPtr) {


 655     //            TypeRawPtr::BOTTOM.  Needs to be investigated.
 656     if (cross_check != NULL &&
 657         cross_check != TypePtr::BOTTOM &&
 658         cross_check != TypeRawPtr::BOTTOM) {
 659       // Recheck the alias index, to see if it has changed (due to a bug).
 660       Compile* C = Compile::current();
 661       assert(C->get_alias_index(cross_check) == C->get_alias_index(tp),
 662              "must stay in the original alias category");
 663       // The type of the address must be contained in the adr_type,
 664       // disregarding "null"-ness.
 665       // (We make an exception for TypeRawPtr::BOTTOM, which is a bit bucket.)
 666       const TypePtr* tp_notnull = tp->join(TypePtr::NOTNULL)->is_ptr();
 667       assert(cross_check->meet(tp_notnull) == cross_check->remove_speculative(),
 668              "real address must not escape from expected memory type");
 669     }
 670     #endif
 671     return tp;
 672   }
 673 }
 674 
 675 //------------------------adr_phi_is_loop_invariant----------------------------
 676 // A helper function for Ideal_DU_postCCP to check if a Phi in a counted
 677 // loop is loop invariant. Make a quick traversal of Phi and associated
 678 // CastPP nodes, looking to see if they are a closed group within the loop.
 679 bool MemNode::adr_phi_is_loop_invariant(Node* adr_phi, Node* cast) {
 680   // The idea is that the phi-nest must boil down to only CastPP nodes
 681   // with the same data. This implies that any path into the loop already
 682   // includes such a CastPP, and so the original cast, whatever its input,
 683   // must be covered by an equivalent cast, with an earlier control input.
 684   ResourceMark rm;
 685 
 686   // The loop entry input of the phi should be the unique dominating
 687   // node for every Phi/CastPP in the loop.
 688   Unique_Node_List closure;
 689   closure.push(adr_phi->in(LoopNode::EntryControl));
 690 
 691   // Add the phi node and the cast to the worklist.
 692   Unique_Node_List worklist;
 693   worklist.push(adr_phi);
 694   if( cast != NULL ){
 695     if( !cast->is_ConstraintCast() ) return false;
 696     worklist.push(cast);
 697   }
 698 
 699   // Begin recursive walk of phi nodes.
 700   while( worklist.size() ){
 701     // Take a node off the worklist
 702     Node *n = worklist.pop();
 703     if( !closure.member(n) ){
 704       // Add it to the closure.
 705       closure.push(n);
 706       // Make a sanity check to ensure we don't waste too much time here.
 707       if( closure.size() > 20) return false;
 708       // This node is OK if:
 709       //  - it is a cast of an identical value
 710       //  - or it is a phi node (then we add its inputs to the worklist)
 711       // Otherwise, the node is not OK, and we presume the cast is not invariant
 712       if( n->is_ConstraintCast() ){
 713         worklist.push(n->in(1));
 714       } else if( n->is_Phi() ) {
 715         for( uint i = 1; i < n->req(); i++ ) {
 716           worklist.push(n->in(i));
 717         }
 718       } else {
 719         return false;
 720       }
 721     }
 722   }
 723 
 724   // Quit when the worklist is empty, and we've found no offending nodes.
 725   return true;
 726 }
 727 
 728 //------------------------------Ideal_DU_postCCP-------------------------------
 729 // Find any cast-away of null-ness and keep its control.  Null cast-aways are
 730 // going away in this pass and we need to make this memory op depend on the
 731 // gating null check.
 732 Node *MemNode::Ideal_DU_postCCP( PhaseCCP *ccp ) {
 733   return Ideal_common_DU_postCCP(ccp, this, in(MemNode::Address));
 734 }
 735 
 736 // I tried to leave the CastPP's in.  This makes the graph more accurate in
 737 // some sense; we get to keep around the knowledge that an oop is not-null
 738 // after some test.  Alas, the CastPP's interfere with GVN (some values are
 739 // the regular oop, some are the CastPP of the oop, all merge at Phi's which
 740 // cannot collapse, etc).  This cost us 10% on SpecJVM, even when I removed
 741 // some of the more trivial cases in the optimizer.  Removing more useless
 742 // Phi's started allowing Loads to illegally float above null checks.  I gave
 743 // up on this approach.  CNC 10/20/2000
 744 // This static method may be called not from MemNode (EncodePNode calls it).
 745 // Only the control edge of the node 'n' might be updated.
 746 Node *MemNode::Ideal_common_DU_postCCP( PhaseCCP *ccp, Node* n, Node* adr ) {
 747   Node *skipped_cast = NULL;
 748   // Need a null check?  Regular static accesses do not because they are
 749   // from constant addresses.  Array ops are gated by the range check (which
 750   // always includes a NULL check).  Just check field ops.
 751   if( n->in(MemNode::Control) == NULL ) {
 752     // Scan upwards for the highest location we can place this memory op.
 753     while( true ) {
 754       switch( adr->Opcode() ) {
 755 
 756       case Op_AddP:             // No change to NULL-ness, so peek thru AddP's
 757         adr = adr->in(AddPNode::Base);
 758         continue;
 759 
 760       case Op_DecodeN:         // No change to NULL-ness, so peek thru
 761       case Op_DecodeNKlass:
 762         adr = adr->in(1);
 763         continue;
 764 
 765       case Op_EncodeP:
 766       case Op_EncodePKlass:
 767         // EncodeP node's control edge could be set by this method
 768         // when EncodeP node depends on CastPP node.
 769         //
 770         // Use its control edge for memory op because EncodeP may go away
 771         // later when it is folded with following or preceding DecodeN node.
 772         if (adr->in(0) == NULL) {
 773           // Keep looking for cast nodes.
 774           adr = adr->in(1);
 775           continue;
 776         }
 777         ccp->hash_delete(n);
 778         n->set_req(MemNode::Control, adr->in(0));
 779         ccp->hash_insert(n);
 780         return n;
 781 
 782       case Op_CastPP:
 783         // If the CastPP is useless, just peek on through it.
 784         if( ccp->type(adr) == ccp->type(adr->in(1)) ) {
 785           // Remember the cast that we've peeked though. If we peek
 786           // through more than one, then we end up remembering the highest
 787           // one, that is, if in a loop, the one closest to the top.
 788           skipped_cast = adr;
 789           adr = adr->in(1);
 790           continue;
 791         }
 792         // CastPP is going away in this pass!  We need this memory op to be
 793         // control-dependent on the test that is guarding the CastPP.
 794         ccp->hash_delete(n);
 795         n->set_req(MemNode::Control, adr->in(0));
 796         ccp->hash_insert(n);
 797         return n;
 798 
 799       case Op_Phi:
 800         // Attempt to float above a Phi to some dominating point.
 801         if (adr->in(0) != NULL && adr->in(0)->is_CountedLoop()) {
 802           // If we've already peeked through a Cast (which could have set the
 803           // control), we can't float above a Phi, because the skipped Cast
 804           // may not be loop invariant.
 805           if (adr_phi_is_loop_invariant(adr, skipped_cast)) {
 806             adr = adr->in(1);
 807             continue;
 808           }
 809         }
 810 
 811         // Intentional fallthrough!
 812 
 813         // No obvious dominating point.  The mem op is pinned below the Phi
 814         // by the Phi itself.  If the Phi goes away (no true value is merged)
 815         // then the mem op can float, but not indefinitely.  It must be pinned
 816         // behind the controls leading to the Phi.
 817       case Op_CheckCastPP:
 818         // These usually stick around to change address type, however a
 819         // useless one can be elided and we still need to pick up a control edge
 820         if (adr->in(0) == NULL) {
 821           // This CheckCastPP node has NO control and is likely useless. But we
 822           // need check further up the ancestor chain for a control input to keep
 823           // the node in place. 4959717.
 824           skipped_cast = adr;
 825           adr = adr->in(1);
 826           continue;
 827         }
 828         ccp->hash_delete(n);
 829         n->set_req(MemNode::Control, adr->in(0));
 830         ccp->hash_insert(n);
 831         return n;
 832 
 833         // List of "safe" opcodes; those that implicitly block the memory
 834         // op below any null check.
 835       case Op_CastX2P:          // no null checks on native pointers
 836       case Op_Parm:             // 'this' pointer is not null
 837       case Op_LoadP:            // Loading from within a klass
 838       case Op_LoadN:            // Loading from within a klass
 839       case Op_LoadKlass:        // Loading from within a klass
 840       case Op_LoadNKlass:       // Loading from within a klass
 841       case Op_ConP:             // Loading from a klass
 842       case Op_ConN:             // Loading from a klass
 843       case Op_ConNKlass:        // Loading from a klass
 844       case Op_CreateEx:         // Sucking up the guts of an exception oop
 845       case Op_Con:              // Reading from TLS
 846       case Op_CMoveP:           // CMoveP is pinned
 847       case Op_CMoveN:           // CMoveN is pinned
 848         break;                  // No progress
 849 
 850       case Op_Proj:             // Direct call to an allocation routine
 851       case Op_SCMemProj:        // Memory state from store conditional ops
 852 #ifdef ASSERT
 853         {
 854           assert(adr->as_Proj()->_con == TypeFunc::Parms, "must be return value");
 855           const Node* call = adr->in(0);
 856           if (call->is_CallJava()) {
 857             const CallJavaNode* call_java = call->as_CallJava();
 858             const TypeTuple *r = call_java->tf()->range();
 859             assert(r->cnt() > TypeFunc::Parms, "must return value");
 860             const Type* ret_type = r->field_at(TypeFunc::Parms);
 861             assert(ret_type && ret_type->isa_ptr(), "must return pointer");
 862             // We further presume that this is one of
 863             // new_instance_Java, new_array_Java, or
 864             // the like, but do not assert for this.
 865           } else if (call->is_Allocate()) {
 866             // similar case to new_instance_Java, etc.
 867           } else if (!call->is_CallLeaf()) {
 868             // Projections from fetch_oop (OSR) are allowed as well.
 869             ShouldNotReachHere();
 870           }
 871         }
 872 #endif
 873         break;
 874       default:
 875         ShouldNotReachHere();
 876       }
 877       break;
 878     }
 879   }
 880 
 881   return  NULL;               // No progress
 882 }
 883 
 884 
 885 //=============================================================================
 886 // Should LoadNode::Ideal() attempt to remove control edges?
 887 bool LoadNode::can_remove_control() const {
 888   return true;
 889 }
 890 uint LoadNode::size_of() const { return sizeof(*this); }
 891 uint LoadNode::cmp( const Node &n ) const
 892 { return !Type::cmp( _type, ((LoadNode&)n)._type ); }
 893 const Type *LoadNode::bottom_type() const { return _type; }
 894 uint LoadNode::ideal_reg() const {
 895   return _type->ideal_reg();
 896 }
 897 
 898 #ifndef PRODUCT
 899 void LoadNode::dump_spec(outputStream *st) const {
 900   MemNode::dump_spec(st);
 901   if( !Verbose && !WizardMode ) {
 902     // standard dump does this in Verbose and WizardMode
 903     st->print(" #"); _type->dump_on(st);
 904   }


1089       return phase->zerocon(memory_type());
1090     }
1091 
1092     // A load from an initialization barrier can match a captured store.
1093     if (st->is_Proj() && st->in(0)->is_Initialize()) {
1094       InitializeNode* init = st->in(0)->as_Initialize();
1095       AllocateNode* alloc = init->allocation();
1096       if ((alloc != NULL) && (alloc == ld_alloc)) {
1097         // examine a captured store value
1098         st = init->find_captured_store(ld_off, memory_size(), phase);
1099         if (st != NULL)
1100           continue;             // take one more trip around
1101       }
1102     }
1103 
1104     // Load boxed value from result of valueOf() call is input parameter.
1105     if (this->is_Load() && ld_adr->is_AddP() &&
1106         (tp != NULL) && tp->is_ptr_to_boxed_value()) {
1107       intptr_t ignore = 0;
1108       Node* base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ignore);





1109       if (base != NULL && base->is_Proj() &&
1110           base->as_Proj()->_con == TypeFunc::Parms &&
1111           base->in(0)->is_CallStaticJava() &&
1112           base->in(0)->as_CallStaticJava()->is_boxing_method()) {
1113         return base->in(0)->in(TypeFunc::Parms);
1114       }
1115     }
1116 
1117     break;
1118   }
1119 
1120   return NULL;
1121 }
1122 
1123 //----------------------is_instance_field_load_with_local_phi------------------
1124 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1125   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1126       in(Address)->is_AddP() ) {
1127     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1128     // Only instances and boxed values.


1138 }
1139 
1140 //------------------------------Identity---------------------------------------
1141 // Loads are identity if previous store is to same address
1142 Node *LoadNode::Identity( PhaseTransform *phase ) {
1143   // If the previous store-maker is the right kind of Store, and the store is
1144   // to the same address, then we are equal to the value stored.
1145   Node* mem = in(Memory);
1146   Node* value = can_see_stored_value(mem, phase);
1147   if( value ) {
1148     // byte, short & char stores truncate naturally.
1149     // A load has to load the truncated value which requires
1150     // some sort of masking operation and that requires an
1151     // Ideal call instead of an Identity call.
1152     if (memory_size() < BytesPerInt) {
1153       // If the input to the store does not fit with the load's result type,
1154       // it must be truncated via an Ideal call.
1155       if (!phase->type(value)->higher_equal(phase->type(this)))
1156         return this;
1157     }


















1158     // (This works even when value is a Con, but LoadNode::Value
1159     // usually runs first, producing the singleton type of the Con.)















1160     return value;
1161   }
1162 
1163   // Search for an existing data phi which was generated before for the same
1164   // instance's field to avoid infinite generation of phis in a loop.
1165   Node *region = mem->in(0);
1166   if (is_instance_field_load_with_local_phi(region)) {
1167     const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1168     int this_index  = phase->C->get_alias_index(addr_t);
1169     int this_offset = addr_t->offset();
1170     int this_iid    = addr_t->instance_id();
1171     if (!addr_t->is_known_instance() &&
1172          addr_t->is_ptr_to_boxed_value()) {
1173       // Use _idx of address base (could be Phi node) for boxed values.
1174       intptr_t   ignore = 0;
1175       Node*      base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1176       if (base == NULL) {
1177         return this;
1178       }
1179       this_iid = base->_idx;


1678   if (tp == NULL || tp->empty())  return Type::TOP;
1679   int off = tp->offset();
1680   assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
1681   Compile* C = phase->C;
1682 
1683   // Try to guess loaded type from pointer type
1684   if (tp->isa_aryptr()) {
1685     const TypeAryPtr* ary = tp->is_aryptr();
1686     const Type* t = ary->elem();
1687 
1688     // Determine whether the reference is beyond the header or not, by comparing
1689     // the offset against the offset of the start of the array's data.
1690     // Different array types begin at slightly different offsets (12 vs. 16).
1691     // We choose T_BYTE as an example base type that is least restrictive
1692     // as to alignment, which will therefore produce the smallest
1693     // possible base offset.
1694     const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1695     const bool off_beyond_header = ((uint)off >= (uint)min_base_off);
1696 
1697     // Try to constant-fold a stable array element.
1698     if (FoldStableValues && ary->is_stable() && ary->const_oop() != NULL) {
1699       // Make sure the reference is not into the header and the offset is constant
1700       if (off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {

1701         const Type* con_type = fold_stable_ary_elem(ary, off, memory_type());
1702         if (con_type != NULL) {
1703           return con_type;
1704         }
1705       }
1706     }
1707 
1708     // Don't do this for integer types. There is only potential profit if
1709     // the element type t is lower than _type; that is, for int types, if _type is
1710     // more restrictive than t.  This only happens here if one is short and the other
1711     // char (both 16 bits), and in those cases we've made an intentional decision
1712     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1713     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1714     //
1715     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1716     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
1717     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1718     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
1719     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1720     // In fact, that could have been the original type of p1, and p1 could have


2534 const Type *StoreNode::Value( PhaseTransform *phase ) const {
2535   // Either input is TOP ==> the result is TOP
2536   const Type *t1 = phase->type( in(MemNode::Memory) );
2537   if( t1 == Type::TOP ) return Type::TOP;
2538   const Type *t2 = phase->type( in(MemNode::Address) );
2539   if( t2 == Type::TOP ) return Type::TOP;
2540   const Type *t3 = phase->type( in(MemNode::ValueIn) );
2541   if( t3 == Type::TOP ) return Type::TOP;
2542   return Type::MEMORY;
2543 }
2544 
2545 //------------------------------Identity---------------------------------------
2546 // Remove redundant stores:
2547 //   Store(m, p, Load(m, p)) changes to m.
2548 //   Store(, p, x) -> Store(m, p, x) changes to Store(m, p, x).
2549 Node *StoreNode::Identity( PhaseTransform *phase ) {
2550   Node* mem = in(MemNode::Memory);
2551   Node* adr = in(MemNode::Address);
2552   Node* val = in(MemNode::ValueIn);
2553 


2554   // Load then Store?  Then the Store is useless
2555   if (val->is_Load() &&
2556       val->in(MemNode::Address)->eqv_uncast(adr) &&
2557       val->in(MemNode::Memory )->eqv_uncast(mem) &&
2558       val->as_Load()->store_Opcode() == Opcode()) {
2559     return mem;
2560   }
2561 
2562   // Two stores in a row of the same value?
2563   if (mem->is_Store() &&
2564       mem->in(MemNode::Address)->eqv_uncast(adr) &&
2565       mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2566       mem->Opcode() == Opcode()) {
2567     return mem;
2568   }
2569 
2570   // Store of zero anywhere into a freshly-allocated object?
2571   // Then the store is useless.
2572   // (It must already have been captured by the InitializeNode.)
2573   if (ReduceFieldZeroing && phase->type(val)->is_zero_type()) {

2574     // a newly allocated object is already all-zeroes everywhere
2575     if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
2576       return mem;
2577     }
2578 
2579     // the store may also apply to zero-bits in an earlier object
2580     Node* prev_mem = find_previous_store(phase);
2581     // Steps (a), (b):  Walk past independent stores to find an exact match.
2582     if (prev_mem != NULL) {
2583       Node* prev_val = can_see_stored_value(prev_mem, phase);
2584       if (prev_val != NULL && phase->eqv(prev_val, val)) {
2585         // prev_val and val might differ by a cast; it would be good
2586         // to keep the more informative of the two.
2587         return mem;


2588       }
2589     }
2590   }
2591 
2592   return this;












2593 }
2594 
2595 //------------------------------match_edge-------------------------------------
2596 // Do we Match on this edge index or not?  Match only memory & value
2597 uint StoreNode::match_edge(uint idx) const {
2598   return idx == MemNode::Address || idx == MemNode::ValueIn;
2599 }
2600 
2601 //------------------------------cmp--------------------------------------------
2602 // Do not common stores up together.  They generally have to be split
2603 // back up anyways, so do not bother.
2604 uint StoreNode::cmp( const Node &n ) const {
2605   return (&n == this);          // Always fail except on self
2606 }
2607 
2608 //------------------------------Ideal_masked_input-----------------------------
2609 // Check for a useless mask before a partial-word store
2610 // (StoreB ... (AndI valIn conIa) )
2611 // If (conIa & mask == mask) this simplifies to
2612 // (StoreB ... (valIn) )


2651 // For simplicity, we actually check if there are any loads from the
2652 // address stored to, not just for loads of the value stored by this node.
2653 //
2654 bool StoreNode::value_never_loaded( PhaseTransform *phase) const {
2655   Node *adr = in(Address);
2656   const TypeOopPtr *adr_oop = phase->type(adr)->isa_oopptr();
2657   if (adr_oop == NULL)
2658     return false;
2659   if (!adr_oop->is_known_instance_field())
2660     return false; // if not a distinct instance, there may be aliases of the address
2661   for (DUIterator_Fast imax, i = adr->fast_outs(imax); i < imax; i++) {
2662     Node *use = adr->fast_out(i);
2663     int opc = use->Opcode();
2664     if (use->is_Load() || use->is_LoadStore()) {
2665       return false;
2666     }
2667   }
2668   return true;
2669 }
2670 


























2671 //=============================================================================
2672 //------------------------------Ideal------------------------------------------
2673 // If the store is from an AND mask that leaves the low bits untouched, then
2674 // we can skip the AND operation.  If the store is from a sign-extension
2675 // (a left shift, then right shift) we can skip both.
2676 Node *StoreBNode::Ideal(PhaseGVN *phase, bool can_reshape){
2677   Node *progress = StoreNode::Ideal_masked_input(phase, 0xFF);
2678   if( progress != NULL ) return progress;
2679 
2680   progress = StoreNode::Ideal_sign_extended_input(phase, 24);
2681   if( progress != NULL ) return progress;
2682 
2683   // Finally check the default case
2684   return StoreNode::Ideal(phase, can_reshape);
2685 }
2686 
2687 //=============================================================================
2688 //------------------------------Ideal------------------------------------------
2689 // If the store is from an AND mask that leaves the low bits untouched, then
2690 // we can skip the AND operation


2763   init_req(MemNode::Control, c  );
2764   init_req(MemNode::Memory , mem);
2765   init_req(MemNode::Address, adr);
2766   init_req(MemNode::ValueIn, val);
2767   init_class_id(Class_LoadStore);
2768 }
2769 
2770 uint LoadStoreNode::ideal_reg() const {
2771   return _type->ideal_reg();
2772 }
2773 
2774 bool LoadStoreNode::result_not_used() const {
2775   for( DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++ ) {
2776     Node *x = fast_out(i);
2777     if (x->Opcode() == Op_SCMemProj) continue;
2778     return false;
2779   }
2780   return true;
2781 }
2782 
























2783 uint LoadStoreNode::size_of() const { return sizeof(*this); }
2784 
2785 //=============================================================================
2786 //----------------------------------LoadStoreConditionalNode--------------------
2787 LoadStoreConditionalNode::LoadStoreConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex ) : LoadStoreNode(c, mem, adr, val, NULL, TypeInt::BOOL, 5) {
2788   init_req(ExpectedIn, ex );
2789 }
2790 
2791 //=============================================================================
2792 //-------------------------------adr_type--------------------------------------
2793 // Do we Match on this edge index or not?  Do not match memory
2794 const TypePtr* ClearArrayNode::adr_type() const {
2795   Node *adr = in(3);
2796   return MemNode::calculate_adr_type(adr->bottom_type());
2797 }
2798 
2799 //------------------------------match_edge-------------------------------------
2800 // Do we Match on this edge index or not?  Do not match memory
2801 uint ClearArrayNode::match_edge(uint idx) const {
2802   return idx > 1;


2997 uint EncodeISOArrayNode::match_edge(uint idx) const {
2998   return idx == 2 || idx == 3; // EncodeISOArray src (Binary dst len)
2999 }
3000 
3001 //------------------------------Ideal------------------------------------------
3002 // Return a node which is more "ideal" than the current node.  Strip out
3003 // control copies
3004 Node *EncodeISOArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3005   return remove_dead_region(phase, can_reshape) ? this : NULL;
3006 }
3007 
3008 //------------------------------Value------------------------------------------
3009 const Type *EncodeISOArrayNode::Value(PhaseTransform *phase) const {
3010   if (in(0) && phase->type(in(0)) == Type::TOP) return Type::TOP;
3011   return bottom_type();
3012 }
3013 
3014 //=============================================================================
3015 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3016   : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3017     _adr_type(C->get_adr_type(alias_idx))



3018 {
3019   init_class_id(Class_MemBar);
3020   Node* top = C->top();
3021   init_req(TypeFunc::I_O,top);
3022   init_req(TypeFunc::FramePtr,top);
3023   init_req(TypeFunc::ReturnAdr,top);
3024   if (precedent != NULL)
3025     init_req(TypeFunc::Parms, precedent);
3026 }
3027 
3028 //------------------------------cmp--------------------------------------------
3029 uint MemBarNode::hash() const { return NO_HASH; }
3030 uint MemBarNode::cmp( const Node &n ) const {
3031   return (&n == this);          // Always fail except on self
3032 }
3033 
3034 //------------------------------make-------------------------------------------
3035 MemBarNode* MemBarNode::make(Compile* C, int opcode, int atp, Node* pn) {
3036   switch (opcode) {
3037   case Op_MemBarAcquire:     return new(C) MemBarAcquireNode(C, atp, pn);
3038   case Op_LoadFence:         return new(C) LoadFenceNode(C, atp, pn);
3039   case Op_MemBarRelease:     return new(C) MemBarReleaseNode(C, atp, pn);
3040   case Op_StoreFence:        return new(C) StoreFenceNode(C, atp, pn);
3041   case Op_MemBarAcquireLock: return new(C) MemBarAcquireLockNode(C, atp, pn);
3042   case Op_MemBarReleaseLock: return new(C) MemBarReleaseLockNode(C, atp, pn);
3043   case Op_MemBarVolatile:    return new(C) MemBarVolatileNode(C, atp, pn);
3044   case Op_MemBarCPUOrder:    return new(C) MemBarCPUOrderNode(C, atp, pn);
3045   case Op_Initialize:        return new(C) InitializeNode(C, atp, pn);
3046   case Op_MemBarStoreStore:  return new(C) MemBarStoreStoreNode(C, atp, pn);
3047   default: ShouldNotReachHere(); return NULL;
3048   }
3049 }
3050 















3051 //------------------------------Ideal------------------------------------------
3052 // Return a node which is more "ideal" than the current node.  Strip out
3053 // control copies
3054 Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
3055   if (remove_dead_region(phase, can_reshape)) return this;
3056   // Don't bother trying to transform a dead node
3057   if (in(0) && in(0)->is_top()) {
3058     return NULL;
3059   }
3060 
3061   // Eliminate volatile MemBars for scalar replaced objects.
3062   if (can_reshape && req() == (Precedent+1)) {
3063     bool eliminate = false;
3064     int opc = Opcode();
3065     if ((opc == Op_MemBarAcquire || opc == Op_MemBarVolatile)) {
3066       // Volatile field loads and stores.
3067       Node* my_mem = in(MemBarNode::Precedent);
3068       // The MembarAquire may keep an unused LoadNode alive through the Precedent edge
3069       if ((my_mem != NULL) && (opc == Op_MemBarAcquire) && (my_mem->outcnt() == 1)) {
3070         // if the Precedent is a decodeN and its input (a Load) is used at more than one place,


3077         } else {
3078           assert(my_mem->unique_out() == this, "sanity");
3079           del_req(Precedent);
3080           phase->is_IterGVN()->_worklist.push(my_mem); // remove dead node later
3081           my_mem = NULL;
3082         }
3083       }
3084       if (my_mem != NULL && my_mem->is_Mem()) {
3085         const TypeOopPtr* t_oop = my_mem->in(MemNode::Address)->bottom_type()->isa_oopptr();
3086         // Check for scalar replaced object reference.
3087         if( t_oop != NULL && t_oop->is_known_instance_field() &&
3088             t_oop->offset() != Type::OffsetBot &&
3089             t_oop->offset() != Type::OffsetTop) {
3090           eliminate = true;
3091         }
3092       }
3093     } else if (opc == Op_MemBarRelease) {
3094       // Final field stores.
3095       Node* alloc = AllocateNode::Ideal_allocation(in(MemBarNode::Precedent), phase);
3096       if ((alloc != NULL) && alloc->is_Allocate() &&
3097           alloc->as_Allocate()->_is_non_escaping) {


3098         // The allocated object does not escape.
3099         eliminate = true;
3100       }
3101     }
3102     if (eliminate) {
3103       // Replace MemBar projections by its inputs.
3104       PhaseIterGVN* igvn = phase->is_IterGVN();
3105       igvn->replace_node(proj_out(TypeFunc::Memory), in(TypeFunc::Memory));
3106       igvn->replace_node(proj_out(TypeFunc::Control), in(TypeFunc::Control));
3107       // Must return either the original node (now dead) or a new node
3108       // (Do not return a top here, since that would break the uniqueness of top.)
3109       return new (phase->C) ConINode(TypeInt::ZERO);
3110     }
3111   }
3112   return NULL;
3113 }
3114 
3115 //------------------------------Value------------------------------------------
3116 const Type *MemBarNode::Value( PhaseTransform *phase ) const {
3117   if( !in(0) ) return Type::TOP;
3118   if( phase->type(in(0)) == Type::TOP )
3119     return Type::TOP;
3120   return TypeTuple::MEMBAR;
3121 }
3122 
3123 //------------------------------match------------------------------------------
3124 // Construct projections for memory.
3125 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3126   switch (proj->_con) {
3127   case TypeFunc::Control:
3128   case TypeFunc::Memory:
3129     return new (m->C) MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3130   }
3131   ShouldNotReachHere();
3132   return NULL;



































































































































3133 }
3134 
3135 //===========================InitializeNode====================================
3136 // SUMMARY:
3137 // This node acts as a memory barrier on raw memory, after some raw stores.
3138 // The 'cooked' oop value feeds from the Initialize, not the Allocation.
3139 // The Initialize can 'capture' suitably constrained stores as raw inits.
3140 // It can coalesce related raw stores into larger units (called 'tiles').
3141 // It can avoid zeroing new storage for memory units which have raw inits.
3142 // At macro-expansion, it is marked 'complete', and does not optimize further.
3143 //
3144 // EXAMPLE:
3145 // The object 'new short[2]' occupies 16 bytes in a 32-bit machine.
3146 //   ctl = incoming control; mem* = incoming memory
3147 // (Note:  A star * on a memory edge denotes I/O and other standard edges.)
3148 // First allocate uninitialized memory and fill in the header:
3149 //   alloc = (Allocate ctl mem* 16 #short[].klass ...)
3150 //   ctl := alloc.Control; mem* := alloc.Memory*
3151 //   rawmem = alloc.Memory; rawoop = alloc.RawAddress
3152 // Then initialize to zero the non-header parts of the raw memory block:




  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/systemDictionary.hpp"
  27 #include "compiler/compileLog.hpp"
  28 #include "memory/allocation.inline.hpp"
  29 #include "oops/objArrayKlass.hpp"
  30 #include "opto/addnode.hpp"
  31 #include "opto/cfgnode.hpp"
  32 #include "opto/compile.hpp"
  33 #include "opto/connode.hpp"
  34 #include "opto/loopnode.hpp"
  35 #include "opto/machnode.hpp"
  36 #include "opto/matcher.hpp"
  37 #include "opto/memnode.hpp"
  38 #include "opto/mulnode.hpp"
  39 #include "opto/phaseX.hpp"
  40 #include "opto/regmask.hpp"
  41 #if INCLUDE_ALL_GCS
  42 #include "gc_implementation/shenandoah/shenandoahForwarding.hpp"
  43 #include "gc_implementation/shenandoah/c2/shenandoahBarrierSetC2.hpp"
  44 #include "gc_implementation/shenandoah/c2/shenandoahSupport.hpp"
  45 #endif
  46 
  47 // Portions of code courtesy of Clifford Click
  48 
  49 // Optimization - Graph Style
  50 
  51 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem,  const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
  52 
  53 //=============================================================================
  54 uint MemNode::size_of() const { return sizeof(*this); }
  55 
  56 const TypePtr *MemNode::adr_type() const {
  57   Node* adr = in(Address);
  58   const TypePtr* cross_check = NULL;
  59   DEBUG_ONLY(cross_check = _adr_type);
  60   return calculate_adr_type(adr->bottom_type(), cross_check);
  61 }
  62 
  63 bool MemNode::check_if_adr_maybe_raw(Node* adr) {
  64   if (adr != NULL) {
  65     if (adr->bottom_type()->base() == Type::RawPtr || adr->bottom_type()->base() == Type::AnyPtr) {


 660     //            TypeRawPtr::BOTTOM.  Needs to be investigated.
 661     if (cross_check != NULL &&
 662         cross_check != TypePtr::BOTTOM &&
 663         cross_check != TypeRawPtr::BOTTOM) {
 664       // Recheck the alias index, to see if it has changed (due to a bug).
 665       Compile* C = Compile::current();
 666       assert(C->get_alias_index(cross_check) == C->get_alias_index(tp),
 667              "must stay in the original alias category");
 668       // The type of the address must be contained in the adr_type,
 669       // disregarding "null"-ness.
 670       // (We make an exception for TypeRawPtr::BOTTOM, which is a bit bucket.)
 671       const TypePtr* tp_notnull = tp->join(TypePtr::NOTNULL)->is_ptr();
 672       assert(cross_check->meet(tp_notnull) == cross_check->remove_speculative(),
 673              "real address must not escape from expected memory type");
 674     }
 675     #endif
 676     return tp;
 677   }
 678 }
 679 


















































































































































































































 680 //=============================================================================
 681 // Should LoadNode::Ideal() attempt to remove control edges?
 682 bool LoadNode::can_remove_control() const {
 683   return true;
 684 }
 685 uint LoadNode::size_of() const { return sizeof(*this); }
 686 uint LoadNode::cmp( const Node &n ) const
 687 { return !Type::cmp( _type, ((LoadNode&)n)._type ); }
 688 const Type *LoadNode::bottom_type() const { return _type; }
 689 uint LoadNode::ideal_reg() const {
 690   return _type->ideal_reg();
 691 }
 692 
 693 #ifndef PRODUCT
 694 void LoadNode::dump_spec(outputStream *st) const {
 695   MemNode::dump_spec(st);
 696   if( !Verbose && !WizardMode ) {
 697     // standard dump does this in Verbose and WizardMode
 698     st->print(" #"); _type->dump_on(st);
 699   }


 884       return phase->zerocon(memory_type());
 885     }
 886 
 887     // A load from an initialization barrier can match a captured store.
 888     if (st->is_Proj() && st->in(0)->is_Initialize()) {
 889       InitializeNode* init = st->in(0)->as_Initialize();
 890       AllocateNode* alloc = init->allocation();
 891       if ((alloc != NULL) && (alloc == ld_alloc)) {
 892         // examine a captured store value
 893         st = init->find_captured_store(ld_off, memory_size(), phase);
 894         if (st != NULL)
 895           continue;             // take one more trip around
 896       }
 897     }
 898 
 899     // Load boxed value from result of valueOf() call is input parameter.
 900     if (this->is_Load() && ld_adr->is_AddP() &&
 901         (tp != NULL) && tp->is_ptr_to_boxed_value()) {
 902       intptr_t ignore = 0;
 903       Node* base = AddPNode::Ideal_base_and_offset(ld_adr, phase, ignore);
 904 #if INCLUDE_ALL_GCS
 905       if (UseShenandoahGC) {
 906         base = ShenandoahBarrierSetC2::bsc2()->step_over_gc_barrier(base);
 907       }
 908 #endif
 909       if (base != NULL && base->is_Proj() &&
 910           base->as_Proj()->_con == TypeFunc::Parms &&
 911           base->in(0)->is_CallStaticJava() &&
 912           base->in(0)->as_CallStaticJava()->is_boxing_method()) {
 913         return base->in(0)->in(TypeFunc::Parms);
 914       }
 915     }
 916 
 917     break;
 918   }
 919 
 920   return NULL;
 921 }
 922 
 923 //----------------------is_instance_field_load_with_local_phi------------------
 924 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
 925   if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
 926       in(Address)->is_AddP() ) {
 927     const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
 928     // Only instances and boxed values.


 938 }
 939 
 940 //------------------------------Identity---------------------------------------
 941 // Loads are identity if previous store is to same address
 942 Node *LoadNode::Identity( PhaseTransform *phase ) {
 943   // If the previous store-maker is the right kind of Store, and the store is
 944   // to the same address, then we are equal to the value stored.
 945   Node* mem = in(Memory);
 946   Node* value = can_see_stored_value(mem, phase);
 947   if( value ) {
 948     // byte, short & char stores truncate naturally.
 949     // A load has to load the truncated value which requires
 950     // some sort of masking operation and that requires an
 951     // Ideal call instead of an Identity call.
 952     if (memory_size() < BytesPerInt) {
 953       // If the input to the store does not fit with the load's result type,
 954       // it must be truncated via an Ideal call.
 955       if (!phase->type(value)->higher_equal(phase->type(this)))
 956         return this;
 957     }
 958     PhaseIterGVN* igvn = phase->is_IterGVN();
 959     if (UseShenandoahGC &&
 960         igvn != NULL &&
 961         value->is_Phi() &&
 962         value->req() > 2 &&
 963         value->in(1) != NULL &&
 964         value->in(1)->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
 965       if (igvn->_worklist.member(value) ||
 966           igvn->_worklist.member(value->in(0)) ||
 967           (value->in(0)->in(1) != NULL &&
 968            value->in(0)->in(1)->is_IfProj() &&
 969            (igvn->_worklist.member(value->in(0)->in(1)) ||
 970             (value->in(0)->in(1)->in(0) != NULL &&
 971              igvn->_worklist.member(value->in(0)->in(1)->in(0)))))) {
 972         igvn->_worklist.push(this);
 973         return this;
 974       }
 975     }
 976     // (This works even when value is a Con, but LoadNode::Value
 977     // usually runs first, producing the singleton type of the Con.)
 978     // TODO!!
 979     if (false && UseShenandoahGC) {
 980       Node* value_no_barrier = ShenandoahBarrierSetC2::bsc2()->step_over_gc_barrier(value->Opcode() == Op_EncodeP ? value->in(1) : value);
 981       if (value->Opcode() == Op_EncodeP) {
 982         if (value_no_barrier != value->in(1)) {
 983           Node* encode = value->clone();
 984           encode->set_req(1, value_no_barrier);
 985           encode = phase->transform(encode);
 986           return encode;
 987         }
 988       } else {
 989         return value_no_barrier;
 990       }
 991     }
 992 
 993     return value;
 994   }
 995 
 996   // Search for an existing data phi which was generated before for the same
 997   // instance's field to avoid infinite generation of phis in a loop.
 998   Node *region = mem->in(0);
 999   if (is_instance_field_load_with_local_phi(region)) {
1000     const TypeOopPtr *addr_t = in(Address)->bottom_type()->isa_oopptr();
1001     int this_index  = phase->C->get_alias_index(addr_t);
1002     int this_offset = addr_t->offset();
1003     int this_iid    = addr_t->instance_id();
1004     if (!addr_t->is_known_instance() &&
1005          addr_t->is_ptr_to_boxed_value()) {
1006       // Use _idx of address base (could be Phi node) for boxed values.
1007       intptr_t   ignore = 0;
1008       Node*      base = AddPNode::Ideal_base_and_offset(in(Address), phase, ignore);
1009       if (base == NULL) {
1010         return this;
1011       }
1012       this_iid = base->_idx;


1511   if (tp == NULL || tp->empty())  return Type::TOP;
1512   int off = tp->offset();
1513   assert(off != Type::OffsetTop, "case covered by TypePtr::empty");
1514   Compile* C = phase->C;
1515 
1516   // Try to guess loaded type from pointer type
1517   if (tp->isa_aryptr()) {
1518     const TypeAryPtr* ary = tp->is_aryptr();
1519     const Type* t = ary->elem();
1520 
1521     // Determine whether the reference is beyond the header or not, by comparing
1522     // the offset against the offset of the start of the array's data.
1523     // Different array types begin at slightly different offsets (12 vs. 16).
1524     // We choose T_BYTE as an example base type that is least restrictive
1525     // as to alignment, which will therefore produce the smallest
1526     // possible base offset.
1527     const int min_base_off = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1528     const bool off_beyond_header = ((uint)off >= (uint)min_base_off);
1529 
1530     // Try to constant-fold a stable array element.
1531     if (FoldStableValues && ary->is_stable()) {
1532       // Make sure the reference is not into the header and the offset is constant
1533       ciObject* aobj = ary->const_oop();
1534       if (aobj != NULL && off_beyond_header && adr->is_AddP() && off != Type::OffsetBot) {
1535         const Type* con_type = fold_stable_ary_elem(ary, off, memory_type());
1536         if (con_type != NULL) {
1537           return con_type;
1538         }
1539       }
1540     }
1541 
1542     // Don't do this for integer types. There is only potential profit if
1543     // the element type t is lower than _type; that is, for int types, if _type is
1544     // more restrictive than t.  This only happens here if one is short and the other
1545     // char (both 16 bits), and in those cases we've made an intentional decision
1546     // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1547     // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1548     //
1549     // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1550     // where the _gvn.type of the AddP is wider than 8.  This occurs when an earlier
1551     // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1552     // subsumed by p1.  If p1 is on the worklist but has not yet been re-transformed,
1553     // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1554     // In fact, that could have been the original type of p1, and p1 could have


2368 const Type *StoreNode::Value( PhaseTransform *phase ) const {
2369   // Either input is TOP ==> the result is TOP
2370   const Type *t1 = phase->type( in(MemNode::Memory) );
2371   if( t1 == Type::TOP ) return Type::TOP;
2372   const Type *t2 = phase->type( in(MemNode::Address) );
2373   if( t2 == Type::TOP ) return Type::TOP;
2374   const Type *t3 = phase->type( in(MemNode::ValueIn) );
2375   if( t3 == Type::TOP ) return Type::TOP;
2376   return Type::MEMORY;
2377 }
2378 
2379 //------------------------------Identity---------------------------------------
2380 // Remove redundant stores:
2381 //   Store(m, p, Load(m, p)) changes to m.
2382 //   Store(, p, x) -> Store(m, p, x) changes to Store(m, p, x).
2383 Node *StoreNode::Identity( PhaseTransform *phase ) {
2384   Node* mem = in(MemNode::Memory);
2385   Node* adr = in(MemNode::Address);
2386   Node* val = in(MemNode::ValueIn);
2387 
2388   Node* result = this;
2389 
2390   // Load then Store?  Then the Store is useless
2391   if (val->is_Load() &&
2392       val->in(MemNode::Address)->eqv_uncast(adr) &&
2393       val->in(MemNode::Memory )->eqv_uncast(mem) &&
2394       val->as_Load()->store_Opcode() == Opcode()) {
2395     result = mem;
2396   }
2397 
2398   // Two stores in a row of the same value?
2399   if (mem->is_Store() &&
2400       mem->in(MemNode::Address)->eqv_uncast(adr) &&
2401       mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2402       mem->Opcode() == Opcode()) {
2403     result = mem;
2404   }
2405 
2406   // Store of zero anywhere into a freshly-allocated object?
2407   // Then the store is useless.
2408   // (It must already have been captured by the InitializeNode.)
2409   if (result == this &&
2410       ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2411     // a newly allocated object is already all-zeroes everywhere
2412     if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
2413       result = mem;
2414     }
2415 
2416     if (result == this) {
2417       // the store may also apply to zero-bits in an earlier object
2418       Node* prev_mem = find_previous_store(phase);
2419       // Steps (a), (b):  Walk past independent stores to find an exact match.
2420       if (prev_mem != NULL) {
2421         Node* prev_val = can_see_stored_value(prev_mem, phase);
2422         if (prev_val != NULL && phase->eqv(prev_val, val)) {
2423           // prev_val and val might differ by a cast; it would be good
2424           // to keep the more informative of the two.
2425           result = mem;
2426         }
2427       }
2428     }
2429   }
2430 
2431   if (result != this && phase->is_IterGVN() != NULL) {
2432     MemBarNode* trailing = trailing_membar();
2433     if (trailing != NULL) {
2434 #ifdef ASSERT
2435       const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2436       assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects");
2437 #endif
2438       PhaseIterGVN* igvn = phase->is_IterGVN();
2439       trailing->remove(igvn);
2440     }
2441   }
2442 
2443   return result;
2444 }
2445 
2446 //------------------------------match_edge-------------------------------------
2447 // Do we Match on this edge index or not?  Match only memory & value
2448 uint StoreNode::match_edge(uint idx) const {
2449   return idx == MemNode::Address || idx == MemNode::ValueIn;
2450 }
2451 
2452 //------------------------------cmp--------------------------------------------
2453 // Do not common stores up together.  They generally have to be split
2454 // back up anyways, so do not bother.
2455 uint StoreNode::cmp( const Node &n ) const {
2456   return (&n == this);          // Always fail except on self
2457 }
2458 
2459 //------------------------------Ideal_masked_input-----------------------------
2460 // Check for a useless mask before a partial-word store
2461 // (StoreB ... (AndI valIn conIa) )
2462 // If (conIa & mask == mask) this simplifies to
2463 // (StoreB ... (valIn) )


2502 // For simplicity, we actually check if there are any loads from the
2503 // address stored to, not just for loads of the value stored by this node.
2504 //
2505 bool StoreNode::value_never_loaded( PhaseTransform *phase) const {
2506   Node *adr = in(Address);
2507   const TypeOopPtr *adr_oop = phase->type(adr)->isa_oopptr();
2508   if (adr_oop == NULL)
2509     return false;
2510   if (!adr_oop->is_known_instance_field())
2511     return false; // if not a distinct instance, there may be aliases of the address
2512   for (DUIterator_Fast imax, i = adr->fast_outs(imax); i < imax; i++) {
2513     Node *use = adr->fast_out(i);
2514     int opc = use->Opcode();
2515     if (use->is_Load() || use->is_LoadStore()) {
2516       return false;
2517     }
2518   }
2519   return true;
2520 }
2521 
2522 MemBarNode* StoreNode::trailing_membar() const {
2523   if (is_release()) {
2524     MemBarNode* trailing_mb = NULL;
2525     for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
2526       Node* u = fast_out(i);
2527       if (u->is_MemBar()) {
2528         if (u->as_MemBar()->trailing_store()) {
2529           assert(u->Opcode() == Op_MemBarVolatile, "");
2530           assert(trailing_mb == NULL, "only one");
2531           trailing_mb = u->as_MemBar();
2532 #ifdef ASSERT
2533           Node* leading = u->as_MemBar()->leading_membar();
2534           assert(leading->Opcode() == Op_MemBarRelease, "incorrect membar");
2535           assert(leading->as_MemBar()->leading_store(), "incorrect membar pair");
2536           assert(leading->as_MemBar()->trailing_membar() == u, "incorrect membar pair");
2537 #endif
2538         } else {
2539           assert(u->as_MemBar()->standalone(), "");
2540         }
2541       }
2542     }
2543     return trailing_mb;
2544   }
2545   return NULL;
2546 }
2547 
2548 //=============================================================================
2549 //------------------------------Ideal------------------------------------------
2550 // If the store is from an AND mask that leaves the low bits untouched, then
2551 // we can skip the AND operation.  If the store is from a sign-extension
2552 // (a left shift, then right shift) we can skip both.
2553 Node *StoreBNode::Ideal(PhaseGVN *phase, bool can_reshape){
2554   Node *progress = StoreNode::Ideal_masked_input(phase, 0xFF);
2555   if( progress != NULL ) return progress;
2556 
2557   progress = StoreNode::Ideal_sign_extended_input(phase, 24);
2558   if( progress != NULL ) return progress;
2559 
2560   // Finally check the default case
2561   return StoreNode::Ideal(phase, can_reshape);
2562 }
2563 
2564 //=============================================================================
2565 //------------------------------Ideal------------------------------------------
2566 // If the store is from an AND mask that leaves the low bits untouched, then
2567 // we can skip the AND operation


2640   init_req(MemNode::Control, c  );
2641   init_req(MemNode::Memory , mem);
2642   init_req(MemNode::Address, adr);
2643   init_req(MemNode::ValueIn, val);
2644   init_class_id(Class_LoadStore);
2645 }
2646 
2647 uint LoadStoreNode::ideal_reg() const {
2648   return _type->ideal_reg();
2649 }
2650 
2651 bool LoadStoreNode::result_not_used() const {
2652   for( DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++ ) {
2653     Node *x = fast_out(i);
2654     if (x->Opcode() == Op_SCMemProj) continue;
2655     return false;
2656   }
2657   return true;
2658 }
2659 
2660 MemBarNode* LoadStoreNode::trailing_membar() const {
2661   MemBarNode* trailing = NULL;
2662   for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
2663     Node* u = fast_out(i);
2664     if (u->is_MemBar()) {
2665       if (u->as_MemBar()->trailing_load_store()) {
2666         assert(u->Opcode() == Op_MemBarAcquire, "");
2667         assert(trailing == NULL, "only one");
2668         trailing = u->as_MemBar();
2669 #ifdef ASSERT
2670         Node* leading = trailing->leading_membar();
2671         assert(support_IRIW_for_not_multiple_copy_atomic_cpu || leading->Opcode() == Op_MemBarRelease, "incorrect membar");
2672         assert(leading->as_MemBar()->leading_load_store(), "incorrect membar pair");
2673         assert(leading->as_MemBar()->trailing_membar() == trailing, "incorrect membar pair");
2674 #endif
2675       } else {
2676         assert(u->as_MemBar()->standalone(), "wrong barrier kind");
2677       }
2678     }
2679   }
2680 
2681   return trailing;
2682 }
2683 
2684 uint LoadStoreNode::size_of() const { return sizeof(*this); }
2685 
2686 //=============================================================================
2687 //----------------------------------LoadStoreConditionalNode--------------------
2688 LoadStoreConditionalNode::LoadStoreConditionalNode( Node *c, Node *mem, Node *adr, Node *val, Node *ex ) : LoadStoreNode(c, mem, adr, val, NULL, TypeInt::BOOL, 5) {
2689   init_req(ExpectedIn, ex );
2690 }
2691 
2692 //=============================================================================
2693 //-------------------------------adr_type--------------------------------------
2694 // Do we Match on this edge index or not?  Do not match memory
2695 const TypePtr* ClearArrayNode::adr_type() const {
2696   Node *adr = in(3);
2697   return MemNode::calculate_adr_type(adr->bottom_type());
2698 }
2699 
2700 //------------------------------match_edge-------------------------------------
2701 // Do we Match on this edge index or not?  Do not match memory
2702 uint ClearArrayNode::match_edge(uint idx) const {
2703   return idx > 1;


2898 uint EncodeISOArrayNode::match_edge(uint idx) const {
2899   return idx == 2 || idx == 3; // EncodeISOArray src (Binary dst len)
2900 }
2901 
2902 //------------------------------Ideal------------------------------------------
2903 // Return a node which is more "ideal" than the current node.  Strip out
2904 // control copies
2905 Node *EncodeISOArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2906   return remove_dead_region(phase, can_reshape) ? this : NULL;
2907 }
2908 
2909 //------------------------------Value------------------------------------------
2910 const Type *EncodeISOArrayNode::Value(PhaseTransform *phase) const {
2911   if (in(0) && phase->type(in(0)) == Type::TOP) return Type::TOP;
2912   return bottom_type();
2913 }
2914 
2915 //=============================================================================
2916 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
2917   : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
2918   _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
2919 #ifdef ASSERT
2920   , _pair_idx(0)
2921 #endif
2922 {
2923   init_class_id(Class_MemBar);
2924   Node* top = C->top();
2925   init_req(TypeFunc::I_O,top);
2926   init_req(TypeFunc::FramePtr,top);
2927   init_req(TypeFunc::ReturnAdr,top);
2928   if (precedent != NULL)
2929     init_req(TypeFunc::Parms, precedent);
2930 }
2931 
2932 //------------------------------cmp--------------------------------------------
2933 uint MemBarNode::hash() const { return NO_HASH; }
2934 uint MemBarNode::cmp( const Node &n ) const {
2935   return (&n == this);          // Always fail except on self
2936 }
2937 
2938 //------------------------------make-------------------------------------------
2939 MemBarNode* MemBarNode::make(Compile* C, int opcode, int atp, Node* pn) {
2940   switch (opcode) {
2941   case Op_MemBarAcquire:     return new(C) MemBarAcquireNode(C, atp, pn);
2942   case Op_LoadFence:         return new(C) LoadFenceNode(C, atp, pn);
2943   case Op_MemBarRelease:     return new(C) MemBarReleaseNode(C, atp, pn);
2944   case Op_StoreFence:        return new(C) StoreFenceNode(C, atp, pn);
2945   case Op_MemBarAcquireLock: return new(C) MemBarAcquireLockNode(C, atp, pn);
2946   case Op_MemBarReleaseLock: return new(C) MemBarReleaseLockNode(C, atp, pn);
2947   case Op_MemBarVolatile:    return new(C) MemBarVolatileNode(C, atp, pn);
2948   case Op_MemBarCPUOrder:    return new(C) MemBarCPUOrderNode(C, atp, pn);
2949   case Op_Initialize:        return new(C) InitializeNode(C, atp, pn);
2950   case Op_MemBarStoreStore:  return new(C) MemBarStoreStoreNode(C, atp, pn);
2951   default: ShouldNotReachHere(); return NULL;
2952   }
2953 }
2954 
2955 void MemBarNode::remove(PhaseIterGVN *igvn) {
2956   if (outcnt() != 2) {
2957     return;
2958   }
2959   if (trailing_store() || trailing_load_store()) {
2960     MemBarNode* leading = leading_membar();
2961     if (leading != NULL) {
2962       assert(leading->trailing_membar() == this, "inconsistent leading/trailing membars");
2963       leading->remove(igvn);
2964     }
2965   }
2966   igvn->replace_node(proj_out(TypeFunc::Memory), in(TypeFunc::Memory));
2967   igvn->replace_node(proj_out(TypeFunc::Control), in(TypeFunc::Control));
2968 }
2969 
2970 //------------------------------Ideal------------------------------------------
2971 // Return a node which is more "ideal" than the current node.  Strip out
2972 // control copies
2973 Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2974   if (remove_dead_region(phase, can_reshape)) return this;
2975   // Don't bother trying to transform a dead node
2976   if (in(0) && in(0)->is_top()) {
2977     return NULL;
2978   }
2979 
2980   // Eliminate volatile MemBars for scalar replaced objects.
2981   if (can_reshape && req() == (Precedent+1)) {
2982     bool eliminate = false;
2983     int opc = Opcode();
2984     if ((opc == Op_MemBarAcquire || opc == Op_MemBarVolatile)) {
2985       // Volatile field loads and stores.
2986       Node* my_mem = in(MemBarNode::Precedent);
2987       // The MembarAquire may keep an unused LoadNode alive through the Precedent edge
2988       if ((my_mem != NULL) && (opc == Op_MemBarAcquire) && (my_mem->outcnt() == 1)) {
2989         // if the Precedent is a decodeN and its input (a Load) is used at more than one place,


2996         } else {
2997           assert(my_mem->unique_out() == this, "sanity");
2998           del_req(Precedent);
2999           phase->is_IterGVN()->_worklist.push(my_mem); // remove dead node later
3000           my_mem = NULL;
3001         }
3002       }
3003       if (my_mem != NULL && my_mem->is_Mem()) {
3004         const TypeOopPtr* t_oop = my_mem->in(MemNode::Address)->bottom_type()->isa_oopptr();
3005         // Check for scalar replaced object reference.
3006         if( t_oop != NULL && t_oop->is_known_instance_field() &&
3007             t_oop->offset() != Type::OffsetBot &&
3008             t_oop->offset() != Type::OffsetTop) {
3009           eliminate = true;
3010         }
3011       }
3012     } else if (opc == Op_MemBarRelease) {
3013       // Final field stores.
3014       Node* alloc = AllocateNode::Ideal_allocation(in(MemBarNode::Precedent), phase);
3015       if ((alloc != NULL) && alloc->is_Allocate() &&
3016           AARCH64_ONLY ( alloc->as_Allocate()->does_not_escape_thread() )
3017           NOT_AARCH64  ( alloc->as_Allocate()->_is_non_escaping )
3018          ) {
3019         // The allocated object does not escape.
3020         eliminate = true;
3021       }
3022     }
3023     if (eliminate) {
3024       // Replace MemBar projections by its inputs.
3025       PhaseIterGVN* igvn = phase->is_IterGVN();
3026       remove(igvn);

3027       // Must return either the original node (now dead) or a new node
3028       // (Do not return a top here, since that would break the uniqueness of top.)
3029       return new (phase->C) ConINode(TypeInt::ZERO);
3030     }
3031   }
3032   return NULL;
3033 }
3034 
3035 //------------------------------Value------------------------------------------
3036 const Type *MemBarNode::Value( PhaseTransform *phase ) const {
3037   if( !in(0) ) return Type::TOP;
3038   if( phase->type(in(0)) == Type::TOP )
3039     return Type::TOP;
3040   return TypeTuple::MEMBAR;
3041 }
3042 
3043 //------------------------------match------------------------------------------
3044 // Construct projections for memory.
3045 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3046   switch (proj->_con) {
3047   case TypeFunc::Control:
3048   case TypeFunc::Memory:
3049     return new (m->C) MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3050   }
3051   ShouldNotReachHere();
3052   return NULL;
3053 }
3054 
3055 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3056   trailing->_kind = TrailingStore;
3057   leading->_kind = LeadingStore;
3058 #ifdef ASSERT
3059   trailing->_pair_idx = leading->_idx;
3060   leading->_pair_idx = leading->_idx;
3061 #endif
3062 }
3063 
3064 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3065   trailing->_kind = TrailingLoadStore;
3066   leading->_kind = LeadingLoadStore;
3067 #ifdef ASSERT
3068   trailing->_pair_idx = leading->_idx;
3069   leading->_pair_idx = leading->_idx;
3070 #endif
3071 }
3072 
3073 MemBarNode* MemBarNode::trailing_membar() const {
3074   ResourceMark rm;
3075   Node* trailing = (Node*)this;
3076   VectorSet seen(Thread::current()->resource_area());
3077 
3078   Node_Stack multis(0);
3079   do {
3080     Node* c = trailing;
3081     uint i = 0;
3082     do {
3083       trailing = NULL;
3084       for (; i < c->outcnt(); i++) {
3085         Node* next = c->raw_out(i);
3086         if (next != c && next->is_CFG()) {
3087           if (c->is_MultiBranch()) {
3088             if (multis.node() == c) {
3089               multis.set_index(i+1);
3090             } else {
3091               multis.push(c, i+1);
3092             }
3093           }
3094           trailing = next;
3095           break;
3096         }
3097       }
3098       if (trailing != NULL && !seen.test_set(trailing->_idx)) {
3099          break;
3100        }
3101       while (multis.size() > 0) {
3102         c = multis.node();
3103         i = multis.index();
3104         if (i < c->req()) {
3105           break;
3106         }
3107         multis.pop();
3108       }
3109     } while (multis.size() > 0);
3110   } while (!trailing->is_MemBar() || !trailing->as_MemBar()->trailing());
3111 
3112   MemBarNode* mb = trailing->as_MemBar();
3113   assert((mb->_kind == TrailingStore && _kind == LeadingStore) ||
3114          (mb->_kind == TrailingLoadStore && _kind == LeadingLoadStore), "bad trailing membar");
3115   assert(mb->_pair_idx == _pair_idx, "bad trailing membar");
3116   return mb;
3117 }
3118 
3119 MemBarNode* MemBarNode::leading_membar() const {
3120   ResourceMark rm;
3121   VectorSet seen(Thread::current()->resource_area());
3122   Node_Stack regions(0);
3123   Node* leading = in(0);
3124   while (leading != NULL && (!leading->is_MemBar() || !leading->as_MemBar()->leading())) {
3125     while (leading == NULL || leading->is_top() || seen.test_set(leading->_idx)) {
3126       leading = NULL;
3127       while (regions.size() > 0 && leading == NULL) {
3128         Node* r = regions.node();
3129         uint i = regions.index();
3130         if (i < r->req()) {
3131           leading = r->in(i);
3132           regions.set_index(i+1);
3133         } else {
3134           regions.pop();
3135         }
3136       }
3137       if (leading == NULL) {
3138         assert(regions.size() == 0, "all paths should have been tried");
3139         return NULL;
3140       }
3141     }
3142     if (leading->is_Region()) {
3143       regions.push(leading, 2);
3144       leading = leading->in(1);
3145     } else {
3146       leading = leading->in(0);
3147     }
3148   }
3149 #ifdef ASSERT
3150   Unique_Node_List wq;
3151   wq.push((Node*)this);
3152   uint found = 0;
3153   for (uint i = 0; i < wq.size(); i++) {
3154     Node* n = wq.at(i);
3155     if (n->is_Region()) {
3156       for (uint j = 1; j < n->req(); j++) {
3157         Node* in = n->in(j);
3158         if (in != NULL && !in->is_top()) {
3159           wq.push(in);
3160         }
3161       }
3162     } else {
3163       if (n->is_MemBar() && n->as_MemBar()->leading()) {
3164         assert(n == leading, "consistency check failed");
3165         found++;
3166       } else {
3167         Node* in = n->in(0);
3168         if (in != NULL && !in->is_top()) {
3169           wq.push(in);
3170         }
3171       }
3172     }
3173   }
3174   assert(found == 1 || (found == 0 && leading == NULL), "consistency check failed");
3175 #endif
3176   if (leading == NULL) {
3177     return NULL;
3178   }
3179   MemBarNode* mb = leading->as_MemBar();
3180   assert((mb->_kind == LeadingStore && _kind == TrailingStore) ||
3181          (mb->_kind == LeadingLoadStore && _kind == TrailingLoadStore), "bad leading membar");
3182   assert(mb->_pair_idx == _pair_idx, "bad leading membar");
3183   return mb;
3184 }
3185 
3186 //===========================InitializeNode====================================
3187 // SUMMARY:
3188 // This node acts as a memory barrier on raw memory, after some raw stores.
3189 // The 'cooked' oop value feeds from the Initialize, not the Allocation.
3190 // The Initialize can 'capture' suitably constrained stores as raw inits.
3191 // It can coalesce related raw stores into larger units (called 'tiles').
3192 // It can avoid zeroing new storage for memory units which have raw inits.
3193 // At macro-expansion, it is marked 'complete', and does not optimize further.
3194 //
3195 // EXAMPLE:
3196 // The object 'new short[2]' occupies 16 bytes in a 32-bit machine.
3197 //   ctl = incoming control; mem* = incoming memory
3198 // (Note:  A star * on a memory edge denotes I/O and other standard edges.)
3199 // First allocate uninitialized memory and fill in the header:
3200 //   alloc = (Allocate ctl mem* 16 #short[].klass ...)
3201 //   ctl := alloc.Control; mem* := alloc.Memory*
3202 //   rawmem = alloc.Memory; rawoop = alloc.RawAddress
3203 // Then initialize to zero the non-header parts of the raw memory block:


< prev index next >