< prev index next >

src/share/vm/opto/loopopts.cpp

Print this page

        

@@ -29,10 +29,11 @@
 #include "opto/divnode.hpp"
 #include "opto/loopnode.hpp"
 #include "opto/matcher.hpp"
 #include "opto/mulnode.hpp"
 #include "opto/rootnode.hpp"
+#include "opto/shenandoahSupport.hpp"
 #include "opto/subnode.hpp"
 
 //=============================================================================
 //------------------------------split_thru_phi---------------------------------
 // Split Node 'n' through merge point if there is enough win.

@@ -115,10 +116,11 @@
       _igvn.set_type(x, t);
       // If x is a TypeNode, capture any more-precise type permanently into Node
       // otherwise it will be not updated during igvn->transform since
       // igvn->type(x) is set to x->Value() already.
       x->raise_bottom_type(t);
+      if (x->Opcode() != Op_ShenandoahWriteBarrier) {
       Node *y = x->Identity(&_igvn);
       if (y != x) {
         wins++;
         x = y;
       } else {

@@ -131,10 +133,13 @@
           // We do not need register_new_node_with_optimizer
           // because set_type has already been called.
           _igvn._worklist.push(x);
         }
       }
+      } else {
+        _igvn._worklist.push(x);
+      }
     }
     if (x != the_clone && the_clone != NULL)
       _igvn.remove_dead_node(the_clone);
     phi->set_req( i, x );
   }

@@ -198,10 +203,48 @@
   }
 
   return phi;
 }
 
+/**
+ * When splitting a Shenandoah write barrier through a phi, we
+ * can not replace the write-barrier input of the ShenandoahWBMemProj
+ * with the phi. We must also split the ShenandoahWBMemProj through the
+ * phi and generate a new memory phi for it.
+ */
+void PhaseIdealLoop::split_mem_thru_phi(Node* n, Node* r, Node* phi) {
+  if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+    if (n->has_out_with(Op_ShenandoahWBMemProj)) {
+      Node* old_mem_phi = n->in(ShenandoahBarrierNode::Memory);
+      assert(r->is_Region(), "need region to control phi");
+      assert(phi->is_Phi(), "expect phi");
+      Node* memphi = PhiNode::make(r, old_mem_phi, Type::MEMORY, C->alias_type(n->adr_type())->adr_type());
+      for (uint i = 1; i < r->req(); i++) {
+        Node* wb = phi->in(i);
+        if (wb->Opcode() == Op_ShenandoahWriteBarrier) {
+          // assert(! wb->has_out_with(Op_ShenandoahWBMemProj), "new clone does not have mem proj");
+          Node* new_proj = new (C) ShenandoahWBMemProjNode(wb);
+          register_new_node(new_proj, r->in(i));
+          memphi->set_req(i, new_proj);
+        } else {
+          if (old_mem_phi->is_Phi() && old_mem_phi->in(0) == r) {
+            memphi->set_req(i, old_mem_phi->in(i));
+          }
+        }
+      }
+      register_new_node(memphi, r);
+      Node* old_mem_out = n->find_out_with(Op_ShenandoahWBMemProj);
+      while (old_mem_out != NULL) {
+        assert(old_mem_out != NULL, "expect memory projection");
+        _igvn.replace_node(old_mem_out, memphi);
+        old_mem_out = n->find_out_with(Op_ShenandoahWBMemProj);
+      }
+    }
+    assert(! n->has_out_with(Op_ShenandoahWBMemProj), "no more memory outs");
+  }
+}
+
 //------------------------------dominated_by------------------------------------
 // Replace the dominated test with an obvious true or false.  Place it on the
 // IGVN worklist for later cleanup.  Move control-dependent data Nodes on the
 // live path up to the dominating control.
 void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff, bool flip, bool exclude_loop_predicate ) {

@@ -302,11 +345,12 @@
       // This allows us to split-up address expressions.
       if (m->is_AddP() &&
           get_ctrl(m->in(2)) != n_ctrl &&
           get_ctrl(m->in(3)) != n_ctrl) {
         // Move the AddP up to dominating point
-        set_ctrl_and_loop(m, find_non_split_ctrl(idom(n_ctrl)));
+        Node* c = find_non_split_ctrl(idom(n_ctrl));
+        set_ctrl_and_loop(m, c);
         continue;
       }
       return NULL;
     }
     assert(m->is_Phi() || is_dominator(get_ctrl(m), n_ctrl), "m has strange control");

@@ -693,10 +737,19 @@
   if( n->is_Con() ) return n;   // No cloning for Con nodes
 
   Node *n_ctrl = get_ctrl(n);
   if( !n_ctrl ) return n;       // Dead node
 
+  try_move_shenandoah_barrier_before_loop(n, n_ctrl);
+
+  Node* res = try_common_shenandoah_barriers(n, n_ctrl);
+  if (res != NULL) {
+    return res;
+  }
+
+  try_move_shenandoah_read_barrier(n, n_ctrl);
+
   // Attempt to remix address expressions for loop invariants
   Node *m = remix_address_expressions( n );
   if( m ) return m;
 
   // Determine if the Node has inputs from some local Phi.

@@ -736,16 +789,22 @@
   Node *phi = split_thru_phi( n, n_blk, policy );
   if (!phi) return n;
 
   // Found a Phi to split thru!
   // Replace 'n' with the new phi
+  split_mem_thru_phi(n, n_blk, phi);
   _igvn.replace_node( n, phi );
   // Moved a load around the loop, 'en-registering' something.
   if (n_blk->is_Loop() && n->is_Load() &&
       !phi->in(LoopNode::LoopBackControl)->is_Load())
     C->set_major_progress();
 
+  // Moved a barrier around the loop, 'en-registering' something.
+  if (n_blk->is_Loop() && n->is_ShenandoahBarrier() &&
+      !phi->in(LoopNode::LoopBackControl)->is_ShenandoahBarrier())
+    C->set_major_progress();
+
   return phi;
 }
 
 static bool merge_point_too_heavy(Compile* C, Node* region) {
   // Bail out if the region and its phis have too many users.

@@ -1004,19 +1063,21 @@
 
             // Find control for 'x' next to use but not inside inner loops.
             // For inner loop uses get the preheader area.
             x_ctrl = place_near_use(x_ctrl);
 
-            if (n->is_Load()) {
+            if (n->is_Load() || n->Opcode() == Op_ShenandoahReadBarrier) {
               // For loads, add a control edge to a CFG node outside of the loop
               // to force them to not combine and return back inside the loop
               // during GVN optimization (4641526).
               //
               // Because we are setting the actual control input, factor in
               // the result from get_late_ctrl() so we respect any
               // anti-dependences. (6233005).
+              if (n->is_Load()) {
               x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
+              }
 
               // Don't allow the control input to be a CFG splitting node.
               // Such nodes should only have ProjNodes as outs, e.g. IfNode
               // should only have IfTrueNode and IfFalseNode (4985384).
               x_ctrl = find_non_split_ctrl(x_ctrl);

@@ -1034,11 +1095,11 @@
             // tried setting control edges on the x's to force them to
             // not combine, but the matching gets worried when it tries
             // to fold a StoreP and an AddP together (as part of an
             // address expression) and the AddP and StoreP have
             // different controls.
-            if (!x->is_Load() && !x->is_DecodeNarrowPtr()) _igvn._worklist.yank(x);
+            if (!x->is_Load() && !x->is_DecodeNarrowPtr() && !x->is_ShenandoahBarrier()) _igvn._worklist.yank(x);
           }
           _igvn.remove_dead_node(n);
         }
       }
     }

@@ -1468,24 +1529,25 @@
         // merge point.  NOW we do need a Phi here.  Each loop exit value
         // is now merged with the peeled body exit; each exit gets its own
         // private Phi and those Phis need to be merged here.
         Node *phi;
         if( prev->is_Region() ) {
-          if( idx == 0 ) {      // Updating control edge?
+          if( idx == 0  && use->Opcode() != Op_ShenandoahWBMemProj) {      // Updating control edge?
             phi = prev;         // Just use existing control
           } else {              // Else need a new Phi
             phi = PhiNode::make( prev, old );
             // Now recursively fix up the new uses of old!
-            for( uint i = 1; i < prev->req(); i++ ) {
+            uint first = use->Opcode() != Op_ShenandoahWBMemProj ? 1 : 0;
+            for( uint i = first; i < prev->req(); i++ ) {
               worklist.push(phi); // Onto worklist once for each 'old' input
             }
           }
         } else {
           // Get new RegionNode merging old and new loop exits
           prev = old_new[prev->_idx];
           assert( prev, "just made this in step 7" );
-          if( idx == 0 ) {      // Updating control edge?
+          if( idx == 0 && use->Opcode() != Op_ShenandoahWBMemProj) {      // Updating control edge?
             phi = prev;         // Just use existing control
           } else {              // Else need a new Phi
             // Make a new Phi merging data values properly
             phi = PhiNode::make( prev, old );
             phi->set_req( 1, nnn );
< prev index next >