< prev index next >

src/hotspot/share/opto/loopopts.cpp

Print this page

        

@@ -42,10 +42,13 @@
 #include "opto/subnode.hpp"
 #include "utilities/macros.hpp"
 #if INCLUDE_ZGC
 #include "gc/z/c2/zBarrierSetC2.hpp"
 #endif
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
+#endif
 
 //=============================================================================
 //------------------------------split_thru_phi---------------------------------
 // Split Node 'n' through merge point if there is enough win.
 Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {

@@ -127,25 +130,29 @@
       _igvn.set_type(x, t);
       // If x is a TypeNode, capture any more-precise type permanently into Node
       // otherwise it will be not updated during igvn->transform since
       // igvn->type(x) is set to x->Value() already.
       x->raise_bottom_type(t);
-      Node *y = x->Identity(&_igvn);
-      if (y != x) {
-        wins++;
-        x = y;
-      } else {
-        y = _igvn.hash_find(x);
-        if (y) {
+      if (x->Opcode() != Op_ShenandoahWriteBarrier) {
+        Node *y = x->Identity(&_igvn);
+        if (y != x) {
           wins++;
           x = y;
         } else {
-          // Else x is a new node we are keeping
-          // We do not need register_new_node_with_optimizer
-          // because set_type has already been called.
-          _igvn._worklist.push(x);
+          y = _igvn.hash_find(x);
+          if (y) {
+            wins++;
+            x = y;
+          } else {
+            // Else x is a new node we are keeping
+            // We do not need register_new_node_with_optimizer
+            // because set_type has already been called.
+            _igvn._worklist.push(x);
+          }
         }
+      } else {
+        _igvn._worklist.push(x);
       }
     }
     if (x != the_clone && the_clone != NULL)
       _igvn.remove_dead_node(the_clone);
     phi->set_req( i, x );

@@ -210,10 +217,50 @@
   }
 
   return phi;
 }
 
+/**
+ * When splitting a Shenandoah write barrier through a phi, we
+ * can not replace the write-barrier input of the ShenandoahWBMemProj
+ * with the phi. We must also split the ShenandoahWBMemProj through the
+ * phi and generate a new memory phi for it.
+ */
+void PhaseIdealLoop::split_mem_thru_phi(Node* n, Node* r, Node* phi) {
+#if INCLUDE_SHENANDOAHGC
+  if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+    if (n->has_out_with(Op_ShenandoahWBMemProj)) {
+      Node* old_mem_phi = n->in(ShenandoahBarrierNode::Memory);
+      assert(r->is_Region(), "need region to control phi");
+      assert(phi->is_Phi(), "expect phi");
+      Node* memphi = PhiNode::make(r, old_mem_phi, Type::MEMORY, C->alias_type(n->adr_type())->adr_type());
+      for (uint i = 1; i < r->req(); i++) {
+        Node* wb = phi->in(i);
+        if (wb->Opcode() == Op_ShenandoahWriteBarrier) {
+          // assert(! wb->has_out_with(Op_ShenandoahWBMemProj), "new clone does not have mem proj");
+          Node* new_proj = new ShenandoahWBMemProjNode(wb);
+          register_new_node(new_proj, r->in(i));
+          memphi->set_req(i, new_proj);
+        } else {
+          if (old_mem_phi->is_Phi() && old_mem_phi->in(0) == r) {
+            memphi->set_req(i, old_mem_phi->in(i));
+          }
+        }
+      }
+      register_new_node(memphi, r);
+      Node* old_mem_out = n->find_out_with(Op_ShenandoahWBMemProj);
+      while (old_mem_out != NULL) {
+        assert(old_mem_out != NULL, "expect memory projection");
+        _igvn.replace_node(old_mem_out, memphi);
+        old_mem_out = n->find_out_with(Op_ShenandoahWBMemProj);
+      }
+    }
+    assert(! n->has_out_with(Op_ShenandoahWBMemProj), "no more memory outs");
+  }
+#endif
+}
+
 //------------------------------dominated_by------------------------------------
 // Replace the dominated test with an obvious true or false.  Place it on the
 // IGVN worklist for later cleanup.  Move control-dependent data Nodes on the
 // live path up to the dominating control.
 void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff, bool flip, bool exclude_loop_predicate ) {

@@ -916,10 +963,16 @@
   Node* res = try_move_store_before_loop(n, n_ctrl);
   if (res != NULL) {
     return n;
   }
 
+#if INCLUDE_SHENANDOAHGC
+  if (n->Opcode() == Op_ShenandoahReadBarrier) {
+    ((ShenandoahReadBarrierNode*)n)->try_move(n_ctrl, this);
+  }
+#endif
+
   // Attempt to remix address expressions for loop invariants
   Node *m = remix_address_expressions( n );
   if( m ) return m;
 
   if (n->is_ConstraintCast()) {

@@ -973,16 +1026,22 @@
   Node *phi = split_thru_phi( n, n_blk, policy );
   if (!phi) return n;
 
   // Found a Phi to split thru!
   // Replace 'n' with the new phi
+  split_mem_thru_phi(n, n_blk, phi);
   _igvn.replace_node( n, phi );
   // Moved a load around the loop, 'en-registering' something.
   if (n_blk->is_Loop() && n->is_Load() &&
       !phi->in(LoopNode::LoopBackControl)->is_Load())
     C->set_major_progress();
 
+  // Moved a barrier around the loop, 'en-registering' something.
+  if (n_blk->is_Loop() && n->is_ShenandoahBarrier() &&
+      !phi->in(LoopNode::LoopBackControl)->is_ShenandoahBarrier())
+    C->set_major_progress();
+
   return phi;
 }
 
 static bool merge_point_too_heavy(Compile* C, Node* region) {
   // Bail out if the region and its phis have too many users.

@@ -999,10 +1058,33 @@
   } else {
     return false;
   }
 }
 
+static bool merge_point_safe_helper(Node* m) {
+  if (m->is_FastLock()) {
+    return false;
+  }
+#ifdef _LP64
+  if (m->Opcode() == Op_ConvI2L) {
+    return false;
+  }
+  if (m->is_CastII() && m->isa_CastII()->has_range_check()) {
+    return false;
+  }
+#endif
+  if (m->is_ShenandoahBarrier()) {
+    for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
+      Node* n = m->fast_out(i);
+      if (!merge_point_safe_helper(n)) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
 static bool merge_point_safe(Node* region) {
   // 4799512: Stop split_if_with_blocks from splitting a block with a ConvI2LNode
   // having a PhiNode input. This sidesteps the dangerous case where the split
   // ConvI2LNode may become TOP if the input Value() does not
   // overlap the ConvI2L range, leaving a node which may not dominate its

@@ -1015,19 +1097,13 @@
   for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
     Node* n = region->fast_out(i);
     if (n->is_Phi()) {
       for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
         Node* m = n->fast_out(j);
-        if (m->is_FastLock())
-          return false;
-#ifdef _LP64
-        if (m->Opcode() == Op_ConvI2L)
-          return false;
-        if (m->is_CastII() && m->isa_CastII()->has_range_check()) {
+        if (!merge_point_safe_helper(m)) {
           return false;
         }
-#endif
       }
     }
   }
   return true;
 }

@@ -1053,18 +1129,33 @@
 
 bool PhaseIdealLoop::identical_backtoback_ifs(Node *n) {
   if (!n->is_If() || n->is_CountedLoopEnd()) {
     return false;
   }
-  if (!n->in(0)->is_Region()) {
+  Node* region = n->in(0);
+
+  if (!region->is_Region()) {
     return false;
   }
-  Node* region = n->in(0);
   Node* dom = idom(region);
-  if (!dom->is_If() || dom->in(1) != n->in(1)) {
+  if (!dom->is_If()) {
     return false;
   }
+
+#if INCLUDE_SHENANDOAHGC
+  bool shenandoah_heap_stable = ShenandoahWriteBarrierNode::is_heap_stable_test(n);
+  if (shenandoah_heap_stable) {
+    if (!ShenandoahWriteBarrierNode::is_heap_stable_test(dom)) {
+      return false;
+    }
+  } else
+#endif
+  {
+    if (dom->in(1) != n->in(1)) {
+      return false;
+    }
+  }
   IfNode* dom_if = dom->as_If();
   Node* proj_true = dom_if->proj_out(1);
   Node* proj_false = dom_if->proj_out(0);
 
   for (uint i = 1; i < region->req(); i++) {

@@ -1079,10 +1170,11 @@
 
   return true;
 }
 
 bool PhaseIdealLoop::can_split_if(Node *n_ctrl) {
+  assert(n_ctrl->is_Region(), "broken");
   if (C->live_nodes() > 35000) {
     return false; // Method too big
   }
 
   // Do not do 'split-if' if irreducible loops are present.

@@ -1333,19 +1425,21 @@
 
             // Find control for 'x' next to use but not inside inner loops.
             // For inner loop uses get the preheader area.
             x_ctrl = place_near_use(x_ctrl);
 
-            if (n->is_Load()) {
+            if (n->is_Load() || n->Opcode() == Op_ShenandoahReadBarrier) {
               // For loads, add a control edge to a CFG node outside of the loop
               // to force them to not combine and return back inside the loop
               // during GVN optimization (4641526).
               //
               // Because we are setting the actual control input, factor in
               // the result from get_late_ctrl() so we respect any
               // anti-dependences. (6233005).
-              x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
+              if (n->is_Load()) {
+                x_ctrl = dom_lca(late_load_ctrl, x_ctrl);
+              }
 
               // Don't allow the control input to be a CFG splitting node.
               // Such nodes should only have ProjNodes as outs, e.g. IfNode
               // should only have IfTrueNode and IfFalseNode (4985384).
               x_ctrl = find_non_split_ctrl(x_ctrl);

@@ -1363,11 +1457,11 @@
             // tried setting control edges on the x's to force them to
             // not combine, but the matching gets worried when it tries
             // to fold a StoreP and an AddP together (as part of an
             // address expression) and the AddP and StoreP have
             // different controls.
-            if (!x->is_Load() && !x->is_DecodeNarrowPtr()) _igvn._worklist.yank(x);
+            if (!x->is_Load() && !x->is_DecodeNarrowPtr() && !x->is_ShenandoahBarrier() && !x->is_MergeMem()) _igvn._worklist.yank(x);
           }
           _igvn.remove_dead_node(n);
         }
       }
     }

@@ -1700,16 +1794,17 @@
       // merge point.  NOW we do need a Phi here.  Each loop exit value
       // is now merged with the peeled body exit; each exit gets its own
       // private Phi and those Phis need to be merged here.
       Node *phi;
       if( prev->is_Region() ) {
-        if( idx == 0 ) {      // Updating control edge?
+        if (idx == 0 && use->Opcode() != Op_ShenandoahWBMemProj) {      // Updating control edge?
           phi = prev;         // Just use existing control
         } else {              // Else need a new Phi
           phi = PhiNode::make( prev, old );
           // Now recursively fix up the new uses of old!
-          for( uint i = 1; i < prev->req(); i++ ) {
+          uint first = use->Opcode() != Op_ShenandoahWBMemProj ? 1 : 0;
+          for (uint i = first; i < prev->req(); i++) {
             worklist.push(phi); // Onto worklist once for each 'old' input
           }
         }
       } else {
         // Get new RegionNode merging old and new loop exits

@@ -1722,11 +1817,11 @@
           phi = PhiNode::make( prev, old );
           phi->set_req( 1, nnn );
         }
       }
       // If inserting a new Phi, check for prior hits
-      if( idx != 0 ) {
+      if (idx != 0 && use->Opcode() != Op_ShenandoahWBMemProj) {
         Node *hit = _igvn.hash_find_insert(phi);
         if( hit == NULL ) {
           _igvn.register_new_node_with_optimizer(phi); // Register new phi
         } else {                                      // or
           // Remove the new phi from the graph and use the hit

@@ -3130,11 +3225,11 @@
 
         if ( !has_use_internal_to_set(n, peel, loop) ) {
 
           // if not pinned and not a load (which maybe anti-dependent on a store)
           // and not a CMove (Matcher expects only bool->cmove).
-          if ( n->in(0) == NULL && !n->is_Load() && !n->is_CMove() ) {
+          if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove() && n->Opcode() != Op_ShenandoahWBMemProj) {
             cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist );
             sink_list.push(n);
             peel     >>= n->_idx; // delete n from peel set.
             not_peel <<= n->_idx; // add n to not_peel set.
             peel_list.remove(i);
< prev index next >