< prev index next >

src/hotspot/share/opto/loopnode.cpp

Print this page

        

@@ -38,10 +38,14 @@
 #include "opto/idealGraphPrinter.hpp"
 #include "opto/loopnode.hpp"
 #include "opto/mulnode.hpp"
 #include "opto/rootnode.hpp"
 #include "opto/superword.hpp"
+#include "utilities/macros.hpp"
+#if INCLUDE_SHENANDOAHGC
+#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
+#endif
 
 //=============================================================================
 //------------------------------is_loop_iv-------------------------------------
 // Determine if a node is Counted loop induction variable.
 // The method is declared in node.hpp.

@@ -994,14 +998,14 @@
           }
         }
         assert(found_sfpt, "no node in loop that's not input to safepoint");
       }
     }
-    CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd();
-    assert(cle == inner->loopexit_or_null(), "mismatch");
     bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0;
     if (has_skeleton) {
+      CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd();
+      assert(cle == inner->loopexit_or_null(), "mismatch");
       assert(expect_skeleton == 1 || expect_skeleton == -1, "unexpected skeleton node");
       assert(outer->outcnt() == 2, "only phis");
     } else {
       assert(expect_skeleton == 0 || expect_skeleton == -1, "no skeleton node?");
       uint phis = 0;

@@ -1013,16 +1017,36 @@
       }
       for (DUIterator_Fast imax, i = outer->fast_outs(imax); i < imax; i++) {
         Node* u = outer->fast_out(i);
         assert(u == outer || u == inner || u->is_Phi(), "nothing between inner and outer loop");
       }
+      Node* c = inner_out;
       uint stores = 0;
-      for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) {
-        Node* u = inner_out->fast_out(i);
-        if (u->is_Store()) {
-          stores++;
+      for (;;) {
+        for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) {
+          Node* u = c->fast_out(i);
+          if (u->is_Store()) {
+            stores++;
+          }
         }
+        if (c->in(0)->is_CountedLoopEnd() || !UseShenandoahGC) {
+          break;
+        }
+#if INCLUDE_SHENANDOAHGC
+        assert(UseShenandoahGC, "only for shenandoah barriers");
+        assert(c->is_Region() && c->req() == 3, "region that ends barrier");
+        uint j = 1;
+        uint req = c->req();
+        for (; j < req; j++) {
+          Node* in = c->in(j);
+          if (in->is_IfProj() && ShenandoahWriteBarrierNode::is_heap_stable_test(in->in(0))) {
+            c = in->in(0)->in(0);
+            break;
+          }
+        }
+        assert(j < req, "should have found heap stable test");
+#endif
       }
       assert(outer->outcnt() >= phis + 2 && outer->outcnt() <= phis + 2 + stores + 1, "only phis");
     }
     assert(sfpt->outcnt() == 1, "no data node");
     assert(outer_tail->outcnt() == 1 || !has_skeleton, "no data node");

@@ -2711,11 +2735,16 @@
 
 //=============================================================================
 //----------------------------build_and_optimize-------------------------------
 // Create a PhaseLoop.  Build the ideal Loop tree.  Map each Ideal Node to
 // its corresponding LoopNode.  If 'optimize' is true, do some loop cleanups.
-void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts, bool last_round) {
+void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) {
+  bool do_split_ifs = (mode == LoopOptsDefault || mode == LoopOptsZgcLastRound);
+  bool skip_loop_opts = (mode == LoopOptsNone) ;
+  bool shenandoah_opts = (mode == LoopOptsShenandoahExpand ||
+                          mode == LoopOptsShenandoahPostExpand);
+
   ResourceMark rm;
 
   int old_progress = C->major_progress();
   uint orig_worklist_size = _igvn._worklist.size();
 

@@ -2775,11 +2804,11 @@
     }
     return;
   }
 
   // Nothing to do, so get out
-  bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only;
+  bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only && !shenandoah_opts;
   bool do_expensive_nodes = C->should_optimize_expensive_nodes(_igvn);
   if (stop_early && !do_expensive_nodes) {
     _igvn.optimize();           // Cleanup NeverBranches
     return;
   }

@@ -2854,21 +2883,22 @@
   visited.set( C->top()->_idx ); // Set C->top() as visited now
   build_loop_early( visited, worklist, nstack );
 
   // Given early legal placement, try finding counted loops.  This placement
   // is good enough to discover most loop invariants.
-  if( !_verify_me && !_verify_only )
-    _ltree_root->counted_loop( this );
+  if (!_verify_me && !_verify_only && !shenandoah_opts) {
+    _ltree_root->counted_loop(this);
+  }
 
   // Find latest loop placement.  Find ideal loop placement.
   visited.Clear();
   init_dom_lca_tags();
   // Need C->root() on worklist when processing outs
   worklist.push( C->root() );
   NOT_PRODUCT( C->verify_graph_edges(); )
   worklist.push( C->top() );
-  build_loop_late( visited, worklist, nstack );
+  build_loop_late(visited, worklist, nstack, !shenandoah_opts);
 
   if (_verify_only) {
     // restore major progress flag
     for (int i = 0; i < old_progress; i++)
       C->set_major_progress();

@@ -2911,10 +2941,29 @@
   if(TraceLoopOpts && C->has_loops()) {
     _ltree_root->dump();
   }
 #endif
 
+#if INCLUDE_SHENANDOAHGC
+  if (mode == LoopOptsShenandoahExpand) {
+    assert(UseShenandoahGC, "only for shenandoah");
+    ShenandoahWriteBarrierNode::pin_and_expand(this);
+  } else if (mode == LoopOptsShenandoahPostExpand) {
+    assert(UseShenandoahGC, "only for shenandoah");
+    visited.Clear();
+    ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, this);
+  }
+
+  if (shenandoah_opts) {
+    _igvn.optimize();
+    if (C->log() != NULL) {
+      log_loop_tree(_ltree_root, _ltree_root, C->log());
+    }
+    return;
+  }
+#endif
+
   if (skip_loop_opts) {
     // restore major progress flag
     for (int i = 0; i < old_progress; i++) {
       C->set_major_progress();
     }

@@ -2926,10 +2975,17 @@
       log_loop_tree(_ltree_root, _ltree_root, C->log());
     }
     return;
   }
 
+#if INCLUDE_SHENANDOAHGC
+  if (UseShenandoahGC) {
+    GrowableArray<MemoryGraphFixer*> memory_graph_fixers;
+    ShenandoahWriteBarrierNode::optimize_before_expansion(this, memory_graph_fixers, false);
+  }
+#endif
+
   if (ReassociateInvariants) {
     // Reassociate invariants and prep for split_thru_phi
     for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) {
       IdealLoopTree* lpt = iter.current();
       bool is_counted = lpt->is_counted();

@@ -2953,13 +3009,13 @@
 
   // Check for aggressive application of split-if and other transforms
   // that require basic-block info (like cloning through Phi's)
   if( SplitIfBlocks && do_split_ifs ) {
     visited.Clear();
-    split_if_with_blocks( visited, nstack, last_round );
+    split_if_with_blocks( visited, nstack, mode == LoopOptsZgcLastRound );
     NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); );
-    if (last_round) {
+    if (mode == LoopOptsZgcLastRound) {
       C->set_major_progress();
     }
   }
 
   if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) {

@@ -3956,11 +4012,12 @@
       Node* s = mem->fast_out(i);
       worklist.push(s);
     }
     while(worklist.size() != 0 && LCA != early) {
       Node* s = worklist.pop();
-      if (s->is_Load() || s->Opcode() == Op_SafePoint) {
+      if (s->is_Load() || s->is_ShenandoahBarrier() || s->Opcode() == Op_SafePoint ||
+          (UseShenandoahGC && s->is_CallStaticJava() && s->as_CallStaticJava()->uncommon_trap_request() != 0)) {
         continue;
       } else if (s->is_MergeMem()) {
         for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) {
           Node* s1 = s->fast_out(i);
           worklist.push(s1);

@@ -4083,11 +4140,11 @@
 }
 
 //------------------------------build_loop_late--------------------------------
 // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping.
 // Second pass finds latest legal placement, and ideal loop placement.
-void PhaseIdealLoop::build_loop_late( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) {
+void PhaseIdealLoop::build_loop_late(VectorSet &visited, Node_List &worklist, Node_Stack &nstack, bool verify_strip_mined) {
   while (worklist.size() != 0) {
     Node *n = worklist.pop();
     // Only visit once
     if (visited.test_set(n->_idx)) continue;
     uint cnt = n->outcnt();

@@ -4119,11 +4176,11 @@
           // push dead code onto a worklist
           _deadlist.push(use);
         }
       } else {
         // All of n's children have been processed, complete post-processing.
-        build_loop_late_post(n);
+        build_loop_late_post(n, verify_strip_mined);
         if (nstack.is_empty()) {
           // Finished all nodes on stack.
           // Process next node on the worklist.
           break;
         }

@@ -4170,11 +4227,11 @@
 
 
 //------------------------------build_loop_late_post---------------------------
 // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping.
 // Second pass finds latest legal placement, and ideal loop placement.
-void PhaseIdealLoop::build_loop_late_post( Node *n ) {
+void PhaseIdealLoop::build_loop_late_post(Node *n, bool verify_strip_mined) {
 
   if (n->req() == 2 && (n->Opcode() == Op_ConvI2L || n->Opcode() == Op_CastII) && !C->major_progress() && !_verify_only) {
     _igvn._worklist.push(n);  // Maybe we'll normalize it, if no more loops.
   }
 

@@ -4221,10 +4278,15 @@
     case Op_StrComp:            // Does a bunch of load-like effects
     case Op_StrEquals:
     case Op_StrIndexOf:
     case Op_StrIndexOfChar:
     case Op_AryEq:
+#if INCLUDE_SHENANDOAHGC
+    case Op_ShenandoahReadBarrier:
+    case Op_ShenandoahWriteBarrier:
+    case Op_ShenandoahWBMemProj:
+#endif
     case Op_HasNegatives:
       pinned = false;
     }
     if( pinned ) {
       IdealLoopTree *chosen_loop = get_loop(n->is_CFG() ? n : get_ctrl(n));

@@ -4326,17 +4388,29 @@
   }
 #endif
 
   // Assign discovered "here or above" point
   least = find_non_split_ctrl(least);
-  verify_strip_mined_scheduling(n, least);
+  if (verify_strip_mined && !_verify_only) {
+    verify_strip_mined_scheduling(n, least);
+  }
   set_ctrl(n, least);
 
   // Collect inner loop bodies
   IdealLoopTree *chosen_loop = get_loop(least);
   if( !chosen_loop->_child )   // Inner loop?
     chosen_loop->_body.push(n);// Collect inner loops
+
+  if (n->Opcode() == Op_ShenandoahWriteBarrier) {
+    // The write barrier and its memory proj must have the same
+    // control otherwise some loop opts could put nodes (Phis) between
+    // them
+    Node* proj = n->find_out_with(Op_ShenandoahWBMemProj);
+    if (proj != NULL) {
+      set_ctrl_and_loop(proj, least);
+    }
+  }
 }
 
 #ifdef ASSERT
 void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA) {
   tty->print_cr("%s", msg);

@@ -4488,10 +4562,11 @@
         }
       }
     }
   }
 }
+#endif
 
 // Collect a R-P-O for the whole CFG.
 // Result list is in post-order (scan backwards for RPO)
 void PhaseIdealLoop::rpo( Node *start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list ) const {
   stk.push(start, 0);

@@ -4510,11 +4585,10 @@
       rpo_list.push(m);
       stk.pop();
     }
   }
 }
-#endif
 
 
 //=============================================================================
 //------------------------------LoopTreeIterator-----------------------------------
 
< prev index next >