< prev index next >

src/hotspot/share/opto/loopnode.cpp

Print this page

        

*** 38,47 **** --- 38,51 ---- #include "opto/idealGraphPrinter.hpp" #include "opto/loopnode.hpp" #include "opto/mulnode.hpp" #include "opto/rootnode.hpp" #include "opto/superword.hpp" + #include "utilities/macros.hpp" + #if INCLUDE_SHENANDOAHGC + #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" + #endif //============================================================================= //------------------------------is_loop_iv------------------------------------- // Determine if a node is Counted loop induction variable. // The method is declared in node.hpp.
*** 994,1007 **** } } assert(found_sfpt, "no node in loop that's not input to safepoint"); } } - CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd(); - assert(cle == inner->loopexit_or_null(), "mismatch"); bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0; if (has_skeleton) { assert(expect_skeleton == 1 || expect_skeleton == -1, "unexpected skeleton node"); assert(outer->outcnt() == 2, "only phis"); } else { assert(expect_skeleton == 0 || expect_skeleton == -1, "no skeleton node?"); uint phis = 0; --- 998,1011 ---- } } assert(found_sfpt, "no node in loop that's not input to safepoint"); } } bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0; if (has_skeleton) { + CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd(); + assert(cle == inner->loopexit_or_null(), "mismatch"); assert(expect_skeleton == 1 || expect_skeleton == -1, "unexpected skeleton node"); assert(outer->outcnt() == 2, "only phis"); } else { assert(expect_skeleton == 0 || expect_skeleton == -1, "no skeleton node?"); uint phis = 0;
*** 1013,1028 **** } for (DUIterator_Fast imax, i = outer->fast_outs(imax); i < imax; i++) { Node* u = outer->fast_out(i); assert(u == outer || u == inner || u->is_Phi(), "nothing between inner and outer loop"); } uint stores = 0; ! for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) { ! Node* u = inner_out->fast_out(i); ! if (u->is_Store()) { ! stores++; } } assert(outer->outcnt() >= phis + 2 && outer->outcnt() <= phis + 2 + stores + 1, "only phis"); } assert(sfpt->outcnt() == 1, "no data node"); assert(outer_tail->outcnt() == 1 || !has_skeleton, "no data node"); --- 1017,1052 ---- } for (DUIterator_Fast imax, i = outer->fast_outs(imax); i < imax; i++) { Node* u = outer->fast_out(i); assert(u == outer || u == inner || u->is_Phi(), "nothing between inner and outer loop"); } + Node* c = inner_out; uint stores = 0; ! for (;;) { ! for (DUIterator_Fast imax, i = c->fast_outs(imax); i < imax; i++) { ! Node* u = c->fast_out(i); ! if (u->is_Store()) { ! stores++; ! } } + if (c->in(0)->is_CountedLoopEnd() || !UseShenandoahGC) { + break; + } + #if INCLUDE_SHENANDOAHGC + assert(UseShenandoahGC, "only for shenandoah barriers"); + assert(c->is_Region() && c->req() == 3, "region that ends barrier"); + uint j = 1; + uint req = c->req(); + for (; j < req; j++) { + Node* in = c->in(j); + if (in->is_IfProj() && ShenandoahWriteBarrierNode::is_heap_stable_test(in->in(0))) { + c = in->in(0)->in(0); + break; + } + } + assert(j < req, "should have found heap stable test"); + #endif } assert(outer->outcnt() >= phis + 2 && outer->outcnt() <= phis + 2 + stores + 1, "only phis"); } assert(sfpt->outcnt() == 1, "no data node"); assert(outer_tail->outcnt() == 1 || !has_skeleton, "no data node");
*** 2711,2721 **** //============================================================================= //----------------------------build_and_optimize------------------------------- // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups. ! void PhaseIdealLoop::build_and_optimize(bool do_split_ifs, bool skip_loop_opts, bool last_round) { ResourceMark rm; int old_progress = C->major_progress(); uint orig_worklist_size = _igvn._worklist.size(); --- 2735,2750 ---- //============================================================================= //----------------------------build_and_optimize------------------------------- // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups. ! void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) { ! bool do_split_ifs = (mode == LoopOptsDefault || mode == LoopOptsZgcLastRound); ! bool skip_loop_opts = (mode == LoopOptsNone) ; ! bool shenandoah_opts = (mode == LoopOptsShenandoahExpand || ! mode == LoopOptsShenandoahPostExpand); ! ResourceMark rm; int old_progress = C->major_progress(); uint orig_worklist_size = _igvn._worklist.size();
*** 2775,2785 **** } return; } // Nothing to do, so get out ! bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only; bool do_expensive_nodes = C->should_optimize_expensive_nodes(_igvn); if (stop_early && !do_expensive_nodes) { _igvn.optimize(); // Cleanup NeverBranches return; } --- 2804,2814 ---- } return; } // Nothing to do, so get out ! bool stop_early = !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only && !shenandoah_opts; bool do_expensive_nodes = C->should_optimize_expensive_nodes(_igvn); if (stop_early && !do_expensive_nodes) { _igvn.optimize(); // Cleanup NeverBranches return; }
*** 2854,2874 **** visited.set( C->top()->_idx ); // Set C->top() as visited now build_loop_early( visited, worklist, nstack ); // Given early legal placement, try finding counted loops. This placement // is good enough to discover most loop invariants. ! if( !_verify_me && !_verify_only ) ! _ltree_root->counted_loop( this ); // Find latest loop placement. Find ideal loop placement. visited.Clear(); init_dom_lca_tags(); // Need C->root() on worklist when processing outs worklist.push( C->root() ); NOT_PRODUCT( C->verify_graph_edges(); ) worklist.push( C->top() ); ! build_loop_late( visited, worklist, nstack ); if (_verify_only) { // restore major progress flag for (int i = 0; i < old_progress; i++) C->set_major_progress(); --- 2883,2904 ---- visited.set( C->top()->_idx ); // Set C->top() as visited now build_loop_early( visited, worklist, nstack ); // Given early legal placement, try finding counted loops. This placement // is good enough to discover most loop invariants. ! if (!_verify_me && !_verify_only && !shenandoah_opts) { ! _ltree_root->counted_loop(this); ! } // Find latest loop placement. Find ideal loop placement. visited.Clear(); init_dom_lca_tags(); // Need C->root() on worklist when processing outs worklist.push( C->root() ); NOT_PRODUCT( C->verify_graph_edges(); ) worklist.push( C->top() ); ! build_loop_late(visited, worklist, nstack, !shenandoah_opts); if (_verify_only) { // restore major progress flag for (int i = 0; i < old_progress; i++) C->set_major_progress();
*** 2911,2920 **** --- 2941,2969 ---- if(TraceLoopOpts && C->has_loops()) { _ltree_root->dump(); } #endif + #if INCLUDE_SHENANDOAHGC + if (mode == LoopOptsShenandoahExpand) { + assert(UseShenandoahGC, "only for shenandoah"); + ShenandoahWriteBarrierNode::pin_and_expand(this); + } else if (mode == LoopOptsShenandoahPostExpand) { + assert(UseShenandoahGC, "only for shenandoah"); + visited.Clear(); + ShenandoahWriteBarrierNode::optimize_after_expansion(visited, nstack, worklist, this); + } + + if (shenandoah_opts) { + _igvn.optimize(); + if (C->log() != NULL) { + log_loop_tree(_ltree_root, _ltree_root, C->log()); + } + return; + } + #endif + if (skip_loop_opts) { // restore major progress flag for (int i = 0; i < old_progress; i++) { C->set_major_progress(); }
*** 2926,2935 **** --- 2975,2991 ---- log_loop_tree(_ltree_root, _ltree_root, C->log()); } return; } + #if INCLUDE_SHENANDOAHGC + if (UseShenandoahGC) { + GrowableArray<MemoryGraphFixer*> memory_graph_fixers; + ShenandoahWriteBarrierNode::optimize_before_expansion(this, memory_graph_fixers, false); + } + #endif + if (ReassociateInvariants) { // Reassociate invariants and prep for split_thru_phi for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { IdealLoopTree* lpt = iter.current(); bool is_counted = lpt->is_counted();
*** 2953,2965 **** // Check for aggressive application of split-if and other transforms // that require basic-block info (like cloning through Phi's) if( SplitIfBlocks && do_split_ifs ) { visited.Clear(); ! split_if_with_blocks( visited, nstack, last_round ); NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); ); ! if (last_round) { C->set_major_progress(); } } if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) { --- 3009,3021 ---- // Check for aggressive application of split-if and other transforms // that require basic-block info (like cloning through Phi's) if( SplitIfBlocks && do_split_ifs ) { visited.Clear(); ! split_if_with_blocks( visited, nstack, mode == LoopOptsZgcLastRound ); NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); ); ! if (mode == LoopOptsZgcLastRound) { C->set_major_progress(); } } if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) {
*** 3956,3966 **** Node* s = mem->fast_out(i); worklist.push(s); } while(worklist.size() != 0 && LCA != early) { Node* s = worklist.pop(); ! if (s->is_Load() || s->Opcode() == Op_SafePoint) { continue; } else if (s->is_MergeMem()) { for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) { Node* s1 = s->fast_out(i); worklist.push(s1); --- 4012,4023 ---- Node* s = mem->fast_out(i); worklist.push(s); } while(worklist.size() != 0 && LCA != early) { Node* s = worklist.pop(); ! if (s->is_Load() || s->is_ShenandoahBarrier() || s->Opcode() == Op_SafePoint || ! (UseShenandoahGC && s->is_CallStaticJava() && s->as_CallStaticJava()->uncommon_trap_request() != 0)) { continue; } else if (s->is_MergeMem()) { for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) { Node* s1 = s->fast_out(i); worklist.push(s1);
*** 4083,4093 **** } //------------------------------build_loop_late-------------------------------- // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. // Second pass finds latest legal placement, and ideal loop placement. ! void PhaseIdealLoop::build_loop_late( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) { while (worklist.size() != 0) { Node *n = worklist.pop(); // Only visit once if (visited.test_set(n->_idx)) continue; uint cnt = n->outcnt(); --- 4140,4150 ---- } //------------------------------build_loop_late-------------------------------- // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. // Second pass finds latest legal placement, and ideal loop placement. ! void PhaseIdealLoop::build_loop_late(VectorSet &visited, Node_List &worklist, Node_Stack &nstack, bool verify_strip_mined) { while (worklist.size() != 0) { Node *n = worklist.pop(); // Only visit once if (visited.test_set(n->_idx)) continue; uint cnt = n->outcnt();
*** 4119,4129 **** // push dead code onto a worklist _deadlist.push(use); } } else { // All of n's children have been processed, complete post-processing. ! build_loop_late_post(n); if (nstack.is_empty()) { // Finished all nodes on stack. // Process next node on the worklist. break; } --- 4176,4186 ---- // push dead code onto a worklist _deadlist.push(use); } } else { // All of n's children have been processed, complete post-processing. ! build_loop_late_post(n, verify_strip_mined); if (nstack.is_empty()) { // Finished all nodes on stack. // Process next node on the worklist. break; }
*** 4170,4180 **** //------------------------------build_loop_late_post--------------------------- // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. // Second pass finds latest legal placement, and ideal loop placement. ! void PhaseIdealLoop::build_loop_late_post( Node *n ) { if (n->req() == 2 && (n->Opcode() == Op_ConvI2L || n->Opcode() == Op_CastII) && !C->major_progress() && !_verify_only) { _igvn._worklist.push(n); // Maybe we'll normalize it, if no more loops. } --- 4227,4237 ---- //------------------------------build_loop_late_post--------------------------- // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. // Second pass finds latest legal placement, and ideal loop placement. ! void PhaseIdealLoop::build_loop_late_post(Node *n, bool verify_strip_mined) { if (n->req() == 2 && (n->Opcode() == Op_ConvI2L || n->Opcode() == Op_CastII) && !C->major_progress() && !_verify_only) { _igvn._worklist.push(n); // Maybe we'll normalize it, if no more loops. }
*** 4221,4230 **** --- 4278,4292 ---- case Op_StrComp: // Does a bunch of load-like effects case Op_StrEquals: case Op_StrIndexOf: case Op_StrIndexOfChar: case Op_AryEq: + #if INCLUDE_SHENANDOAHGC + case Op_ShenandoahReadBarrier: + case Op_ShenandoahWriteBarrier: + case Op_ShenandoahWBMemProj: + #endif case Op_HasNegatives: pinned = false; } if( pinned ) { IdealLoopTree *chosen_loop = get_loop(n->is_CFG() ? n : get_ctrl(n));
*** 4326,4342 **** } #endif // Assign discovered "here or above" point least = find_non_split_ctrl(least); ! verify_strip_mined_scheduling(n, least); set_ctrl(n, least); // Collect inner loop bodies IdealLoopTree *chosen_loop = get_loop(least); if( !chosen_loop->_child ) // Inner loop? chosen_loop->_body.push(n);// Collect inner loops } #ifdef ASSERT void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA) { tty->print_cr("%s", msg); --- 4388,4416 ---- } #endif // Assign discovered "here or above" point least = find_non_split_ctrl(least); ! if (verify_strip_mined && !_verify_only) { ! verify_strip_mined_scheduling(n, least); ! } set_ctrl(n, least); // Collect inner loop bodies IdealLoopTree *chosen_loop = get_loop(least); if( !chosen_loop->_child ) // Inner loop? chosen_loop->_body.push(n);// Collect inner loops + + if (n->Opcode() == Op_ShenandoahWriteBarrier) { + // The write barrier and its memory proj must have the same + // control otherwise some loop opts could put nodes (Phis) between + // them + Node* proj = n->find_out_with(Op_ShenandoahWBMemProj); + if (proj != NULL) { + set_ctrl_and_loop(proj, least); + } + } } #ifdef ASSERT void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA) { tty->print_cr("%s", msg);
*** 4488,4497 **** --- 4562,4572 ---- } } } } } + #endif // Collect a R-P-O for the whole CFG. // Result list is in post-order (scan backwards for RPO) void PhaseIdealLoop::rpo( Node *start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list ) const { stk.push(start, 0);
*** 4510,4520 **** rpo_list.push(m); stk.pop(); } } } - #endif //============================================================================= //------------------------------LoopTreeIterator----------------------------------- --- 4585,4594 ----
< prev index next >