< prev index next >

src/hotspot/share/opto/loopopts.cpp

Print this page

        

*** 42,51 **** --- 42,54 ---- #include "opto/subnode.hpp" #include "utilities/macros.hpp" #if INCLUDE_ZGC #include "gc/z/c2/zBarrierSetC2.hpp" #endif + #if INCLUDE_SHENANDOAHGC + #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" + #endif //============================================================================= //------------------------------split_thru_phi--------------------------------- // Split Node 'n' through merge point if there is enough win. Node *PhaseIdealLoop::split_thru_phi( Node *n, Node *region, int policy ) {
*** 127,151 **** _igvn.set_type(x, t); // If x is a TypeNode, capture any more-precise type permanently into Node // otherwise it will be not updated during igvn->transform since // igvn->type(x) is set to x->Value() already. x->raise_bottom_type(t); ! Node *y = x->Identity(&_igvn); ! if (y != x) { ! wins++; ! x = y; ! } else { ! y = _igvn.hash_find(x); ! if (y) { wins++; x = y; } else { ! // Else x is a new node we are keeping ! // We do not need register_new_node_with_optimizer ! // because set_type has already been called. ! _igvn._worklist.push(x); } } } if (x != the_clone && the_clone != NULL) _igvn.remove_dead_node(the_clone); phi->set_req( i, x ); --- 130,158 ---- _igvn.set_type(x, t); // If x is a TypeNode, capture any more-precise type permanently into Node // otherwise it will be not updated during igvn->transform since // igvn->type(x) is set to x->Value() already. x->raise_bottom_type(t); ! if (x->Opcode() != Op_ShenandoahWriteBarrier) { ! Node *y = x->Identity(&_igvn); ! if (y != x) { wins++; x = y; } else { ! y = _igvn.hash_find(x); ! if (y) { ! wins++; ! x = y; ! } else { ! // Else x is a new node we are keeping ! // We do not need register_new_node_with_optimizer ! // because set_type has already been called. ! _igvn._worklist.push(x); ! } } + } else { + _igvn._worklist.push(x); } } if (x != the_clone && the_clone != NULL) _igvn.remove_dead_node(the_clone); phi->set_req( i, x );
*** 210,219 **** --- 217,266 ---- } return phi; } + /** + * When splitting a Shenandoah write barrier through a phi, we + * can not replace the write-barrier input of the ShenandoahWBMemProj + * with the phi. We must also split the ShenandoahWBMemProj through the + * phi and generate a new memory phi for it. + */ + void PhaseIdealLoop::split_mem_thru_phi(Node* n, Node* r, Node* phi) { + #if INCLUDE_SHENANDOAHGC + if (n->Opcode() == Op_ShenandoahWriteBarrier) { + if (n->has_out_with(Op_ShenandoahWBMemProj)) { + Node* old_mem_phi = n->in(ShenandoahBarrierNode::Memory); + assert(r->is_Region(), "need region to control phi"); + assert(phi->is_Phi(), "expect phi"); + Node* memphi = PhiNode::make(r, old_mem_phi, Type::MEMORY, C->alias_type(n->adr_type())->adr_type()); + for (uint i = 1; i < r->req(); i++) { + Node* wb = phi->in(i); + if (wb->Opcode() == Op_ShenandoahWriteBarrier) { + // assert(! wb->has_out_with(Op_ShenandoahWBMemProj), "new clone does not have mem proj"); + Node* new_proj = new ShenandoahWBMemProjNode(wb); + register_new_node(new_proj, r->in(i)); + memphi->set_req(i, new_proj); + } else { + if (old_mem_phi->is_Phi() && old_mem_phi->in(0) == r) { + memphi->set_req(i, old_mem_phi->in(i)); + } + } + } + register_new_node(memphi, r); + Node* old_mem_out = n->find_out_with(Op_ShenandoahWBMemProj); + while (old_mem_out != NULL) { + assert(old_mem_out != NULL, "expect memory projection"); + _igvn.replace_node(old_mem_out, memphi); + old_mem_out = n->find_out_with(Op_ShenandoahWBMemProj); + } + } + assert(! n->has_out_with(Op_ShenandoahWBMemProj), "no more memory outs"); + } + #endif + } + //------------------------------dominated_by------------------------------------ // Replace the dominated test with an obvious true or false. Place it on the // IGVN worklist for later cleanup. Move control-dependent data Nodes on the // live path up to the dominating control. void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff, bool flip, bool exclude_loop_predicate ) {
*** 916,925 **** --- 963,978 ---- Node* res = try_move_store_before_loop(n, n_ctrl); if (res != NULL) { return n; } + #if INCLUDE_SHENANDOAHGC + if (n->Opcode() == Op_ShenandoahReadBarrier) { + ((ShenandoahReadBarrierNode*)n)->try_move(n_ctrl, this); + } + #endif + // Attempt to remix address expressions for loop invariants Node *m = remix_address_expressions( n ); if( m ) return m; if (n->is_ConstraintCast()) {
*** 973,988 **** --- 1026,1047 ---- Node *phi = split_thru_phi( n, n_blk, policy ); if (!phi) return n; // Found a Phi to split thru! // Replace 'n' with the new phi + split_mem_thru_phi(n, n_blk, phi); _igvn.replace_node( n, phi ); // Moved a load around the loop, 'en-registering' something. if (n_blk->is_Loop() && n->is_Load() && !phi->in(LoopNode::LoopBackControl)->is_Load()) C->set_major_progress(); + // Moved a barrier around the loop, 'en-registering' something. + if (n_blk->is_Loop() && n->is_ShenandoahBarrier() && + !phi->in(LoopNode::LoopBackControl)->is_ShenandoahBarrier()) + C->set_major_progress(); + return phi; } static bool merge_point_too_heavy(Compile* C, Node* region) { // Bail out if the region and its phis have too many users.
*** 999,1008 **** --- 1058,1090 ---- } else { return false; } } + static bool merge_point_safe_helper(Node* m) { + if (m->is_FastLock()) { + return false; + } + #ifdef _LP64 + if (m->Opcode() == Op_ConvI2L) { + return false; + } + if (m->is_CastII() && m->isa_CastII()->has_range_check()) { + return false; + } + #endif + if (m->is_ShenandoahBarrier()) { + for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) { + Node* n = m->fast_out(i); + if (!merge_point_safe_helper(n)) { + return false; + } + } + } + return true; + } + static bool merge_point_safe(Node* region) { // 4799512: Stop split_if_with_blocks from splitting a block with a ConvI2LNode // having a PhiNode input. This sidesteps the dangerous case where the split // ConvI2LNode may become TOP if the input Value() does not // overlap the ConvI2L range, leaving a node which may not dominate its
*** 1015,1033 **** for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { Node* n = region->fast_out(i); if (n->is_Phi()) { for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { Node* m = n->fast_out(j); ! if (m->is_FastLock()) ! return false; ! #ifdef _LP64 ! if (m->Opcode() == Op_ConvI2L) ! return false; ! if (m->is_CastII() && m->isa_CastII()->has_range_check()) { return false; } - #endif } } } return true; } --- 1097,1109 ---- for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { Node* n = region->fast_out(i); if (n->is_Phi()) { for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) { Node* m = n->fast_out(j); ! if (!merge_point_safe_helper(m)) { return false; } } } } return true; }
*** 1053,1070 **** bool PhaseIdealLoop::identical_backtoback_ifs(Node *n) { if (!n->is_If() || n->is_CountedLoopEnd()) { return false; } ! if (!n->in(0)->is_Region()) { return false; } - Node* region = n->in(0); Node* dom = idom(region); ! if (!dom->is_If() || dom->in(1) != n->in(1)) { return false; } IfNode* dom_if = dom->as_If(); Node* proj_true = dom_if->proj_out(1); Node* proj_false = dom_if->proj_out(0); for (uint i = 1; i < region->req(); i++) { --- 1129,1161 ---- bool PhaseIdealLoop::identical_backtoback_ifs(Node *n) { if (!n->is_If() || n->is_CountedLoopEnd()) { return false; } ! Node* region = n->in(0); ! ! if (!region->is_Region()) { return false; } Node* dom = idom(region); ! if (!dom->is_If()) { return false; } + + #if INCLUDE_SHENANDOAHGC + bool shenandoah_heap_stable = ShenandoahWriteBarrierNode::is_heap_stable_test(n); + if (shenandoah_heap_stable) { + if (!ShenandoahWriteBarrierNode::is_heap_stable_test(dom)) { + return false; + } + } else + #endif + { + if (dom->in(1) != n->in(1)) { + return false; + } + } IfNode* dom_if = dom->as_If(); Node* proj_true = dom_if->proj_out(1); Node* proj_false = dom_if->proj_out(0); for (uint i = 1; i < region->req(); i++) {
*** 1079,1088 **** --- 1170,1180 ---- return true; } bool PhaseIdealLoop::can_split_if(Node *n_ctrl) { + assert(n_ctrl->is_Region(), "broken"); if (C->live_nodes() > 35000) { return false; // Method too big } // Do not do 'split-if' if irreducible loops are present.
*** 1333,1351 **** // Find control for 'x' next to use but not inside inner loops. // For inner loop uses get the preheader area. x_ctrl = place_near_use(x_ctrl); ! if (n->is_Load()) { // For loads, add a control edge to a CFG node outside of the loop // to force them to not combine and return back inside the loop // during GVN optimization (4641526). // // Because we are setting the actual control input, factor in // the result from get_late_ctrl() so we respect any // anti-dependences. (6233005). ! x_ctrl = dom_lca(late_load_ctrl, x_ctrl); // Don't allow the control input to be a CFG splitting node. // Such nodes should only have ProjNodes as outs, e.g. IfNode // should only have IfTrueNode and IfFalseNode (4985384). x_ctrl = find_non_split_ctrl(x_ctrl); --- 1425,1445 ---- // Find control for 'x' next to use but not inside inner loops. // For inner loop uses get the preheader area. x_ctrl = place_near_use(x_ctrl); ! if (n->is_Load() || n->Opcode() == Op_ShenandoahReadBarrier) { // For loads, add a control edge to a CFG node outside of the loop // to force them to not combine and return back inside the loop // during GVN optimization (4641526). // // Because we are setting the actual control input, factor in // the result from get_late_ctrl() so we respect any // anti-dependences. (6233005). ! if (n->is_Load()) { ! x_ctrl = dom_lca(late_load_ctrl, x_ctrl); ! } // Don't allow the control input to be a CFG splitting node. // Such nodes should only have ProjNodes as outs, e.g. IfNode // should only have IfTrueNode and IfFalseNode (4985384). x_ctrl = find_non_split_ctrl(x_ctrl);
*** 1363,1373 **** // tried setting control edges on the x's to force them to // not combine, but the matching gets worried when it tries // to fold a StoreP and an AddP together (as part of an // address expression) and the AddP and StoreP have // different controls. ! if (!x->is_Load() && !x->is_DecodeNarrowPtr()) _igvn._worklist.yank(x); } _igvn.remove_dead_node(n); } } } --- 1457,1467 ---- // tried setting control edges on the x's to force them to // not combine, but the matching gets worried when it tries // to fold a StoreP and an AddP together (as part of an // address expression) and the AddP and StoreP have // different controls. ! if (!x->is_Load() && !x->is_DecodeNarrowPtr() && !x->is_ShenandoahBarrier() && !x->is_MergeMem()) _igvn._worklist.yank(x); } _igvn.remove_dead_node(n); } } }
*** 1700,1715 **** // merge point. NOW we do need a Phi here. Each loop exit value // is now merged with the peeled body exit; each exit gets its own // private Phi and those Phis need to be merged here. Node *phi; if( prev->is_Region() ) { ! if( idx == 0 ) { // Updating control edge? phi = prev; // Just use existing control } else { // Else need a new Phi phi = PhiNode::make( prev, old ); // Now recursively fix up the new uses of old! ! for( uint i = 1; i < prev->req(); i++ ) { worklist.push(phi); // Onto worklist once for each 'old' input } } } else { // Get new RegionNode merging old and new loop exits --- 1794,1810 ---- // merge point. NOW we do need a Phi here. Each loop exit value // is now merged with the peeled body exit; each exit gets its own // private Phi and those Phis need to be merged here. Node *phi; if( prev->is_Region() ) { ! if (idx == 0 && use->Opcode() != Op_ShenandoahWBMemProj) { // Updating control edge? phi = prev; // Just use existing control } else { // Else need a new Phi phi = PhiNode::make( prev, old ); // Now recursively fix up the new uses of old! ! uint first = use->Opcode() != Op_ShenandoahWBMemProj ? 1 : 0; ! for (uint i = first; i < prev->req(); i++) { worklist.push(phi); // Onto worklist once for each 'old' input } } } else { // Get new RegionNode merging old and new loop exits
*** 1722,1732 **** phi = PhiNode::make( prev, old ); phi->set_req( 1, nnn ); } } // If inserting a new Phi, check for prior hits ! if( idx != 0 ) { Node *hit = _igvn.hash_find_insert(phi); if( hit == NULL ) { _igvn.register_new_node_with_optimizer(phi); // Register new phi } else { // or // Remove the new phi from the graph and use the hit --- 1817,1827 ---- phi = PhiNode::make( prev, old ); phi->set_req( 1, nnn ); } } // If inserting a new Phi, check for prior hits ! if (idx != 0 && use->Opcode() != Op_ShenandoahWBMemProj) { Node *hit = _igvn.hash_find_insert(phi); if( hit == NULL ) { _igvn.register_new_node_with_optimizer(phi); // Register new phi } else { // or // Remove the new phi from the graph and use the hit
*** 3130,3140 **** if ( !has_use_internal_to_set(n, peel, loop) ) { // if not pinned and not a load (which maybe anti-dependent on a store) // and not a CMove (Matcher expects only bool->cmove). ! if ( n->in(0) == NULL && !n->is_Load() && !n->is_CMove() ) { cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist ); sink_list.push(n); peel >>= n->_idx; // delete n from peel set. not_peel <<= n->_idx; // add n to not_peel set. peel_list.remove(i); --- 3225,3235 ---- if ( !has_use_internal_to_set(n, peel, loop) ) { // if not pinned and not a load (which maybe anti-dependent on a store) // and not a CMove (Matcher expects only bool->cmove). ! if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove() && n->Opcode() != Op_ShenandoahWBMemProj) { cloned_for_outside_use += clone_for_use_outside_loop( loop, n, worklist ); sink_list.push(n); peel >>= n->_idx; // delete n from peel set. not_peel <<= n->_idx; // add n to not_peel set. peel_list.remove(i);
< prev index next >