< prev index next >

src/hotspot/share/opto/loopPredicate.cpp

Print this page




 284     igvn->register_new_node_with_optimizer(opq);
 285     igvn->register_new_node_with_optimizer(bol);
 286   }
 287   igvn->hash_delete(iff);
 288   iff->set_req(1, bol);
 289   return new_predicate_proj;
 290 }
 291 
 292 
 293 //--------------------------clone_loop_predicates-----------------------
 294 // Interface from IGVN
 295 Node* PhaseIterGVN::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) {
 296   return PhaseIdealLoop::clone_loop_predicates(old_entry, new_entry, clone_limit_check, NULL, this);
 297 }
 298 
 299 // Interface from PhaseIdealLoop
 300 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) {
 301   return clone_loop_predicates(old_entry, new_entry, clone_limit_check, this, &this->_igvn);
 302 }
 303 






































































 304 // Clone loop predicates to cloned loops (peeled, unswitched, split_if).
 305 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry,
 306                                             bool clone_limit_check,
 307                                             PhaseIdealLoop* loop_phase,
 308                                             PhaseIterGVN* igvn) {
 309 #ifdef ASSERT
 310   if (new_entry == NULL || !(new_entry->is_Proj() || new_entry->is_Region() || new_entry->is_SafePoint())) {
 311     if (new_entry != NULL)
 312       new_entry->dump();
 313     assert(false, "not IfTrue, IfFalse, Region or SafePoint");
 314   }
 315 #endif
 316   // Search original predicates
 317   Node* entry = old_entry;
 318   ProjNode* limit_check_proj = NULL;
 319   limit_check_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
 320   if (limit_check_proj != NULL) {
 321     entry = skip_loop_predicates(entry);
 322   }
 323   ProjNode* profile_predicate_proj = NULL;
 324   ProjNode* predicate_proj = NULL;
 325   if (UseProfiledLoopPredicate) {
 326     profile_predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
 327     if (profile_predicate_proj != NULL) {
 328       entry = skip_loop_predicates(entry);
 329     }
 330   }
 331   if (UseLoopPredicate) {
 332     predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
 333   }
 334   if (predicate_proj != NULL) { // right pattern that can be used by loop predication
 335     // clone predicate
 336     new_entry = clone_predicate(predicate_proj, new_entry,
 337                                 Deoptimization::Reason_predicate,
 338                                 loop_phase, igvn);
 339     assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate");

 340     if (TraceLoopPredicate) {
 341       tty->print("Loop Predicate cloned: ");
 342       debug_only( new_entry->in(0)->dump(); );
 343     }









 344   }
 345   if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication
 346     // clone predicate
 347     new_entry = clone_predicate(profile_predicate_proj, new_entry,
 348                                 Deoptimization::Reason_profile_predicate,
 349                                 loop_phase, igvn);
 350     assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate");
 351     if (TraceLoopPredicate) {
 352       tty->print("Loop Predicate cloned: ");
 353       debug_only( new_entry->in(0)->dump(); );
 354     }
 355   }
 356   if (limit_check_proj != NULL && clone_limit_check) {
 357     // Clone loop limit check last to insert it before loop.
 358     // Don't clone a limit check which was already finalized
 359     // for this counted loop (only one limit check is needed).
 360     new_entry = clone_predicate(limit_check_proj, new_entry,
 361                                 Deoptimization::Reason_loop_limit_check,
 362                                 loop_phase, igvn);
 363     assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone limit check");


 442 }
 443 
 444 //------------------------------Invariance-----------------------------------
 445 // Helper class for loop_predication_impl to compute invariance on the fly and
 446 // clone invariants.
 447 class Invariance : public StackObj {
 448   VectorSet _visited, _invariant;
 449   Node_Stack _stack;
 450   VectorSet _clone_visited;
 451   Node_List _old_new; // map of old to new (clone)
 452   IdealLoopTree* _lpt;
 453   PhaseIdealLoop* _phase;
 454 
 455   // Helper function to set up the invariance for invariance computation
 456   // If n is a known invariant, set up directly. Otherwise, look up the
 457   // the possibility to push n onto the stack for further processing.
 458   void visit(Node* use, Node* n) {
 459     if (_lpt->is_invariant(n)) { // known invariant
 460       _invariant.set(n->_idx);
 461     } else if (!n->is_CFG()) {



 462       Node *n_ctrl = _phase->ctrl_or_self(n);
 463       Node *u_ctrl = _phase->ctrl_or_self(use); // self if use is a CFG
 464       if (_phase->is_dominator(n_ctrl, u_ctrl)) {
 465         _stack.push(n, n->in(0) == NULL ? 1 : 0);
 466       }
 467     }
 468   }
 469 
 470   // Compute invariance for "the_node" and (possibly) all its inputs recursively
 471   // on the fly
 472   void compute_invariance(Node* n) {
 473     assert(_visited.test(n->_idx), "must be");
 474     visit(n, n);
 475     while (_stack.is_nonempty()) {
 476       Node*  n = _stack.node();
 477       uint idx = _stack.index();
 478       if (idx == n->req()) { // all inputs are processed
 479         _stack.pop();
 480         // n is invariant if it's inputs are all invariant
 481         bool all_inputs_invariant = true;
 482         for (uint i = 0; i < n->req(); i++) {
 483           Node* in = n->in(i);
 484           if (in == NULL) continue;
 485           assert(_visited.test(in->_idx), "must have visited input");
 486           if (!_invariant.test(in->_idx)) { // bad guy
 487             all_inputs_invariant = false;
 488             break;
 489           }
 490         }
 491         if (all_inputs_invariant) {
 492           // If n's control is a predicate that was moved out of the
 493           // loop, it was marked invariant but n is only invariant if
 494           // it depends only on that test. Otherwise, unless that test
 495           // is out of the loop, it's not invariant.
 496           if (n->is_CFG() || n->depends_only_on_test() || n->in(0) == NULL || !_phase->is_member(_lpt, n->in(0))) {
 497             _invariant.set(n->_idx); // I am a invariant too
 498           }
 499         }
 500       } else { // process next input
 501         _stack.set_index(idx + 1);
 502         Node* m = n->in(idx);
 503         if (m != NULL && !_visited.test_set(m->_idx)) {
 504           visit(n, m);
 505         }
 506       }
 507     }
 508   }
 509 
 510   // Helper function to set up _old_new map for clone_nodes.
 511   // If n is a known invariant, set up directly ("clone" of n == n).
 512   // Otherwise, push n onto the stack for real cloning.
 513   void clone_visit(Node* n) {
 514     assert(_invariant.test(n->_idx), "must be invariant");
 515     if (_lpt->is_invariant(n)) { // known invariant
 516       _old_new.map(n->_idx, n);




 284     igvn->register_new_node_with_optimizer(opq);
 285     igvn->register_new_node_with_optimizer(bol);
 286   }
 287   igvn->hash_delete(iff);
 288   iff->set_req(1, bol);
 289   return new_predicate_proj;
 290 }
 291 
 292 
 293 //--------------------------clone_loop_predicates-----------------------
 294 // Interface from IGVN
 295 Node* PhaseIterGVN::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) {
 296   return PhaseIdealLoop::clone_loop_predicates(old_entry, new_entry, clone_limit_check, NULL, this);
 297 }
 298 
 299 // Interface from PhaseIdealLoop
 300 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) {
 301   return clone_loop_predicates(old_entry, new_entry, clone_limit_check, this, &this->_igvn);
 302 }
 303 
 304 void PhaseIdealLoop::clone_loop_predicates_fix_mem(ProjNode* dom_proj , ProjNode* proj,
 305                                                    PhaseIdealLoop* loop_phase,
 306                                                    PhaseIterGVN* igvn) {
 307   if (!UseShenandoahGC) {
 308     return;
 309   }
 310   Compile* C = NULL;
 311   if (loop_phase != NULL) {
 312     igvn = &loop_phase->igvn();
 313   }
 314   C = igvn->C;
 315   ProjNode* other_dom_proj = dom_proj->in(0)->as_Multi()->proj_out(1-dom_proj->_con);
 316   Node* dom_r = other_dom_proj->unique_ctrl_out();
 317   if (dom_r->is_Region()) {
 318     assert(dom_r->unique_ctrl_out()->is_Call(), "unc expected");
 319     ProjNode* other_proj = proj->in(0)->as_Multi()->proj_out(1-proj->_con);
 320     Node* r = other_proj->unique_ctrl_out();
 321     assert(r->is_Region() && r->unique_ctrl_out()->is_Call(), "cloned predicate should have caused region to be added");
 322     for (DUIterator_Fast imax, i = dom_r->fast_outs(imax); i < imax; i++) {
 323       Node* dom_use = dom_r->fast_out(i);
 324       if (dom_use->is_Phi() && dom_use->bottom_type() == Type::MEMORY) {
 325         assert(dom_use->in(0) == dom_r, "");
 326         Node* phi = NULL;
 327         for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) {
 328           Node* use = r->fast_out(j);
 329           if (use->is_Phi() && use->bottom_type() == Type::MEMORY &&
 330               use->adr_type() == dom_use->adr_type()) {
 331             assert(use->in(0) == r, "");
 332             assert(phi == NULL, "only one phi");
 333             phi = use;
 334           }
 335         }
 336         if (phi == NULL) {
 337           const TypePtr* adr_type = dom_use->adr_type();
 338           int alias = C->get_alias_index(adr_type);
 339           Node* call = r->unique_ctrl_out();
 340           Node* mem = call->in(TypeFunc::Memory);
 341           MergeMemNode* mm = NULL;
 342           if (mem->is_MergeMem()) {
 343             mm = mem->clone()->as_MergeMem();
 344             if (adr_type == TypePtr::BOTTOM) {
 345               mem = mem->as_MergeMem()->base_memory();
 346             } else {
 347               mem = mem->as_MergeMem()->memory_at(alias);
 348             }
 349           } else {
 350             mm = MergeMemNode::make(mem);
 351           }
 352           phi = PhiNode::make(r, mem, Type::MEMORY, adr_type);
 353           if (adr_type == TypePtr::BOTTOM) {
 354             mm->set_base_memory(phi);
 355           } else {
 356             mm->set_memory_at(alias, phi);
 357           }
 358           if (loop_phase != NULL) {
 359             loop_phase->register_new_node(mm, r);
 360             loop_phase->register_new_node(phi, r);
 361           } else {
 362             igvn->register_new_node_with_optimizer(mm);
 363             igvn->register_new_node_with_optimizer(phi);
 364           }
 365           igvn->replace_input_of(call, TypeFunc::Memory, mm);
 366         }
 367         igvn->replace_input_of(phi, r->req()-1, dom_use->in(dom_r->find_edge(other_dom_proj)));
 368       }
 369     }
 370   }
 371 }
 372 
 373 
 374 // Clone loop predicates to cloned loops (peeled, unswitched, split_if).
 375 Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry,
 376                                             bool clone_limit_check,
 377                                             PhaseIdealLoop* loop_phase,
 378                                             PhaseIterGVN* igvn) {
 379 #ifdef ASSERT
 380   if (new_entry == NULL || !(new_entry->is_Proj() || new_entry->is_Region() || new_entry->is_SafePoint())) {
 381     if (new_entry != NULL)
 382       new_entry->dump();
 383     assert(false, "not IfTrue, IfFalse, Region or SafePoint");
 384   }
 385 #endif
 386   // Search original predicates
 387   Node* entry = old_entry;
 388   ProjNode* limit_check_proj = NULL;
 389   limit_check_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
 390   if (limit_check_proj != NULL) {
 391     entry = skip_loop_predicates(entry);
 392   }
 393   ProjNode* profile_predicate_proj = NULL;
 394   ProjNode* predicate_proj = NULL;
 395   if (UseProfiledLoopPredicate) {
 396     profile_predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
 397     if (profile_predicate_proj != NULL) {
 398       entry = skip_loop_predicates(entry);
 399     }
 400   }
 401   if (UseLoopPredicate) {
 402     predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
 403   }
 404   if (predicate_proj != NULL) { // right pattern that can be used by loop predication
 405     // clone predicate
 406     ProjNode* proj = clone_predicate(predicate_proj, new_entry,
 407                                      Deoptimization::Reason_predicate,
 408                                      loop_phase, igvn);
 409     assert(proj != NULL, "IfTrue or IfFalse after clone predicate");
 410     new_entry = proj;
 411     if (TraceLoopPredicate) {
 412       tty->print("Loop Predicate cloned: ");
 413       debug_only( new_entry->in(0)->dump(); );
 414     }
 415     if (profile_predicate_proj != NULL) {
 416       // A node that produces memory may be out of loop and depend on
 417       // a profiled predicates. In that case the memory state at the
 418       // end of profiled predicates and at the end of predicates are
 419       // not the same. The cloned predicates are dominated by the
 420       // profiled predicates but may have the wrong memory
 421       // state. Update it.
 422       clone_loop_predicates_fix_mem(profile_predicate_proj, proj, loop_phase, igvn);
 423     }
 424   }
 425   if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication
 426     // clone predicate
 427     new_entry = clone_predicate(profile_predicate_proj, new_entry,
 428                                 Deoptimization::Reason_profile_predicate,
 429                                 loop_phase, igvn);
 430     assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate");
 431     if (TraceLoopPredicate) {
 432       tty->print("Loop Predicate cloned: ");
 433       debug_only( new_entry->in(0)->dump(); );
 434     }
 435   }
 436   if (limit_check_proj != NULL && clone_limit_check) {
 437     // Clone loop limit check last to insert it before loop.
 438     // Don't clone a limit check which was already finalized
 439     // for this counted loop (only one limit check is needed).
 440     new_entry = clone_predicate(limit_check_proj, new_entry,
 441                                 Deoptimization::Reason_loop_limit_check,
 442                                 loop_phase, igvn);
 443     assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone limit check");


 522 }
 523 
 524 //------------------------------Invariance-----------------------------------
 525 // Helper class for loop_predication_impl to compute invariance on the fly and
 526 // clone invariants.
 527 class Invariance : public StackObj {
 528   VectorSet _visited, _invariant;
 529   Node_Stack _stack;
 530   VectorSet _clone_visited;
 531   Node_List _old_new; // map of old to new (clone)
 532   IdealLoopTree* _lpt;
 533   PhaseIdealLoop* _phase;
 534 
 535   // Helper function to set up the invariance for invariance computation
 536   // If n is a known invariant, set up directly. Otherwise, look up the
 537   // the possibility to push n onto the stack for further processing.
 538   void visit(Node* use, Node* n) {
 539     if (_lpt->is_invariant(n)) { // known invariant
 540       _invariant.set(n->_idx);
 541     } else if (!n->is_CFG()) {
 542       if (n->Opcode() == Op_ShenandoahWriteBarrier) {
 543         return;
 544       }
 545       Node *n_ctrl = _phase->ctrl_or_self(n);
 546       Node *u_ctrl = _phase->ctrl_or_self(use); // self if use is a CFG
 547       if (_phase->is_dominator(n_ctrl, u_ctrl)) {
 548         _stack.push(n, n->in(0) == NULL ? 1 : 0);
 549       }
 550     }
 551   }
 552 
 553   // Compute invariance for "the_node" and (possibly) all its inputs recursively
 554   // on the fly
 555   void compute_invariance(Node* n) {
 556     assert(_visited.test(n->_idx), "must be");
 557     visit(n, n);
 558     while (_stack.is_nonempty()) {
 559       Node*  n = _stack.node();
 560       uint idx = _stack.index();
 561       if (idx == n->req()) { // all inputs are processed
 562         _stack.pop();
 563         // n is invariant if it's inputs are all invariant
 564         bool all_inputs_invariant = true;
 565         for (uint i = 0; i < n->req(); i++) {
 566           Node* in = n->in(i);
 567           if (in == NULL) continue;
 568           assert(_visited.test(in->_idx), "must have visited input");
 569           if (!_invariant.test(in->_idx)) { // bad guy
 570             all_inputs_invariant = false;
 571             break;
 572           }
 573         }
 574         if (all_inputs_invariant) {
 575           // If n's control is a predicate that was moved out of the
 576           // loop, it was marked invariant but n is only invariant if
 577           // it depends only on that test. Otherwise, unless that test
 578           // is out of the loop, it's not invariant.
 579           if (n->Opcode() == Op_ShenandoahWBMemProj || n->is_CFG() || n->depends_only_on_test() || n->in(0) == NULL || !_phase->is_member(_lpt, n->in(0))) {
 580             _invariant.set(n->_idx); // I am a invariant too
 581           }
 582         }
 583       } else { // process next input
 584         _stack.set_index(idx + 1);
 585         Node* m = n->in(idx);
 586         if (m != NULL && !_visited.test_set(m->_idx)) {
 587           visit(n, m);
 588         }
 589       }
 590     }
 591   }
 592 
 593   // Helper function to set up _old_new map for clone_nodes.
 594   // If n is a known invariant, set up directly ("clone" of n == n).
 595   // Otherwise, push n onto the stack for real cloning.
 596   void clone_visit(Node* n) {
 597     assert(_invariant.test(n->_idx), "must be invariant");
 598     if (_lpt->is_invariant(n)) { // known invariant
 599       _old_new.map(n->_idx, n);


< prev index next >