< prev index next > src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
Print this page
#include "opto/runtime.hpp"
#include "opto/subnode.hpp"
bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) {
ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
- if ((state->iu_barriers_count() +
- state->load_reference_barriers_count()) > 0) {
+ if (state->load_reference_barriers_count() > 0) {
assert(C->post_loop_opts_phase(), "no loop opts allowed");
C->reset_post_loop_opts_phase(); // ... but we know what we are doing
C->clear_major_progress();
PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand);
if (C->failing()) return false;
-
- C->set_major_progress();
- if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) {
- return false;
- }
- C->clear_major_progress();
C->process_for_post_loop_opts_igvn(igvn);
if (C->failing()) return false;
C->set_post_loop_opts_phase(); // now for real!
}
if (trace) {
tty->print("Found raw LoadP (OSR argument?)");
}
} else if (in->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
if (t == ShenandoahOopStore) {
- uint i = 0;
- for (; i < phis.size(); i++) {
- Node* n = phis.node_at(i);
- if (n->Opcode() == Op_ShenandoahIUBarrier) {
- break;
- }
- }
- if (i == phis.size()) {
- return false;
- }
+ return false;
}
barriers_used.push(in);
if (trace) {tty->print("Found barrier"); in->dump();}
- } else if (in->Opcode() == Op_ShenandoahIUBarrier) {
- if (t != ShenandoahOopStore) {
- in = in->in(1);
- continue;
- }
- if (trace) {tty->print("Found enqueue barrier"); in->dump();}
- phis.push(in, in->req());
- in = in->in(1);
- continue;
} else if (in->is_Proj() && in->in(0)->is_Allocate()) {
if (trace) {
tty->print("Found alloc");
in->in(0)->dump();
}
verify = false;
}
}
}
- if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
+ if (verify && !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: Store should have barriers", n);
}
}
if (!verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: Store (address) should have barriers", n);
visited.reset();
}
}
} else if (n->is_LoadStore()) {
if (n->in(MemNode::ValueIn)->bottom_type()->make_ptr() &&
- !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahIUBarrier ? ShenandoahOopStore : ShenandoahValue, trace, barriers_used)) {
+ !verify_helper(n->in(MemNode::ValueIn), phis, visited, ShenandoahValue, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: LoadStore (value) should have barriers", n);
}
if (n->in(MemNode::Address)->bottom_type()->make_oopptr() && !verify_helper(n->in(MemNode::Address), phis, visited, ShenandoahStore, trace, barriers_used)) {
report_verify_failure("Shenandoah verification: LoadStore (address) should have barriers", n);
struct {
int pos;
verify_type t;
} args[6];
} calls[] = {
+ "array_partition_stub",
+ { { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+4, ShenandoahStore }, { -1, ShenandoahNone },
+ { -1, ShenandoahNone }, { -1, ShenandoahNone }, { -1, ShenandoahNone } },
+ "arraysort_stub",
+ { { TypeFunc::Parms, ShenandoahStore }, { -1, ShenandoahNone }, { -1, ShenandoahNone },
+ { -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
"aescrypt_encryptBlock",
{ { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
{ -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
"aescrypt_decryptBlock",
{ { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
{ { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
{ TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
"cipherBlockChaining_decryptAESCrypt",
{ { TypeFunc::Parms, ShenandoahLoad }, { TypeFunc::Parms+1, ShenandoahStore }, { TypeFunc::Parms+2, ShenandoahLoad },
{ TypeFunc::Parms+3, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
- "shenandoah_clone_barrier",
+ "shenandoah_clone",
{ { TypeFunc::Parms, ShenandoahLoad }, { -1, ShenandoahNone}, { -1, ShenandoahNone},
{ -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
"ghash_processBlocks",
{ { TypeFunc::Parms, ShenandoahStore }, { TypeFunc::Parms+1, ShenandoahLoad }, { TypeFunc::Parms+2, ShenandoahLoad },
{ -1, ShenandoahNone}, { -1, ShenandoahNone}, { -1, ShenandoahNone} },
fatal("%s not covered", call->_name);
}
}
}
}
- } else if (n->Opcode() == Op_ShenandoahIUBarrier || n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
+ } else if (n->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
// skip
} else if (n->is_AddP()
|| n->is_Phi()
|| n->is_ConstraintCast()
|| n->Opcode() == Op_Return
void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
ShenandoahBarrierSetC2State* state = ShenandoahBarrierSetC2::bsc2()->state();
Unique_Node_List uses;
- for (int i = 0; i < state->iu_barriers_count(); i++) {
- Node* barrier = state->iu_barrier(i);
- Node* ctrl = phase->get_ctrl(barrier);
- IdealLoopTree* loop = phase->get_loop(ctrl);
- Node* head = loop->head();
- if (head->is_OuterStripMinedLoop()) {
- // Expanding a barrier here will break loop strip mining
- // verification. Transform the loop so the loop nest doesn't
- // appear as strip mined.
- OuterStripMinedLoopNode* outer = head->as_OuterStripMinedLoop();
- hide_strip_mined_loop(outer, outer->unique_ctrl_out()->as_CountedLoop(), phase);
- }
- }
-
Node_Stack stack(0);
Node_List clones;
for (int i = state->load_reference_barriers_count() - 1; i >= 0; i--) {
ShenandoahLoadReferenceBarrierNode* lrb = state->load_reference_barrier(i);
}
fixer.record_new_ctrl(ctrl, region, raw_mem, raw_mem_for_ctrl);
}
// Done expanding load-reference-barriers.
assert(ShenandoahBarrierSetC2::bsc2()->state()->load_reference_barriers_count() == 0, "all load reference barrier nodes should have been replaced");
-
- for (int i = state->iu_barriers_count() - 1; i >= 0; i--) {
- Node* barrier = state->iu_barrier(i);
- Node* pre_val = barrier->in(1);
-
- if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) {
- ShouldNotReachHere();
- continue;
- }
-
- Node* ctrl = phase->get_ctrl(barrier);
-
- if (ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) {
- assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0)->in(0), pre_val, ctrl->in(0), phase), "can't move");
- ctrl = ctrl->in(0)->in(0);
- phase->set_ctrl(barrier, ctrl);
- } else if (ctrl->is_CallRuntime()) {
- assert(is_dominator(phase->get_ctrl(pre_val), ctrl->in(0), pre_val, ctrl, phase), "can't move");
- ctrl = ctrl->in(0);
- phase->set_ctrl(barrier, ctrl);
- }
-
- Node* init_ctrl = ctrl;
- IdealLoopTree* loop = phase->get_loop(ctrl);
- Node* raw_mem = fixer.find_mem(ctrl, barrier);
- Node* init_raw_mem = raw_mem;
- Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, nullptr);
- Node* heap_stable_ctrl = nullptr;
- Node* null_ctrl = nullptr;
- uint last = phase->C->unique();
-
- enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
- Node* region = new RegionNode(PATH_LIMIT);
- Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
-
- enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 };
- Node* region2 = new RegionNode(PATH_LIMIT2);
- Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
-
- // Stable path.
- test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING);
- region->init_req(_heap_stable, heap_stable_ctrl);
- phi->init_req(_heap_stable, raw_mem);
-
- // Null path
- Node* reg2_ctrl = nullptr;
- test_null(ctrl, pre_val, null_ctrl, phase);
- if (null_ctrl != nullptr) {
- reg2_ctrl = null_ctrl->in(0);
- region2->init_req(_null_path, null_ctrl);
- phi2->init_req(_null_path, raw_mem);
- } else {
- region2->del_req(_null_path);
- phi2->del_req(_null_path);
- }
-
- const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
- const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
- Node* thread = new ThreadLocalNode();
- phase->register_new_node(thread, ctrl);
- Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset));
- phase->register_new_node(buffer_adr, ctrl);
- Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset));
- phase->register_new_node(index_adr, ctrl);
-
- BasicType index_bt = TypeX_X->basic_type();
- assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading Shenandoah SATBMarkQueue::_index with wrong size.");
- const TypePtr* adr_type = TypeRawPtr::BOTTOM;
- Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered);
- phase->register_new_node(index, ctrl);
- Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0));
- phase->register_new_node(index_cmp, ctrl);
- Node* index_test = new BoolNode(index_cmp, BoolTest::ne);
- phase->register_new_node(index_test, ctrl);
- IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN);
- if (reg2_ctrl == nullptr) reg2_ctrl = queue_full_iff;
- phase->register_control(queue_full_iff, loop, ctrl);
- Node* not_full = new IfTrueNode(queue_full_iff);
- phase->register_control(not_full, loop, queue_full_iff);
- Node* full = new IfFalseNode(queue_full_iff);
- phase->register_control(full, loop, queue_full_iff);
-
- ctrl = not_full;
-
- Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t)));
- phase->register_new_node(next_index, ctrl);
-
- Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered);
- phase->register_new_node(buffer, ctrl);
- Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index);
- phase->register_new_node(log_addr, ctrl);
- Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, pre_val, MemNode::unordered);
- phase->register_new_node(log_store, ctrl);
- // update the index
- Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered);
- phase->register_new_node(index_update, ctrl);
-
- // Fast-path case
- region2->init_req(_fast_path, ctrl);
- phi2->init_req(_fast_path, index_update);
-
- ctrl = full;
-
- Node* base = find_bottom_mem(ctrl, phase);
-
- MergeMemNode* mm = MergeMemNode::make(base);
- mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
- phase->register_new_node(mm, ctrl);
-
- Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM);
- call->init_req(TypeFunc::Control, ctrl);
- call->init_req(TypeFunc::I_O, phase->C->top());
- call->init_req(TypeFunc::Memory, mm);
- call->init_req(TypeFunc::FramePtr, phase->C->top());
- call->init_req(TypeFunc::ReturnAdr, phase->C->top());
- call->init_req(TypeFunc::Parms, pre_val);
- call->init_req(TypeFunc::Parms+1, thread);
- phase->register_control(call, loop, ctrl);
-
- Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
- phase->register_control(ctrl_proj, loop, call);
- Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
- phase->register_new_node(mem_proj, call);
-
- // Slow-path case
- region2->init_req(_slow_path, ctrl_proj);
- phi2->init_req(_slow_path, mem_proj);
-
- phase->register_control(region2, loop, reg2_ctrl);
- phase->register_new_node(phi2, region2);
-
- region->init_req(_heap_unstable, region2);
- phi->init_req(_heap_unstable, phi2);
-
- phase->register_control(region, loop, heap_stable_ctrl->in(0));
- phase->register_new_node(phi, region);
-
- fix_ctrl(barrier, region, fixer, uses, uses_to_ignore, last, phase);
- for(uint next = 0; next < uses.size(); next++ ) {
- Node *n = uses.at(next);
- assert(phase->get_ctrl(n) == init_ctrl, "bad control");
- assert(n != init_raw_mem, "should leave input raw mem above the barrier");
- phase->set_ctrl(n, region);
- follow_barrier_uses(n, init_ctrl, uses, phase);
- }
- fixer.fix_mem(init_ctrl, region, init_raw_mem, raw_mem_for_ctrl, phi, uses);
-
- phase->igvn().replace_node(barrier, pre_val);
- }
- assert(state->iu_barriers_count() == 0, "all enqueue barrier nodes should have been replaced");
-
}
Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) {
if (visited.test_set(in->_idx)) {
return nullptr;
}
return addr;
}
case Op_ShenandoahLoadReferenceBarrier:
return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn));
- case Op_ShenandoahIUBarrier:
- return get_load_addr(phase, visited, in->in(1));
case Op_CallDynamicJava:
case Op_CallLeaf:
case Op_CallStaticJava:
case Op_ConN:
case Op_ConP:
return phase->igvn().zerocon(T_OBJECT);
}
}
- void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) {
- IdealLoopTree *loop = phase->get_loop(iff);
- Node* loop_head = loop->_head;
- Node* entry_c = loop_head->in(LoopNode::EntryControl);
-
- Node* bol = iff->in(1);
- Node* cmp = bol->in(1);
- Node* andi = cmp->in(1);
- Node* load = andi->in(1);
-
- assert(is_gc_state_load(load), "broken");
- if (!phase->is_dominator(load->in(0), entry_c)) {
- Node* mem_ctrl = nullptr;
- Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase);
- load = load->clone();
- load->set_req(MemNode::Memory, mem);
- load->set_req(0, entry_c);
- phase->register_new_node(load, entry_c);
- andi = andi->clone();
- andi->set_req(1, load);
- phase->register_new_node(andi, entry_c);
- cmp = cmp->clone();
- cmp->set_req(1, andi);
- phase->register_new_node(cmp, entry_c);
- bol = bol->clone();
- bol->set_req(1, cmp);
- phase->register_new_node(bol, entry_c);
-
- phase->igvn().replace_input_of(iff, 1, bol);
- }
- }
-
- bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) {
- if (!n->is_If() || n->is_CountedLoopEnd()) {
- return false;
- }
- Node* region = n->in(0);
-
- if (!region->is_Region()) {
- return false;
- }
- Node* dom = phase->idom(region);
- if (!dom->is_If()) {
- return false;
- }
-
- if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) {
- return false;
- }
-
- IfNode* dom_if = dom->as_If();
- Node* proj_true = dom_if->proj_out(1);
- Node* proj_false = dom_if->proj_out(0);
-
- for (uint i = 1; i < region->req(); i++) {
- if (phase->is_dominator(proj_true, region->in(i))) {
- continue;
- }
- if (phase->is_dominator(proj_false, region->in(i))) {
- continue;
- }
- return false;
- }
-
- return true;
- }
-
- bool ShenandoahBarrierC2Support::merge_point_safe(Node* region) {
- for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
- Node* n = region->fast_out(i);
- if (n->is_LoadStore()) {
- // Splitting a LoadStore node through phi, causes it to lose its SCMemProj: the split if code doesn't have support
- // for a LoadStore at the region the if is split through because that's not expected to happen (LoadStore nodes
- // should be between barrier nodes). It does however happen with Shenandoah though because barriers can get
- // expanded around a LoadStore node.
- return false;
- }
- }
- return true;
- }
-
-
- void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
- assert(is_heap_stable_test(n), "no other tests");
- if (identical_backtoback_ifs(n, phase)) {
- Node* n_ctrl = n->in(0);
- if (phase->can_split_if(n_ctrl) && merge_point_safe(n_ctrl)) {
- IfNode* dom_if = phase->idom(n_ctrl)->as_If();
- if (is_heap_stable_test(n)) {
- Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1);
- assert(is_gc_state_load(gc_state_load), "broken");
- Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1);
- assert(is_gc_state_load(dom_gc_state_load), "broken");
- if (gc_state_load != dom_gc_state_load) {
- phase->igvn().replace_node(gc_state_load, dom_gc_state_load);
- }
- }
- PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1));
- Node* proj_true = dom_if->proj_out(1);
- Node* proj_false = dom_if->proj_out(0);
- Node* con_true = phase->igvn().makecon(TypeInt::ONE);
- Node* con_false = phase->igvn().makecon(TypeInt::ZERO);
-
- for (uint i = 1; i < n_ctrl->req(); i++) {
- if (phase->is_dominator(proj_true, n_ctrl->in(i))) {
- bolphi->init_req(i, con_true);
- } else {
- assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if");
- bolphi->init_req(i, con_false);
- }
- }
- phase->register_new_node(bolphi, n_ctrl);
- phase->igvn().replace_input_of(n, 1, bolphi);
- phase->do_split_if(n);
- }
- }
- }
-
- IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) {
- // Find first invariant test that doesn't exit the loop
- LoopNode *head = loop->_head->as_Loop();
- IfNode* unswitch_iff = nullptr;
- Node* n = head->in(LoopNode::LoopBackControl);
- int loop_has_sfpts = -1;
- while (n != head) {
- Node* n_dom = phase->idom(n);
- if (n->is_Region()) {
- if (n_dom->is_If()) {
- IfNode* iff = n_dom->as_If();
- if (iff->in(1)->is_Bool()) {
- BoolNode* bol = iff->in(1)->as_Bool();
- if (bol->in(1)->is_Cmp()) {
- // If condition is invariant and not a loop exit,
- // then found reason to unswitch.
- if (is_heap_stable_test(iff) &&
- (loop_has_sfpts == -1 || loop_has_sfpts == 0)) {
- assert(!loop->is_loop_exit(iff), "both branches should be in the loop");
- if (loop_has_sfpts == -1) {
- for(uint i = 0; i < loop->_body.size(); i++) {
- Node *m = loop->_body[i];
- if (m->is_SafePoint() && !m->is_CallLeaf()) {
- loop_has_sfpts = 1;
- break;
- }
- }
- if (loop_has_sfpts == -1) {
- loop_has_sfpts = 0;
- }
- }
- if (!loop_has_sfpts) {
- unswitch_iff = iff;
- }
- }
- }
- }
- }
- }
- n = n_dom;
- }
- return unswitch_iff;
- }
-
-
- void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) {
- Node_List heap_stable_tests;
- stack.push(phase->C->start(), 0);
- do {
- Node* n = stack.node();
- uint i = stack.index();
-
- if (i < n->outcnt()) {
- Node* u = n->raw_out(i);
- stack.set_index(i+1);
- if (!visited.test_set(u->_idx)) {
- stack.push(u, 0);
- }
- } else {
- stack.pop();
- if (n->is_If() && is_heap_stable_test(n)) {
- heap_stable_tests.push(n);
- }
- }
- } while (stack.size() > 0);
-
- for (uint i = 0; i < heap_stable_tests.size(); i++) {
- Node* n = heap_stable_tests.at(i);
- assert(is_heap_stable_test(n), "only evacuation test");
- merge_back_to_back_tests(n, phase);
- }
-
- if (!phase->C->major_progress()) {
- VectorSet seen;
- for (uint i = 0; i < heap_stable_tests.size(); i++) {
- Node* n = heap_stable_tests.at(i);
- IdealLoopTree* loop = phase->get_loop(n);
- if (loop != phase->ltree_root() &&
- loop->_child == nullptr &&
- !loop->_irreducible) {
- Node* head = loop->_head;
- if (head->is_Loop() &&
- (!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) &&
- !seen.test_set(head->_idx)) {
- IfNode* iff = find_unswitching_candidate(loop, phase);
- if (iff != nullptr) {
- Node* bol = iff->in(1);
- if (head->as_Loop()->is_strip_mined()) {
- head->as_Loop()->verify_strip_mined(0);
- }
- move_gc_state_test_out_of_loop(iff, phase);
-
- AutoNodeBudget node_budget(phase);
-
- if (loop->policy_unswitching(phase)) {
- if (head->as_Loop()->is_strip_mined()) {
- OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop();
- hide_strip_mined_loop(outer, head->as_CountedLoop(), phase);
- }
- phase->do_unswitching(loop, old_new);
- } else {
- // Not proceeding with unswitching. Move load back in
- // the loop.
- phase->igvn().replace_input_of(iff, 1, bol);
- }
- }
- }
- }
- }
- }
- }
-
- ShenandoahIUBarrierNode::ShenandoahIUBarrierNode(Node* val) : Node(nullptr, val) {
- ShenandoahBarrierSetC2::bsc2()->state()->add_iu_barrier(this);
- }
-
- const Type* ShenandoahIUBarrierNode::bottom_type() const {
- if (in(1) == nullptr || in(1)->is_top()) {
- return Type::TOP;
- }
- const Type* t = in(1)->bottom_type();
- if (t == TypePtr::NULL_PTR) {
- return t;
- }
- return t->is_oopptr();
- }
-
- const Type* ShenandoahIUBarrierNode::Value(PhaseGVN* phase) const {
- if (in(1) == nullptr) {
- return Type::TOP;
- }
- const Type* t = phase->type(in(1));
- if (t == Type::TOP) {
- return Type::TOP;
- }
- if (t == TypePtr::NULL_PTR) {
- return t;
- }
- return t->is_oopptr();
- }
-
- int ShenandoahIUBarrierNode::needed(Node* n) {
- if (n == nullptr ||
- n->is_Allocate() ||
- n->Opcode() == Op_ShenandoahIUBarrier ||
- n->bottom_type() == TypePtr::NULL_PTR ||
- (n->bottom_type()->make_oopptr() != nullptr && n->bottom_type()->make_oopptr()->const_oop() != nullptr)) {
- return NotNeeded;
- }
- if (n->is_Phi() ||
- n->is_CMove()) {
- return MaybeNeeded;
- }
- return Needed;
- }
-
- Node* ShenandoahIUBarrierNode::next(Node* n) {
- for (;;) {
- if (n == nullptr) {
- return n;
- } else if (n->bottom_type() == TypePtr::NULL_PTR) {
- return n;
- } else if (n->bottom_type()->make_oopptr() != nullptr && n->bottom_type()->make_oopptr()->const_oop() != nullptr) {
- return n;
- } else if (n->is_ConstraintCast() ||
- n->Opcode() == Op_DecodeN ||
- n->Opcode() == Op_EncodeP) {
- n = n->in(1);
- } else if (n->is_Proj()) {
- n = n->in(0);
- } else {
- return n;
- }
- }
- ShouldNotReachHere();
- return nullptr;
- }
-
- Node* ShenandoahIUBarrierNode::Identity(PhaseGVN* phase) {
- PhaseIterGVN* igvn = phase->is_IterGVN();
-
- Node* n = next(in(1));
-
- int cont = needed(n);
-
- if (cont == NotNeeded) {
- return in(1);
- } else if (cont == MaybeNeeded) {
- if (igvn == nullptr) {
- phase->record_for_igvn(this);
- return this;
- } else {
- ResourceMark rm;
- Unique_Node_List wq;
- uint wq_i = 0;
-
- for (;;) {
- if (n->is_Phi()) {
- for (uint i = 1; i < n->req(); i++) {
- Node* m = n->in(i);
- if (m != nullptr) {
- wq.push(m);
- }
- }
- } else {
- assert(n->is_CMove(), "nothing else here");
- Node* m = n->in(CMoveNode::IfFalse);
- wq.push(m);
- m = n->in(CMoveNode::IfTrue);
- wq.push(m);
- }
- Node* orig_n = nullptr;
- do {
- if (wq_i >= wq.size()) {
- return in(1);
- }
- n = wq.at(wq_i);
- wq_i++;
- orig_n = n;
- n = next(n);
- cont = needed(n);
- if (cont == Needed) {
- return this;
- }
- } while (cont != MaybeNeeded || (orig_n != n && wq.member(n)));
- }
- }
- }
-
- return this;
- }
-
#ifdef ASSERT
static bool has_never_branch(Node* root) {
for (uint i = 1; i < root->req(); i++) {
Node* in = root->in(i);
if (in != nullptr && in->Opcode() == Op_Halt && in->in(0)->is_Proj() && in->in(0)->in(0)->is_NeverBranch()) {
for (uint j = 1; j < c->req() && unique != NodeSentinel; j++) {
Node* m = _memory_nodes[c->in(j)->_idx];
assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
if (m != nullptr) {
if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
- assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
+ assert((c->is_Loop() && j == LoopNode::LoopBackControl) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
// continue
} else if (unique == nullptr) {
unique = m;
} else if (m == unique) {
// continue
return true;
case Op_CMoveN:
case Op_CMoveP:
return needs_barrier_impl(phase, n->in(2), visited) ||
needs_barrier_impl(phase, n->in(3), visited);
- case Op_ShenandoahIUBarrier:
- return needs_barrier_impl(phase, n->in(1), visited);
case Op_CreateEx:
return false;
default:
break;
}
< prev index next >