< prev index next >

src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp

Print this page

        

*** 20,45 **** * or visit www.oracle.com if you need additional information or have any * questions. */ #include "precompiled.hpp" - #include "opto/castnode.hpp" #include "opto/compile.hpp" #include "opto/escape.hpp" #include "opto/graphKit.hpp" #include "opto/loopnode.hpp" - #include "opto/machnode.hpp" #include "opto/macro.hpp" - #include "opto/memnode.hpp" - #include "opto/movenode.hpp" #include "opto/node.hpp" - #include "opto/phase.hpp" - #include "opto/phaseX.hpp" - #include "opto/rootnode.hpp" #include "opto/type.hpp" - #include "utilities/copy.hpp" - #include "utilities/growableArray.hpp" #include "utilities/macros.hpp" #include "gc/z/zBarrierSet.hpp" #include "gc/z/c2/zBarrierSetC2.hpp" #include "gc/z/zThreadLocalData.hpp" #include "gc/z/zBarrierSetRuntime.hpp" --- 20,38 ---- * or visit www.oracle.com if you need additional information or have any * questions. */ #include "precompiled.hpp" #include "opto/compile.hpp" + #include "opto/castnode.hpp" #include "opto/escape.hpp" #include "opto/graphKit.hpp" + #include "opto/idealKit.hpp" #include "opto/loopnode.hpp" #include "opto/macro.hpp" #include "opto/node.hpp" #include "opto/type.hpp" #include "utilities/macros.hpp" #include "gc/z/zBarrierSet.hpp" #include "gc/z/c2/zBarrierSetC2.hpp" #include "gc/z/zThreadLocalData.hpp" #include "gc/z/zBarrierSetRuntime.hpp"
*** 89,99 **** // 3. This step checks for the phi corresponding to an optimized load barrier expansion if (node->is_Phi()) { PhiNode* phi = node->as_Phi(); Node* n = phi->in(1); ! if (n != NULL && n->is_LoadBarrierSlowReg()) { return true; } } return false; --- 82,92 ---- // 3. This step checks for the phi corresponding to an optimized load barrier expansion if (node->is_Phi()) { PhiNode* phi = node->as_Phi(); Node* n = phi->in(1); ! if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) { return true; } } return false;
*** 126,158 **** if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) { igvn->_worklist.push(node); } } ! static bool load_require_barrier(LoadNode* load) { return ((load->barrier_data() & RequireBarrier) != 0); } ! static bool load_has_weak_barrier(LoadNode* load) { return ((load->barrier_data() & WeakBarrier) != 0); } ! static bool load_has_expanded_barrier(LoadNode* load) { return ((load->barrier_data() & ExpandedBarrier) != 0); } ! static void load_set_expanded_barrier(LoadNode* load) { return load->set_barrier_data(ExpandedBarrier); } ! ! static void load_set_barrier(LoadNode* load, bool weak) { ! if (weak) { ! load->set_barrier_data(WeakBarrier); ! } else { ! load->set_barrier_data(RequireBarrier); } } // == LoadBarrierNode == LoadBarrierNode::LoadBarrierNode(Compile* C, Node* c, Node* mem, Node* val, Node* adr, ! bool weak) : MultiNode(Number_of_Inputs), ! _weak(weak) { init_req(Control, c); init_req(Memory, mem); init_req(Oop, val); init_req(Address, adr); init_req(Similar, C->top()); --- 119,186 ---- if (node->is_LoadBarrier() && !node->as_LoadBarrier()->has_true_uses()) { igvn->_worklist.push(node); } } ! void ZBarrierSetC2::find_dominating_barriers(PhaseIterGVN& igvn) { ! // Look for dominating barriers on the same address only once all ! // other loop opts are over. Loop opts may cause a safepoint to be ! // inserted between a barrier and its dominating barrier. ! Compile* C = Compile::current(); ! ZBarrierSetC2* bs = (ZBarrierSetC2*)BarrierSet::barrier_set()->barrier_set_c2(); ! ZBarrierSetC2State* s = bs->state(); ! if (s->load_barrier_count() >= 2) { ! Compile::TracePhase tp("idealLoop", &C->timers[Phase::_t_idealLoop]); ! PhaseIdealLoop::optimize(igvn, LoopOptsLastRound); ! if (C->major_progress()) C->print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2); } } + void ZBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const { + // Permanent temporary workaround + // Loadbarriers may have non-obvious dead uses keeping them alive during parsing. The use is + // removed by RemoveUseless (after parsing, before optimize) but the barriers won't be added to + // the worklist. Unless we add them explicitly they are not guaranteed to end up there. + ZBarrierSetC2State* s = state(); + + for (int i = 0; i < s->load_barrier_count(); i++) { + LoadBarrierNode* n = s->load_barrier_node(i); + worklist->push(n); + } + } + + const TypeFunc* ZBarrierSetC2::load_barrier_Type() const { + const Type** fields; + + // Create input types (domain) + fields = TypeTuple::fields(2); + fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; + fields[TypeFunc::Parms+1] = TypeOopPtr::BOTTOM; + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields); + + // Create result type (range) + fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeInstPtr::BOTTOM; + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); + + return TypeFunc::make(domain, range); + } + // == LoadBarrierNode == LoadBarrierNode::LoadBarrierNode(Compile* C, Node* c, Node* mem, Node* val, Node* adr, ! bool weak, ! bool writeback, ! bool oop_reload_allowed) : MultiNode(Number_of_Inputs), ! _weak(weak), ! _writeback(writeback), ! _oop_reload_allowed(oop_reload_allowed) { init_req(Control, c); init_req(Memory, mem); init_req(Oop, val); init_req(Address, adr); init_req(Similar, C->top());
*** 187,198 **** const Type *LoadBarrierNode::Value(PhaseGVN *phase) const { const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); const Type* val_t = phase->type(in(Oop)); floadbarrier[Control] = Type::CONTROL; ! floadbarrier[Memory] = Type::MEMORY; ! floadbarrier[Oop] = val_t; return TypeTuple::make(Number_of_Outputs, floadbarrier); } bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) { if (phase != NULL) { --- 215,226 ---- const Type *LoadBarrierNode::Value(PhaseGVN *phase) const { const Type** floadbarrier = (const Type **)(phase->C->type_arena()->Amalloc_4((Number_of_Outputs)*sizeof(Type*))); const Type* val_t = phase->type(in(Oop)); floadbarrier[Control] = Type::CONTROL; ! floadbarrier[Memory] = Type::MEMORY; ! floadbarrier[Oop] = val_t; return TypeTuple::make(Number_of_Outputs, floadbarrier); } bool LoadBarrierNode::is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n) { if (phase != NULL) {
*** 208,222 **** return false; } LoadBarrierNode* LoadBarrierNode::has_dominating_barrier(PhaseIdealLoop* phase, bool linear_only, bool look_for_similar) { - if (is_weak()) { - // Weak barriers can't be eliminated - return NULL; - } - Node* val = in(LoadBarrierNode::Oop); if (in(Similar)->is_Proj() && in(Similar)->in(0)->is_LoadBarrier()) { LoadBarrierNode* lb = in(Similar)->in(0)->as_LoadBarrier(); assert(lb->in(Address) == in(Address), ""); // Load barrier on Similar edge dominates so if it now has the Oop field it can replace this barrier. --- 236,245 ----
*** 241,251 **** return u->as_LoadBarrier(); } } } ! if (can_be_eliminated()) { return NULL; } if (!look_for_similar) { return NULL; --- 264,274 ---- return u->as_LoadBarrier(); } } } ! if (ZVerifyLoadBarriers || can_be_eliminated()) { return NULL; } if (!look_for_similar) { return NULL;
*** 291,301 **** } } } if (ok) { assert(dom_found, ""); ! return u->as_LoadBarrier(); } break; } } } --- 314,324 ---- } } } if (ok) { assert(dom_found, ""); ! return u->as_LoadBarrier();; } break; } } }
*** 303,313 **** return NULL; } void LoadBarrierNode::push_dominated_barriers(PhaseIterGVN* igvn) const { // Change to that barrier may affect a dominated barrier so re-push those - assert(!is_weak(), "sanity"); Node* val = in(LoadBarrierNode::Oop); for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { Node* u = val->fast_out(i); if (u != this && u->is_LoadBarrier() && u->in(Oop) == val) { --- 326,335 ----
*** 331,343 **** } } } Node *LoadBarrierNode::Identity(PhaseGVN *phase) { LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false); if (dominating_barrier != NULL) { - assert(!is_weak(), "Weak barriers cant be eliminated"); assert(dominating_barrier->in(Oop) == in(Oop), ""); return dominating_barrier; } return this; --- 353,369 ---- } } } Node *LoadBarrierNode::Identity(PhaseGVN *phase) { + if (!phase->C->directive()->ZOptimizeLoadBarriersOption) { + return this; + } + + bool redundant_addr = false; LoadBarrierNode* dominating_barrier = has_dominating_barrier(NULL, true, false); if (dominating_barrier != NULL) { assert(dominating_barrier->in(Oop) == in(Oop), ""); return dominating_barrier; } return this;
*** 346,382 **** Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) { return this; } ! Node *val = in(Oop); ! Node *mem = in(Memory); ! Node *ctrl = in(Control); ! assert(val->Opcode() != Op_LoadN, ""); - assert(val->Opcode() != Op_DecodeN, ""); if (mem->is_MergeMem()) { ! Node *new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); set_req(Memory, new_mem); if (mem->outcnt() == 0 && can_reshape) { phase->is_IterGVN()->_worklist.push(mem); } return this; } ! LoadBarrierNode *dominating_barrier = NULL; ! if (!is_weak()) { ! dominating_barrier = has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()); ! if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) { ! assert(in(Address) == dominating_barrier->in(Address), ""); ! set_req(Similar, dominating_barrier->proj_out(Oop)); ! return this; ! } } ! bool eliminate = can_reshape && (dominating_barrier != NULL || !has_true_uses()); if (eliminate) { if (can_reshape) { PhaseIterGVN* igvn = phase->is_IterGVN(); Node* out_ctrl = proj_out_or_null(Control); Node* out_res = proj_out_or_null(Oop); --- 372,408 ---- Node *LoadBarrierNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) { return this; } ! Node* val = in(Oop); ! Node* mem = in(Memory); ! Node* ctrl = in(Control); ! Node* adr = in(Address); assert(val->Opcode() != Op_LoadN, ""); if (mem->is_MergeMem()) { ! Node* new_mem = mem->as_MergeMem()->memory_at(Compile::AliasIdxRaw); set_req(Memory, new_mem); if (mem->outcnt() == 0 && can_reshape) { phase->is_IterGVN()->_worklist.push(mem); } + return this; } ! bool optimizeLoadBarriers = phase->C->directive()->ZOptimizeLoadBarriersOption; ! LoadBarrierNode* dominating_barrier = optimizeLoadBarriers ? has_dominating_barrier(NULL, !can_reshape, !phase->C->major_progress()) : NULL; ! if (dominating_barrier != NULL && dominating_barrier->in(Oop) != in(Oop)) { ! assert(in(Address) == dominating_barrier->in(Address), ""); ! set_req(Similar, dominating_barrier->proj_out(Oop)); ! return this; } ! bool eliminate = (optimizeLoadBarriers && !(val->is_Phi() || val->Opcode() == Op_LoadP || val->Opcode() == Op_GetAndSetP || val->is_DecodeN())) || ! (can_reshape && (dominating_barrier != NULL || !has_true_uses())); ! if (eliminate) { if (can_reshape) { PhaseIterGVN* igvn = phase->is_IterGVN(); Node* out_ctrl = proj_out_or_null(Control); Node* out_res = proj_out_or_null(Oop);
*** 387,414 **** // That transformation may cause the Similar edge on the load barrier to be invalid fix_similar_in_uses(igvn); if (out_res != NULL) { if (dominating_barrier != NULL) { - assert(!is_weak(), "Sanity"); igvn->replace_node(out_res, dominating_barrier->proj_out(Oop)); } else { igvn->replace_node(out_res, val); } } } return new ConINode(TypeInt::ZERO); } // If the Similar edge is no longer a load barrier, clear it Node* similar = in(Similar); if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) { set_req(Similar, phase->C->top()); return this; } ! if (can_reshape && !is_weak()) { // If this barrier is linked through the Similar edge by a // dominated barrier and both barriers have the same Oop field, // the dominated barrier can go away, so push it for reprocessing. // We also want to avoid a barrier to depend on another dominating // barrier through its Similar edge that itself depend on another --- 413,440 ---- // That transformation may cause the Similar edge on the load barrier to be invalid fix_similar_in_uses(igvn); if (out_res != NULL) { if (dominating_barrier != NULL) { igvn->replace_node(out_res, dominating_barrier->proj_out(Oop)); } else { igvn->replace_node(out_res, val); } } } + return new ConINode(TypeInt::ZERO); } // If the Similar edge is no longer a load barrier, clear it Node* similar = in(Similar); if (!similar->is_top() && !(similar->is_Proj() && similar->in(0)->is_LoadBarrier())) { set_req(Similar, phase->C->top()); return this; } ! if (can_reshape) { // If this barrier is linked through the Similar edge by a // dominated barrier and both barriers have the same Oop field, // the dominated barrier can go away, so push it for reprocessing. // We also want to avoid a barrier to depend on another dominating // barrier through its Similar edge that itself depend on another
*** 418,431 **** Node* out_res = proj_out(Oop); for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { Node* u = out_res->fast_out(i); if (u->is_LoadBarrier() && u->in(Similar) == out_res && (u->in(Oop) == val || !u->in(Similar)->is_top())) { - assert(!u->as_LoadBarrier()->is_weak(), "Sanity"); igvn->_worklist.push(u); } } push_dominated_barriers(igvn); } return NULL; } --- 444,457 ---- Node* out_res = proj_out(Oop); for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { Node* u = out_res->fast_out(i); if (u->is_LoadBarrier() && u->in(Similar) == out_res && (u->in(Oop) == val || !u->in(Similar)->is_top())) { igvn->_worklist.push(u); } } + push_dominated_barriers(igvn); } return NULL; }
*** 451,471 **** } } bool LoadBarrierNode::has_true_uses() const { Node* out_res = proj_out_or_null(Oop); ! if (out_res != NULL) { ! for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { ! Node *u = out_res->fast_out(i); ! if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { ! return true; ! } } } return false; } static bool barrier_needed(C2Access& access) { return ZBarrierSet::barrier_needed(access.decorators(), access.type()); } Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { --- 477,691 ---- } } bool LoadBarrierNode::has_true_uses() const { Node* out_res = proj_out_or_null(Oop); ! if (out_res == NULL) { ! return false; ! } ! ! for (DUIterator_Fast imax, i = out_res->fast_outs(imax); i < imax; i++) { ! Node* u = out_res->fast_out(i); ! if (!u->is_LoadBarrier() || u->in(Similar) != out_res) { ! return true; } } + return false; } + // == Accesses == + + Node* ZBarrierSetC2::make_cas_loadbarrier(C2AtomicParseAccess& access) const { + assert(!UseCompressedOops, "Not allowed"); + CompareAndSwapNode* cas = (CompareAndSwapNode*)access.raw_access(); + PhaseGVN& gvn = access.gvn(); + Compile* C = Compile::current(); + GraphKit* kit = access.kit(); + + Node* in_ctrl = cas->in(MemNode::Control); + Node* in_mem = cas->in(MemNode::Memory); + Node* in_adr = cas->in(MemNode::Address); + Node* in_val = cas->in(MemNode::ValueIn); + Node* in_expected = cas->in(LoadStoreConditionalNode::ExpectedIn); + + float likely = PROB_LIKELY(0.999); + + const TypePtr *adr_type = gvn.type(in_adr)->isa_ptr(); + Compile::AliasType* alias_type = C->alias_type(adr_type); + int alias_idx = C->get_alias_index(adr_type); + + // Outer check - true: continue, false: load and check + Node* region = new RegionNode(3); + Node* phi = new PhiNode(region, TypeInt::BOOL); + Node* phi_mem = new PhiNode(region, Type::MEMORY, adr_type); + + // Inner check - is the healed ref equal to the expected + Node* region2 = new RegionNode(3); + Node* phi2 = new PhiNode(region2, TypeInt::BOOL); + Node* phi_mem2 = new PhiNode(region2, Type::MEMORY, adr_type); + + // CAS node returns 0 or 1 + Node* cmp = gvn.transform(new CmpINode(cas, kit->intcon(0))); + Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); + IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If(); + Node* then = gvn.transform(new IfTrueNode(iff)); + Node* elsen = gvn.transform(new IfFalseNode(iff)); + + Node* scmemproj1 = gvn.transform(new SCMemProjNode(cas)); + + kit->set_memory(scmemproj1, alias_idx); + phi_mem->init_req(1, scmemproj1); + phi_mem2->init_req(2, scmemproj1); + + // CAS fail - reload and heal oop + Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); + Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false)); + Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)); + Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop)); + + // Check load + Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata)); + Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected)); + Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX)); + Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool(); + IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If(); + Node* then2 = gvn.transform(new IfTrueNode(iff2)); + Node* elsen2 = gvn.transform(new IfFalseNode(iff2)); + + // redo CAS + Node* cas2 = gvn.transform(new CompareAndSwapPNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, cas->order())); + Node* scmemproj2 = gvn.transform(new SCMemProjNode(cas2)); + kit->set_control(elsen2); + kit->set_memory(scmemproj2, alias_idx); + + // Merge inner flow - check if healed oop was equal too expected. + region2->set_req(1, kit->control()); + region2->set_req(2, then2); + phi2->set_req(1, cas2); + phi2->set_req(2, kit->intcon(0)); + phi_mem2->init_req(1, scmemproj2); + kit->set_memory(phi_mem2, alias_idx); + + // Merge outer flow - then check if first CAS succeeded + region->set_req(1, then); + region->set_req(2, region2); + phi->set_req(1, kit->intcon(1)); + phi->set_req(2, phi2); + phi_mem->init_req(2, phi_mem2); + kit->set_memory(phi_mem, alias_idx); + + gvn.transform(region2); + gvn.transform(phi2); + gvn.transform(phi_mem2); + gvn.transform(region); + gvn.transform(phi); + gvn.transform(phi_mem); + + kit->set_control(region); + kit->insert_mem_bar(Op_MemBarCPUOrder); + + return phi; + } + + Node* ZBarrierSetC2::make_cmpx_loadbarrier(C2AtomicParseAccess& access) const { + CompareAndExchangePNode* cmpx = (CompareAndExchangePNode*)access.raw_access(); + GraphKit* kit = access.kit(); + PhaseGVN& gvn = kit->gvn(); + Compile* C = Compile::current(); + + Node* in_ctrl = cmpx->in(MemNode::Control); + Node* in_mem = cmpx->in(MemNode::Memory); + Node* in_adr = cmpx->in(MemNode::Address); + Node* in_val = cmpx->in(MemNode::ValueIn); + Node* in_expected = cmpx->in(LoadStoreConditionalNode::ExpectedIn); + + float likely = PROB_LIKELY(0.999); + + const TypePtr *adr_type = cmpx->get_ptr_type(); + Compile::AliasType* alias_type = C->alias_type(adr_type); + int alias_idx = C->get_alias_index(adr_type); + + // Outer check - true: continue, false: load and check + Node* region = new RegionNode(3); + Node* phi = new PhiNode(region, adr_type); + + // Inner check - is the healed ref equal to the expected + Node* region2 = new RegionNode(3); + Node* phi2 = new PhiNode(region2, adr_type); + + // Check if cmpx succeeded + Node* cmp = gvn.transform(new CmpPNode(cmpx, in_expected)); + Node* bol = gvn.transform(new BoolNode(cmp, BoolTest::eq))->as_Bool(); + IfNode* iff = gvn.transform(new IfNode(in_ctrl, bol, likely, COUNT_UNKNOWN))->as_If(); + Node* then = gvn.transform(new IfTrueNode(iff)); + Node* elsen = gvn.transform(new IfFalseNode(iff)); + + Node* scmemproj1 = gvn.transform(new SCMemProjNode(cmpx)); + kit->set_memory(scmemproj1, alias_idx); + + // CAS fail - reload and heal oop + Node* reload = kit->make_load(elsen, in_adr, TypeOopPtr::BOTTOM, T_OBJECT, MemNode::unordered); + Node* barrier = gvn.transform(new LoadBarrierNode(C, elsen, scmemproj1, reload, in_adr, false, true, false)); + Node* barrierctrl = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control)); + Node* barrierdata = gvn.transform(new ProjNode(barrier, LoadBarrierNode::Oop)); + + // Check load + Node* tmpX = gvn.transform(new CastP2XNode(NULL, barrierdata)); + Node* in_expX = gvn.transform(new CastP2XNode(NULL, in_expected)); + Node* cmp2 = gvn.transform(new CmpXNode(tmpX, in_expX)); + Node *bol2 = gvn.transform(new BoolNode(cmp2, BoolTest::ne))->as_Bool(); + IfNode* iff2 = gvn.transform(new IfNode(barrierctrl, bol2, likely, COUNT_UNKNOWN))->as_If(); + Node* then2 = gvn.transform(new IfTrueNode(iff2)); + Node* elsen2 = gvn.transform(new IfFalseNode(iff2)); + + // Redo CAS + Node* cmpx2 = gvn.transform(new CompareAndExchangePNode(elsen2, kit->memory(alias_idx), in_adr, in_val, in_expected, adr_type, cmpx->get_ptr_type(), cmpx->order())); + Node* scmemproj2 = gvn.transform(new SCMemProjNode(cmpx2)); + kit->set_control(elsen2); + kit->set_memory(scmemproj2, alias_idx); + + // Merge inner flow - check if healed oop was equal too expected. + region2->set_req(1, kit->control()); + region2->set_req(2, then2); + phi2->set_req(1, cmpx2); + phi2->set_req(2, barrierdata); + + // Merge outer flow - then check if first cas succeeded + region->set_req(1, then); + region->set_req(2, region2); + phi->set_req(1, cmpx); + phi->set_req(2, phi2); + + gvn.transform(region2); + gvn.transform(phi2); + gvn.transform(region); + gvn.transform(phi); + + kit->set_control(region); + kit->set_memory(in_mem, alias_idx); + kit->insert_mem_bar(Op_MemBarCPUOrder); + + return phi; + } + + Node* ZBarrierSetC2::load_barrier(GraphKit* kit, Node* val, Node* adr, bool weak, bool writeback, bool oop_reload_allowed) const { + PhaseGVN& gvn = kit->gvn(); + Node* barrier = new LoadBarrierNode(Compile::current(), kit->control(), kit->memory(TypeRawPtr::BOTTOM), val, adr, weak, writeback, oop_reload_allowed); + Node* transformed_barrier = gvn.transform(barrier); + + if (transformed_barrier->is_LoadBarrier()) { + if (barrier == transformed_barrier) { + kit->set_control(gvn.transform(new ProjNode(barrier, LoadBarrierNode::Control))); + } + Node* result = gvn.transform(new ProjNode(transformed_barrier, LoadBarrierNode::Oop)); + return result; + } else { + return val; + } + } + static bool barrier_needed(C2Access& access) { return ZBarrierSet::barrier_needed(access.decorators(), access.type()); } Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
*** 473,534 **** if (!barrier_needed(access)) { return p; } bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; ! if (p->isa_Load()) { ! load_set_barrier(p->as_Load(), weak); } - return p; } Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* val_type) const { Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); ! LoadStoreNode* lsn = result->as_LoadStore(); ! if (barrier_needed(access)) { ! lsn->set_has_barrier(); } ! return lsn; } Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); ! LoadStoreNode* lsn = result->as_LoadStore(); ! if (barrier_needed(access)) { ! lsn->set_has_barrier(); } ! return lsn; } Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const { Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); ! LoadStoreNode* lsn = result->as_LoadStore(); ! if (barrier_needed(access)) { ! lsn->set_has_barrier(); } ! return lsn; } // == Macro Expansion == - // Optimized, low spill, loadbarrier variant using stub specialized on register used void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const { PhaseIterGVN &igvn = phase->igvn(); float unlikely = PROB_UNLIKELY(0.999); Node* in_ctrl = barrier->in(LoadBarrierNode::Control); Node* in_mem = barrier->in(LoadBarrierNode::Memory); Node* in_val = barrier->in(LoadBarrierNode::Oop); Node* in_adr = barrier->in(LoadBarrierNode::Address); ! Node* out_ctrl = barrier->proj_out_or_null(LoadBarrierNode::Control); Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null"); Node* jthread = igvn.transform(new ThreadLocalNode()); Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered)); --- 693,962 ---- if (!barrier_needed(access)) { return p; } bool weak = (access.decorators() & ON_WEAK_OOP_REF) != 0; ! ! assert(access.is_parse_access(), "entry not supported at optimization time"); ! C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); ! GraphKit* kit = parse_access.kit(); ! PhaseGVN& gvn = kit->gvn(); ! Node* adr = access.addr().node(); ! Node* heap_base_oop = access.base(); ! bool unsafe = (access.decorators() & C2_UNSAFE_ACCESS) != 0; ! if (unsafe) { ! if (!ZVerifyLoadBarriers) { ! p = load_barrier(kit, p, adr); ! } else { ! if (!TypePtr::NULL_PTR->higher_equal(gvn.type(heap_base_oop))) { ! p = load_barrier(kit, p, adr); ! } else { ! IdealKit ideal(kit); ! IdealVariable res(ideal); ! #define __ ideal. ! __ declarations_done(); ! __ set(res, p); ! __ if_then(heap_base_oop, BoolTest::ne, kit->null(), PROB_UNLIKELY(0.999)); { ! kit->sync_kit(ideal); ! p = load_barrier(kit, p, adr); ! __ set(res, p); ! __ sync_kit(kit); ! } __ end_if(); ! kit->final_sync(ideal); ! p = __ value(res); ! #undef __ ! } ! } ! return p; ! } else { ! return load_barrier(parse_access.kit(), p, access.addr().node(), weak, true, true); } } Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* val_type) const { Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); ! if (!barrier_needed(access)) { ! return result; } ! ! access.set_needs_pinning(false); ! return make_cmpx_loadbarrier(access); } Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { Node* result = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); ! if (!barrier_needed(access)) { ! return result; ! } ! ! Node* load_store = access.raw_access(); ! bool weak_cas = (access.decorators() & C2_WEAK_CMPXCHG) != 0; ! bool expected_is_null = (expected_val->get_ptr_type() == TypePtr::NULL_PTR); ! ! if (!expected_is_null) { ! if (weak_cas) { ! access.set_needs_pinning(false); ! load_store = make_cas_loadbarrier(access); ! } else { ! access.set_needs_pinning(false); ! load_store = make_cas_loadbarrier(access); ! } } ! ! return load_store; } Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const { Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); ! if (!barrier_needed(access)) { ! return result; } ! ! Node* load_store = access.raw_access(); ! Node* adr = access.addr().node(); ! ! assert(access.is_parse_access(), "entry not supported at optimization time"); ! C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access); ! return load_barrier(parse_access.kit(), load_store, adr, false, false, false); } // == Macro Expansion == void ZBarrierSetC2::expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const { + Node* in_ctrl = barrier->in(LoadBarrierNode::Control); + Node* in_mem = barrier->in(LoadBarrierNode::Memory); + Node* in_val = barrier->in(LoadBarrierNode::Oop); + Node* in_adr = barrier->in(LoadBarrierNode::Address); + + Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); + Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); + PhaseIterGVN &igvn = phase->igvn(); + + if (ZVerifyLoadBarriers) { + igvn.replace_node(out_res, in_val); + igvn.replace_node(out_ctrl, in_ctrl); + return; + } + + if (barrier->can_be_eliminated()) { + // Clone and pin the load for this barrier below the dominating + // barrier: the load cannot be allowed to float above the + // dominating barrier + Node* load = in_val; + + if (load->is_Load()) { + Node* new_load = load->clone(); + Node* addp = new_load->in(MemNode::Address); + assert(addp->is_AddP() || addp->is_Phi() || addp->is_Load(), "bad address"); + Node* cast = new CastPPNode(addp, igvn.type(addp), true); + Node* ctrl = NULL; + Node* similar = barrier->in(LoadBarrierNode::Similar); + if (similar->is_Phi()) { + // already expanded + ctrl = similar->in(0); + } else { + assert(similar->is_Proj() && similar->in(0)->is_LoadBarrier(), "unexpected graph shape"); + ctrl = similar->in(0)->as_LoadBarrier()->proj_out(LoadBarrierNode::Control); + } + assert(ctrl != NULL, "bad control"); + cast->set_req(0, ctrl); + igvn.transform(cast); + new_load->set_req(MemNode::Address, cast); + igvn.transform(new_load); + + igvn.replace_node(out_res, new_load); + igvn.replace_node(out_ctrl, in_ctrl); + return; + } + // cannot eliminate + } + + // There are two cases that require the basic loadbarrier + // 1) When the writeback of a healed oop must be avoided (swap) + // 2) When we must guarantee that no reload of is done (swap, cas, cmpx) + if (!barrier->is_writeback()) { + assert(!barrier->oop_reload_allowed(), "writeback barriers should be marked as requires oop"); + } + + if (!barrier->oop_reload_allowed()) { + expand_loadbarrier_basic(phase, barrier); + } else { + expand_loadbarrier_optimized(phase, barrier); + } + } + + // Basic loadbarrier using conventional argument passing + void ZBarrierSetC2::expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const { + PhaseIterGVN &igvn = phase->igvn(); + + Node* in_ctrl = barrier->in(LoadBarrierNode::Control); + Node* in_mem = barrier->in(LoadBarrierNode::Memory); + Node* in_val = barrier->in(LoadBarrierNode::Oop); + Node* in_adr = barrier->in(LoadBarrierNode::Address); + + Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); + Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); + float unlikely = PROB_UNLIKELY(0.999); + const Type* in_val_maybe_null_t = igvn.type(in_val); + + Node* jthread = igvn.transform(new ThreadLocalNode()); + Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); + Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered)); + Node* cast = igvn.transform(new CastP2XNode(in_ctrl, in_val)); + Node* obj_masked = igvn.transform(new AndXNode(cast, bad_mask)); + Node* cmp = igvn.transform(new CmpXNode(obj_masked, igvn.zerocon(TypeX_X->basic_type()))); + Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); + IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); + Node* then = igvn.transform(new IfTrueNode(iff)); + Node* elsen = igvn.transform(new IfFalseNode(iff)); + + Node* result_region; + Node* result_val; + + result_region = new RegionNode(3); + result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM); + + result_region->set_req(1, elsen); + Node* res = igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t)); + res->init_req(0, elsen); + result_val->set_req(1, res); + + const TypeFunc *tf = load_barrier_Type(); + Node* call; + if (barrier->is_weak()) { + call = new CallLeafNode(tf, + ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr(), + "ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded", + TypeRawPtr::BOTTOM); + } else { + call = new CallLeafNode(tf, + ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(), + "ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded", + TypeRawPtr::BOTTOM); + } + + call->init_req(TypeFunc::Control, then); + call->init_req(TypeFunc::I_O , phase->top()); + call->init_req(TypeFunc::Memory , in_mem); + call->init_req(TypeFunc::FramePtr, phase->top()); + call->init_req(TypeFunc::ReturnAdr, phase->top()); + call->init_req(TypeFunc::Parms+0, in_val); + if (barrier->is_writeback()) { + call->init_req(TypeFunc::Parms+1, in_adr); + } else { + // When slow path is called with a null address, the healed oop will not be written back + call->init_req(TypeFunc::Parms+1, igvn.zerocon(T_OBJECT)); + } + call = igvn.transform(call); + + Node* ctrl = igvn.transform(new ProjNode(call, TypeFunc::Control)); + res = igvn.transform(new ProjNode(call, TypeFunc::Parms)); + res = igvn.transform(new CheckCastPPNode(ctrl, res, in_val_maybe_null_t)); + + result_region->set_req(2, ctrl); + result_val->set_req(2, res); + + result_region = igvn.transform(result_region); + result_val = igvn.transform(result_val); + + if (out_ctrl != NULL) { // Added if cond + igvn.replace_node(out_ctrl, result_region); + } + igvn.replace_node(out_res, result_val); + } + + // Optimized, low spill, loadbarrier variant using stub specialized on register used + void ZBarrierSetC2::expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const { + PhaseIterGVN &igvn = phase->igvn(); + #ifdef PRINT_NODE_TRAVERSALS + Node* preceding_barrier_node = barrier->in(LoadBarrierNode::Oop); + #endif Node* in_ctrl = barrier->in(LoadBarrierNode::Control); Node* in_mem = barrier->in(LoadBarrierNode::Memory); Node* in_val = barrier->in(LoadBarrierNode::Oop); Node* in_adr = barrier->in(LoadBarrierNode::Address); ! Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control); Node* out_res = barrier->proj_out(LoadBarrierNode::Oop); assert(barrier->in(LoadBarrierNode::Oop) != NULL, "oop to loadbarrier node cannot be null"); + #ifdef PRINT_NODE_TRAVERSALS + tty->print("\n\n\nBefore barrier optimization:\n"); + traverse(barrier, out_ctrl, out_res, -1); + + tty->print("\nBefore barrier optimization: preceding_barrier_node\n"); + traverse(preceding_barrier_node, out_ctrl, out_res, -1); + #endif + + float unlikely = PROB_UNLIKELY(0.999); + Node* jthread = igvn.transform(new ThreadLocalNode()); Node* adr = phase->basic_plus_adr(jthread, in_bytes(ZThreadLocalData::address_bad_mask_offset())); Node* bad_mask = igvn.transform(LoadNode::make(igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
*** 538,581 **** Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); Node* then = igvn.transform(new IfTrueNode(iff)); Node* elsen = igvn.transform(new IfFalseNode(iff)); ! Node* new_loadp = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), ! (const TypePtr*) in_val->bottom_type(), MemNode::unordered, barrier->is_weak())); // Create the final region/phi pair to converge cntl/data paths to downstream code Node* result_region = igvn.transform(new RegionNode(3)); result_region->set_req(1, then); result_region->set_req(2, elsen); Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM)); result_phi->set_req(1, new_loadp); result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop)); ! if (out_ctrl != NULL) { igvn.replace_node(out_ctrl, result_region); } igvn.replace_node(out_res, result_phi); assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!"); ! igvn.remove_dead_node(barrier); ! igvn.remove_dead_node(out_ctrl); ! igvn.remove_dead_node(out_res); assert(is_gc_barrier_node(result_phi), "sanity"); assert(step_over_gc_barrier(result_phi) == in_val, "sanity"); - - phase->C->print_method(PHASE_BARRIER_EXPANSION, 4, barrier->_idx); } bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const { ZBarrierSetC2State* s = state(); if (s->load_barrier_count() > 0) { PhaseMacroExpand macro(igvn); ! int skipped = 0; while (s->load_barrier_count() > skipped) { int load_barrier_count = s->load_barrier_count(); LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped); if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) { --- 966,1026 ---- Node *bol = igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool(); IfNode* iff = igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If(); Node* then = igvn.transform(new IfTrueNode(iff)); Node* elsen = igvn.transform(new IfFalseNode(iff)); ! Node* slow_path_surrogate; ! if (!barrier->is_weak()) { ! slow_path_surrogate = igvn.transform(new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), ! (const TypePtr*) in_val->bottom_type(), MemNode::unordered)); ! } else { ! slow_path_surrogate = igvn.transform(new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), ! (const TypePtr*) in_val->bottom_type(), MemNode::unordered)); ! } + Node *new_loadp; + new_loadp = slow_path_surrogate; // Create the final region/phi pair to converge cntl/data paths to downstream code Node* result_region = igvn.transform(new RegionNode(3)); result_region->set_req(1, then); result_region->set_req(2, elsen); Node* result_phi = igvn.transform(new PhiNode(result_region, TypeInstPtr::BOTTOM)); result_phi->set_req(1, new_loadp); result_phi->set_req(2, barrier->in(LoadBarrierNode::Oop)); ! // Finally, connect the original outputs to the barrier region and phi to complete the expansion/substitution ! // igvn.replace_node(out_ctrl, result_region); ! if (out_ctrl != NULL) { // added if cond igvn.replace_node(out_ctrl, result_region); } igvn.replace_node(out_res, result_phi); assert(barrier->outcnt() == 0,"LoadBarrier macro node has non-null outputs after expansion!"); ! #ifdef PRINT_NODE_TRAVERSALS ! tty->print("\nAfter barrier optimization: old out_ctrl\n"); ! traverse(out_ctrl, out_ctrl, out_res, -1); ! tty->print("\nAfter barrier optimization: old out_res\n"); ! traverse(out_res, out_ctrl, out_res, -1); ! tty->print("\nAfter barrier optimization: old barrier\n"); ! traverse(barrier, out_ctrl, out_res, -1); ! tty->print("\nAfter barrier optimization: preceding_barrier_node\n"); ! traverse(preceding_barrier_node, result_region, result_phi, -1); ! #endif assert(is_gc_barrier_node(result_phi), "sanity"); assert(step_over_gc_barrier(result_phi) == in_val, "sanity"); } bool ZBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const { ZBarrierSetC2State* s = state(); if (s->load_barrier_count() > 0) { PhaseMacroExpand macro(igvn); ! #ifdef ASSERT ! verify_gc_barriers(false); ! #endif int skipped = 0; while (s->load_barrier_count() > skipped) { int load_barrier_count = s->load_barrier_count(); LoadBarrierNode * n = s->load_barrier_node(load_barrier_count-1-skipped); if (igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
*** 612,621 **** --- 1057,1435 ---- } return false; } + // == Loop optimization == + + static bool replace_with_dominating_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, bool last_round) { + PhaseIterGVN &igvn = phase->igvn(); + Compile* C = Compile::current(); + + LoadBarrierNode* lb2 = lb->has_dominating_barrier(phase, false, last_round); + if (lb2 == NULL) { + return false; + } + + if (lb->in(LoadBarrierNode::Oop) != lb2->in(LoadBarrierNode::Oop)) { + assert(lb->in(LoadBarrierNode::Address) == lb2->in(LoadBarrierNode::Address), "Invalid address"); + igvn.replace_input_of(lb, LoadBarrierNode::Similar, lb2->proj_out(LoadBarrierNode::Oop)); + C->set_major_progress(); + return false; + } + + // That transformation may cause the Similar edge on dominated load barriers to be invalid + lb->fix_similar_in_uses(&igvn); + + Node* val = lb->proj_out(LoadBarrierNode::Oop); + assert(lb2->has_true_uses(), "Invalid uses"); + assert(lb2->in(LoadBarrierNode::Oop) == lb->in(LoadBarrierNode::Oop), "Invalid oop"); + phase->lazy_update(lb, lb->in(LoadBarrierNode::Control)); + phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control)); + igvn.replace_node(val, lb2->proj_out(LoadBarrierNode::Oop)); + + return true; + } + + static Node* find_dominating_memory(PhaseIdealLoop* phase, Node* mem, Node* dom, int i) { + assert(dom->is_Region() || i == -1, ""); + + Node* m = mem; + while(phase->is_dominator(dom, phase->has_ctrl(m) ? phase->get_ctrl(m) : m->in(0))) { + if (m->is_Mem()) { + assert(m->as_Mem()->adr_type() == TypeRawPtr::BOTTOM, ""); + m = m->in(MemNode::Memory); + } else if (m->is_MergeMem()) { + m = m->as_MergeMem()->memory_at(Compile::AliasIdxRaw); + } else if (m->is_Phi()) { + if (m->in(0) == dom && i != -1) { + m = m->in(i); + break; + } else { + m = m->in(LoopNode::EntryControl); + } + } else if (m->is_Proj()) { + m = m->in(0); + } else if (m->is_SafePoint() || m->is_MemBar()) { + m = m->in(TypeFunc::Memory); + } else { + #ifdef ASSERT + m->dump(); + #endif + ShouldNotReachHere(); + } + } + + return m; + } + + static LoadBarrierNode* clone_load_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* ctl, Node* mem, Node* oop_in) { + PhaseIterGVN &igvn = phase->igvn(); + Compile* C = Compile::current(); + Node* the_clone = lb->clone(); + the_clone->set_req(LoadBarrierNode::Control, ctl); + the_clone->set_req(LoadBarrierNode::Memory, mem); + if (oop_in != NULL) { + the_clone->set_req(LoadBarrierNode::Oop, oop_in); + } + + LoadBarrierNode* new_lb = the_clone->as_LoadBarrier(); + igvn.register_new_node_with_optimizer(new_lb); + IdealLoopTree *loop = phase->get_loop(new_lb->in(0)); + phase->set_ctrl(new_lb, new_lb->in(0)); + phase->set_loop(new_lb, loop); + phase->set_idom(new_lb, new_lb->in(0), phase->dom_depth(new_lb->in(0))+1); + if (!loop->_child) { + loop->_body.push(new_lb); + } + + Node* proj_ctl = new ProjNode(new_lb, LoadBarrierNode::Control); + igvn.register_new_node_with_optimizer(proj_ctl); + phase->set_ctrl(proj_ctl, proj_ctl->in(0)); + phase->set_loop(proj_ctl, loop); + phase->set_idom(proj_ctl, new_lb, phase->dom_depth(new_lb)+1); + if (!loop->_child) { + loop->_body.push(proj_ctl); + } + + Node* proj_oop = new ProjNode(new_lb, LoadBarrierNode::Oop); + phase->register_new_node(proj_oop, new_lb); + + if (!new_lb->in(LoadBarrierNode::Similar)->is_top()) { + LoadBarrierNode* similar = new_lb->in(LoadBarrierNode::Similar)->in(0)->as_LoadBarrier(); + if (!phase->is_dominator(similar, ctl)) { + igvn.replace_input_of(new_lb, LoadBarrierNode::Similar, C->top()); + } + } + + return new_lb; + } + + static void replace_barrier(PhaseIdealLoop* phase, LoadBarrierNode* lb, Node* new_val) { + PhaseIterGVN &igvn = phase->igvn(); + Node* val = lb->proj_out(LoadBarrierNode::Oop); + igvn.replace_node(val, new_val); + phase->lazy_update(lb, lb->in(LoadBarrierNode::Control)); + phase->lazy_replace(lb->proj_out(LoadBarrierNode::Control), lb->in(LoadBarrierNode::Control)); + } + + static bool split_barrier_thru_phi(PhaseIdealLoop* phase, LoadBarrierNode* lb) { + PhaseIterGVN &igvn = phase->igvn(); + Compile* C = Compile::current(); + + if (lb->in(LoadBarrierNode::Oop)->is_Phi()) { + Node* oop_phi = lb->in(LoadBarrierNode::Oop); + + if ((oop_phi->req() != 3) || (oop_phi->in(2) == oop_phi)) { + // Ignore phis with only one input + return false; + } + + if (phase->is_dominator(phase->get_ctrl(lb->in(LoadBarrierNode::Address)), + oop_phi->in(0)) && phase->get_ctrl(lb->in(LoadBarrierNode::Address)) != oop_phi->in(0)) { + // That transformation may cause the Similar edge on dominated load barriers to be invalid + lb->fix_similar_in_uses(&igvn); + + RegionNode* region = oop_phi->in(0)->as_Region(); + + int backedge = LoopNode::LoopBackControl; + if (region->is_Loop() && region->in(backedge)->is_Proj() && region->in(backedge)->in(0)->is_If()) { + Node* c = region->in(backedge)->in(0)->in(0); + assert(c->unique_ctrl_out() == region->in(backedge)->in(0), ""); + Node* oop = lb->in(LoadBarrierNode::Oop)->in(backedge); + Node* oop_c = phase->has_ctrl(oop) ? phase->get_ctrl(oop) : oop; + if (!phase->is_dominator(oop_c, c)) { + return false; + } + } + + // If the node on the backedge above the phi is the node itself - we have a self loop. + // Don't clone - this will be folded later. + if (oop_phi->in(LoopNode::LoopBackControl) == lb->proj_out(LoadBarrierNode::Oop)) { + return false; + } + + bool is_strip_mined = region->is_CountedLoop() && region->as_CountedLoop()->is_strip_mined(); + Node *phi = oop_phi->clone(); + + for (uint i = 1; i < region->req(); i++) { + Node* ctrl = region->in(i); + if (ctrl != C->top()) { + assert(!phase->is_dominator(ctrl, region) || region->is_Loop(), ""); + + Node* mem = lb->in(LoadBarrierNode::Memory); + Node* m = find_dominating_memory(phase, mem, region, i); + + if (region->is_Loop() && i == LoopNode::LoopBackControl && ctrl->is_Proj() && ctrl->in(0)->is_If()) { + ctrl = ctrl->in(0)->in(0); + } else if (region->is_Loop() && is_strip_mined) { + // If this is a strip mined loop, control must move above OuterStripMinedLoop + assert(i == LoopNode::EntryControl, "check"); + assert(ctrl->is_OuterStripMinedLoop(), "sanity"); + ctrl = ctrl->as_OuterStripMinedLoop()->in(LoopNode::EntryControl); + } + + LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, ctrl, m, lb->in(LoadBarrierNode::Oop)->in(i)); + Node* out_ctrl = new_lb->proj_out(LoadBarrierNode::Control); + + if (is_strip_mined && (i == LoopNode::EntryControl)) { + assert(region->in(i)->is_OuterStripMinedLoop(), ""); + igvn.replace_input_of(region->in(i), i, out_ctrl); + phase->set_idom(region->in(i), out_ctrl, phase->dom_depth(out_ctrl)); + } else if (ctrl == region->in(i)) { + igvn.replace_input_of(region, i, out_ctrl); + // Only update the idom if is the loop entry we are updating + // - A loop backedge doesn't change the idom + if (region->is_Loop() && i == LoopNode::EntryControl) { + phase->set_idom(region, out_ctrl, phase->dom_depth(out_ctrl)); + } + } else { + Node* iff = region->in(i)->in(0); + igvn.replace_input_of(iff, 0, out_ctrl); + phase->set_idom(iff, out_ctrl, phase->dom_depth(out_ctrl)+1); + } + phi->set_req(i, new_lb->proj_out(LoadBarrierNode::Oop)); + } + } + phase->register_new_node(phi, region); + replace_barrier(phase, lb, phi); + + if (region->is_Loop()) { + // Load barrier moved to the back edge of the Loop may now + // have a safepoint on the path to the barrier on the Similar + // edge + igvn.replace_input_of(phi->in(LoopNode::LoopBackControl)->in(0), LoadBarrierNode::Similar, C->top()); + Node* head = region->in(LoopNode::EntryControl); + phase->set_idom(region, head, phase->dom_depth(head)+1); + phase->recompute_dom_depth(); + if (head->is_CountedLoop() && head->as_CountedLoop()->is_main_loop()) { + head->as_CountedLoop()->set_normal_loop(); + } + } + + return true; + } + } + + return false; + } + + static bool move_out_of_loop(PhaseIdealLoop* phase, LoadBarrierNode* lb) { + PhaseIterGVN &igvn = phase->igvn(); + IdealLoopTree *lb_loop = phase->get_loop(lb->in(0)); + if (lb_loop != phase->ltree_root() && !lb_loop->_irreducible) { + Node* oop_ctrl = phase->get_ctrl(lb->in(LoadBarrierNode::Oop)); + IdealLoopTree *oop_loop = phase->get_loop(oop_ctrl); + IdealLoopTree* adr_loop = phase->get_loop(phase->get_ctrl(lb->in(LoadBarrierNode::Address))); + if (!lb_loop->is_member(oop_loop) && !lb_loop->is_member(adr_loop)) { + // That transformation may cause the Similar edge on dominated load barriers to be invalid + lb->fix_similar_in_uses(&igvn); + + Node* head = lb_loop->_head; + assert(head->is_Loop(), ""); + + if (phase->is_dominator(head, oop_ctrl)) { + assert(oop_ctrl->Opcode() == Op_CProj && oop_ctrl->in(0)->Opcode() == Op_NeverBranch, ""); + assert(lb_loop->is_member(phase->get_loop(oop_ctrl->in(0)->in(0))), ""); + return false; + } + + if (head->is_CountedLoop()) { + CountedLoopNode* cloop = head->as_CountedLoop(); + if (cloop->is_main_loop()) { + cloop->set_normal_loop(); + } + // When we are moving barrier out of a counted loop, + // make sure we move it all the way out of the strip mined outer loop. + if (cloop->is_strip_mined()) { + head = cloop->outer_loop(); + } + } + + Node* mem = lb->in(LoadBarrierNode::Memory); + Node* m = find_dominating_memory(phase, mem, head, -1); + + LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, head->in(LoopNode::EntryControl), m, NULL); + + assert(phase->idom(head) == head->in(LoopNode::EntryControl), ""); + Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control); + igvn.replace_input_of(head, LoopNode::EntryControl, proj_ctl); + phase->set_idom(head, proj_ctl, phase->dom_depth(proj_ctl) + 1); + + replace_barrier(phase, lb, new_lb->proj_out(LoadBarrierNode::Oop)); + + phase->recompute_dom_depth(); + + return true; + } + } + + return false; + } + + static bool common_barriers(PhaseIdealLoop* phase, LoadBarrierNode* lb) { + PhaseIterGVN &igvn = phase->igvn(); + Node* in_val = lb->in(LoadBarrierNode::Oop); + for (DUIterator_Fast imax, i = in_val->fast_outs(imax); i < imax; i++) { + Node* u = in_val->fast_out(i); + if (u != lb && u->is_LoadBarrier() && u->as_LoadBarrier()->has_true_uses()) { + Node* this_ctrl = lb->in(LoadBarrierNode::Control); + Node* other_ctrl = u->in(LoadBarrierNode::Control); + + Node* lca = phase->dom_lca(this_ctrl, other_ctrl); + bool ok = true; + + Node* proj1 = NULL; + Node* proj2 = NULL; + + while (this_ctrl != lca && ok) { + if (this_ctrl->in(0) != NULL && + this_ctrl->in(0)->is_MultiBranch()) { + if (this_ctrl->in(0)->in(0) == lca) { + assert(proj1 == NULL, ""); + assert(this_ctrl->is_Proj(), ""); + proj1 = this_ctrl; + } else if (!(this_ctrl->in(0)->is_If() && this_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) { + ok = false; + } + } + this_ctrl = phase->idom(this_ctrl); + } + while (other_ctrl != lca && ok) { + if (other_ctrl->in(0) != NULL && + other_ctrl->in(0)->is_MultiBranch()) { + if (other_ctrl->in(0)->in(0) == lca) { + assert(other_ctrl->is_Proj(), ""); + assert(proj2 == NULL, ""); + proj2 = other_ctrl; + } else if (!(other_ctrl->in(0)->is_If() && other_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none))) { + ok = false; + } + } + other_ctrl = phase->idom(other_ctrl); + } + assert(proj1 == NULL || proj2 == NULL || proj1->in(0) == proj2->in(0), ""); + if (ok && proj1 && proj2 && proj1 != proj2 && proj1->in(0)->is_If()) { + // That transformation may cause the Similar edge on dominated load barriers to be invalid + lb->fix_similar_in_uses(&igvn); + u->as_LoadBarrier()->fix_similar_in_uses(&igvn); + + Node* split = lca->unique_ctrl_out(); + assert(split->in(0) == lca, ""); + + Node* mem = lb->in(LoadBarrierNode::Memory); + Node* m = find_dominating_memory(phase, mem, split, -1); + LoadBarrierNode* new_lb = clone_load_barrier(phase, lb, lca, m, NULL); + + Node* proj_ctl = new_lb->proj_out(LoadBarrierNode::Control); + igvn.replace_input_of(split, 0, new_lb->proj_out(LoadBarrierNode::Control)); + phase->set_idom(split, proj_ctl, phase->dom_depth(proj_ctl)+1); + + Node* proj_oop = new_lb->proj_out(LoadBarrierNode::Oop); + replace_barrier(phase, lb, proj_oop); + replace_barrier(phase, u->as_LoadBarrier(), proj_oop); + + phase->recompute_dom_depth(); + + return true; + } + } + } + + return false; + } + + void ZBarrierSetC2::loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round) { + if (!Compile::current()->directive()->ZOptimizeLoadBarriersOption) { + return; + } + + if (!node->is_LoadBarrier()) { + return; + } + + if (!node->as_LoadBarrier()->has_true_uses()) { + return; + } + + if (replace_with_dominating_barrier(phase, node->as_LoadBarrier(), last_round)) { + return; + } + + if (split_barrier_thru_phi(phase, node->as_LoadBarrier())) { + return; + } + + if (move_out_of_loop(phase, node->as_LoadBarrier())) { + return; + } + + if (common_barriers(phase, node->as_LoadBarrier())) { + return; + } + } + Node* ZBarrierSetC2::step_over_gc_barrier(Node* c) const { Node* node = c; // 1. This step follows potential oop projections of a load barrier before expansion if (node->is_Proj()) {
*** 629,639 **** // 3. This step checks for the phi corresponding to an optimized load barrier expansion if (node->is_Phi()) { PhiNode* phi = node->as_Phi(); Node* n = phi->in(1); ! if (n != NULL && n->is_LoadBarrierSlowReg()) { assert(c == node, "projections from step 1 should only be seen before macro expansion"); return phi->in(2); } } --- 1443,1453 ---- // 3. This step checks for the phi corresponding to an optimized load barrier expansion if (node->is_Phi()) { PhiNode* phi = node->as_Phi(); Node* n = phi->in(1); ! if (n != NULL && (n->is_LoadBarrierSlowReg() || n->is_LoadBarrierWeakSlowReg())) { assert(c == node, "projections from step 1 should only be seen before macro expansion"); return phi->in(2); } }
*** 643,722 **** bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { return type == T_OBJECT || type == T_ARRAY; } bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const { ! switch (opcode) { ! case Op_LoadBarrier: ! assert(0, "There should be no load barriers left"); ! case Op_ZGetAndSetP: ! case Op_ZCompareAndExchangeP: ! case Op_ZCompareAndSwapP: ! case Op_ZWeakCompareAndSwapP: ! case Op_LoadBarrierSlowReg: #ifdef ASSERT ! if (VerifyOptoOopOffsets) { ! MemNode *mem = n->as_Mem(); ! // Check to see if address types have grounded out somehow. ! const TypeInstPtr *tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr(); ! ciInstanceKlass *k = tp->klass()->as_instance_klass(); ! bool oop_offset_is_sane = k->contains_field_offset(tp->offset()); ! assert(!tp || oop_offset_is_sane, ""); ! } ! #endif ! return true; ! default: ! return false; } } bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const { ! switch(opcode) { ! case Op_CallLeaf: ! if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() || ! n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) { ! mem_op = true; ! mem_addr_idx = TypeFunc::Parms + 1; ! return true; ! } ! return false; ! default: ! return false; } - } ! bool ZBarrierSetC2::matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const { ! switch(opcode) { ! case Op_ZCompareAndExchangeP: ! case Op_ZCompareAndSwapP: ! case Op_ZWeakCompareAndSwapP: { ! Node *mem = n->in(MemNode::Address); ! Node *keepalive = n->in(5); ! Node *pair1 = new BinaryNode(mem, keepalive); ! ! Node *newval = n->in(MemNode::ValueIn); ! Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn); ! Node *pair2 = new BinaryNode(oldval, newval); ! ! n->set_req(MemNode::Address, pair1); ! n->set_req(MemNode::ValueIn, pair2); ! n->del_req(5); ! n->del_req(LoadStoreConditionalNode::ExpectedIn); ! return true; ! } ! case Op_ZGetAndSetP: { ! Node *keepalive = n->in(4); ! Node *newval = n->in(MemNode::ValueIn); ! Node *pair = new BinaryNode(newval, keepalive); ! n->set_req(MemNode::ValueIn, pair); ! n->del_req(4); ! return true; ! } ! ! default: ! return false; ! } } // == Verification == #ifdef ASSERT --- 1457,1495 ---- bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { return type == T_OBJECT || type == T_ARRAY; } bool ZBarrierSetC2::final_graph_reshaping(Compile* compile, Node* n, uint opcode) const { ! if (opcode != Op_LoadBarrierSlowReg && ! opcode != Op_LoadBarrierWeakSlowReg) { ! return false; ! } ! #ifdef ASSERT ! if (VerifyOptoOopOffsets) { ! MemNode* mem = n->as_Mem(); ! // Check to see if address types have grounded out somehow. ! const TypeInstPtr* tp = mem->in(MemNode::Address)->bottom_type()->isa_instptr(); ! ciInstanceKlass* k = tp->klass()->as_instance_klass(); ! bool oop_offset_is_sane = k->contains_field_offset(tp->offset()); ! assert(!tp || oop_offset_is_sane, ""); } + #endif + + return true; } bool ZBarrierSetC2::matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const { ! if (opcode == Op_CallLeaf && ! (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() || ! n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr())) { ! mem_op = true; ! mem_addr_idx = TypeFunc::Parms + 1; ! return true; } ! return false; } // == Verification == #ifdef ASSERT
*** 745,780 **** return true; } void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const { ! switch(phase) { ! case BarrierSetC2::BeforeOptimize: ! case BarrierSetC2::BeforeLateInsertion: ! assert(state()->load_barrier_count() == 0, "No barriers inserted yet"); ! break; ! case BarrierSetC2::BeforeMacroExpand: ! // Barrier placement should be set by now. ! verify_gc_barriers(false /*post_parse*/); ! break; ! case BarrierSetC2::BeforeCodeGen: ! // Barriers has been fully expanded. ! assert(state()->load_barrier_count() == 0, "No more macro barriers"); ! break; ! default: ! assert(0, "Phase without verification"); ! } } - // post_parse implies that there might be load barriers without uses after parsing - // That only applies when adding barriers at parse time. void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const { ZBarrierSetC2State* s = state(); Compile* C = Compile::current(); ResourceMark rm; VectorSet visited(Thread::current()->resource_area()); - for (int i = 0; i < s->load_barrier_count(); i++) { LoadBarrierNode* n = s->load_barrier_node(i); // The dominating barrier on the same address if it exists and // this barrier must not be applied on the value from the same --- 1518,1537 ---- return true; } void ZBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const { ! if (phase == BarrierSetC2::BeforeCodeGen) return; ! bool post_parse = phase == BarrierSetC2::BeforeOptimize; ! verify_gc_barriers(post_parse); } void ZBarrierSetC2::verify_gc_barriers(bool post_parse) const { ZBarrierSetC2State* s = state(); Compile* C = Compile::current(); ResourceMark rm; VectorSet visited(Thread::current()->resource_area()); for (int i = 0; i < s->load_barrier_count(); i++) { LoadBarrierNode* n = s->load_barrier_node(i); // The dominating barrier on the same address if it exists and // this barrier must not be applied on the value from the same
*** 784,794 **** (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) && n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)), "broken similar edge"); ! assert(n->as_LoadBarrier()->has_true_uses(), "found unneeded load barrier"); // Several load barrier nodes chained through their Similar edge // break the code that remove the barriers in final graph reshape. assert(n->in(LoadBarrierNode::Similar)->is_top() || --- 1541,1551 ---- (n->in(LoadBarrierNode::Similar)->in(0)->is_LoadBarrier() && n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Address) == n->in(LoadBarrierNode::Address) && n->in(LoadBarrierNode::Similar)->in(0)->in(LoadBarrierNode::Oop) != n->in(LoadBarrierNode::Oop)), "broken similar edge"); ! assert(post_parse || n->as_LoadBarrier()->has_true_uses(), "found unneeded load barrier"); // Several load barrier nodes chained through their Similar edge // break the code that remove the barriers in final graph reshape. assert(n->in(LoadBarrierNode::Similar)->is_top() ||
*** 799,1633 **** if (!n->in(LoadBarrierNode::Similar)->is_top()) { ResourceMark rm; Unique_Node_List wq; Node* other = n->in(LoadBarrierNode::Similar)->in(0); wq.push(n); for (uint next = 0; next < wq.size(); ++next) { ! Node *nn = wq.at(next); ! assert(nn->is_CFG(), ""); ! assert(!nn->is_SafePoint(), ""); ! if (nn == other) { continue; } ! if (nn->is_Region()) { ! for (uint i = 1; i < nn->req(); i++) { ! Node* m = nn->in(i); if (m != NULL) { wq.push(m); } } } else { ! Node* m = nn->in(0); if (m != NULL) { wq.push(m); } } } } - } - } - - #endif // end verification code - - static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl); - - // This code is cloning all uses of a load that is between a call and the catch blocks, - // to each use. - - static bool fixup_uses_in_catch(PhaseIdealLoop *phase, Node *start_ctrl, Node *node) { - - if (!phase->has_ctrl(node)) { - // This node is floating - doesn't need to be cloned. - assert(node != start_ctrl, "check"); - return false; - } - - Node* ctrl = phase->get_ctrl(node); - if (ctrl != start_ctrl) { - // We are in a successor block - the node is ok. - return false; // Unwind - } - - // Process successor nodes - int outcnt = node->outcnt(); - for (int i = 0; i < outcnt; i++) { - Node* n = node->raw_out(0); - assert(!n->is_LoadBarrier(), "Sanity"); - // Calling recursively, visiting leafs first - fixup_uses_in_catch(phase, start_ctrl, n); - } - - // Now all successors are outside - // - Clone this node to both successors - int no_succs = node->outcnt(); - assert(!node->is_Store(), "Stores not expected here"); - - // In some very rare cases a load that doesn't need a barrier will end up here - // Treat it as a LoadP and the insertion of phis will be done correctly. - if (node->is_Load()) { - call_catch_cleanup_one(phase, node->as_Load(), phase->get_ctrl(node)); - } else { - for (DUIterator_Fast jmax, i = node->fast_outs(jmax); i < jmax; i++) { - Node* use = node->fast_out(i); - Node* clone = node->clone(); - assert(clone->outcnt() == 0, ""); - - assert(use->find_edge(node) != -1, "check"); - phase->igvn().rehash_node_delayed(use); - use->replace_edge(node, clone); - - Node* new_ctrl; - if (use->is_block_start()) { - new_ctrl = use; - } else if (use->is_CFG()) { - new_ctrl = use->in(0); - assert (new_ctrl != NULL, ""); - } else { - new_ctrl = phase->get_ctrl(use); - } - - phase->set_ctrl(clone, new_ctrl); - - if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone op %i as %i to control %i", node->_idx, clone->_idx, new_ctrl->_idx); - phase->igvn().register_new_node_with_optimizer(clone); - --i, --jmax; - } - assert(node->outcnt() == 0, "must be empty now"); - - // Node node is dead. - phase->igvn().remove_dead_node(node); - } - return true; // unwind - return if a use was processed - } - - // Clone a load to a specific catch_proj - static Node* clone_load_to_catchproj(PhaseIdealLoop* phase, Node* load, Node* catch_proj) { - Node* cloned_load = load->clone(); - cloned_load->set_req(0, catch_proj); // set explicit control - phase->set_ctrl(cloned_load, catch_proj); // update - if (phase->C->directive()->ZTraceLoadBarriersOption) tty->print_cr(" Clone LOAD %i as %i to control %i", load->_idx, cloned_load->_idx, catch_proj->_idx); - phase->igvn().register_new_node_with_optimizer(cloned_load); - return cloned_load; - } - - static Node* get_dominating_region(PhaseIdealLoop* phase, Node* node, Node* stop) { - Node* region = node; - while (!region->isa_Region()) { - Node *up = phase->idom(region); - assert(up != region, "Must not loop"); - assert(up != stop, "Must not find original control"); - region = up; - } - return region; - } - - // Clone this load to each catch block - static void call_catch_cleanup_one(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) { - bool trace = phase->C->directive()->ZTraceLoadBarriersOption; - phase->igvn().set_delay_transform(true); - - // Verify pre conditions - assert(ctrl->isa_Proj() && ctrl->in(0)->isa_Call(), "Must be a call proj"); - assert(ctrl->raw_out(0)->isa_Catch(), "Must be a catch"); ! if (ctrl->raw_out(0)->isa_Catch()->outcnt() == 1) { ! if (trace) tty->print_cr("Cleaning up catch: Skipping load %i, call with single catch", load->_idx); ! return; ! } ! ! // Process the loads successor nodes - if any is between ! // the call and the catch blocks, they need to be cloned to. ! // This is done recursively ! int outcnt = load->outcnt(); ! uint index = 0; ! for (int i = 0; i < outcnt; i++) { ! if (index < load->outcnt()) { ! Node *n = load->raw_out(index); ! assert(!n->is_LoadBarrier(), "Sanity"); ! if (!fixup_uses_in_catch(phase, ctrl, n)) { ! // if no successor was cloned, progress to next out. ! index++; ! } ! } ! } ! ! // Now all the loads uses has been cloned down ! // Only thing left is to clone the loads, but they must end up ! // first in the catch blocks. ! ! // We clone the loads oo the catch blocks only when needed. ! // An array is used to map the catch blocks to each lazily cloned load. ! // In that way no extra unnecessary loads are cloned. ! ! // Any use dominated by original block must have an phi and a region added ! ! Node* catch_node = ctrl->raw_out(0); ! int number_of_catch_projs = catch_node->outcnt(); ! Node** proj_to_load_mapping = NEW_RESOURCE_ARRAY(Node*, number_of_catch_projs); ! Copy::zero_to_bytes(proj_to_load_mapping, sizeof(Node*) * number_of_catch_projs); ! ! // The phi_map is used to keep track of where phis have already been inserted ! int phi_map_len = phase->C->unique(); ! Node** phi_map = NEW_RESOURCE_ARRAY(Node*, phi_map_len); ! Copy::zero_to_bytes(phi_map, sizeof(Node*) * phi_map_len); ! ! for (unsigned int i = 0; i < load->outcnt(); i++) { ! Node* load_use_control = NULL; ! Node* load_use = load->raw_out(i); ! ! if (phase->has_ctrl(load_use)) { ! load_use_control = phase->get_ctrl(load_use); ! } else { ! load_use_control = load_use->in(0); ! } ! assert(load_use_control != NULL, "sanity"); ! if (trace) tty->print_cr(" Handling use: %i, with control: %i", load_use->_idx, load_use_control->_idx); ! ! // Some times the loads use is a phi. For them we need to determine from which catch block ! // the use is defined. ! bool load_use_is_phi = false; ! unsigned int load_use_phi_index = 0; ! Node* phi_ctrl = NULL; ! if (load_use->is_Phi()) { ! // Find phi input that matches load ! for (unsigned int u = 1; u < load_use->req(); u++) { ! if (load_use->in(u) == load) { ! load_use_is_phi = true; ! load_use_phi_index = u; ! assert(load_use->in(0)->is_Region(), "Region or broken"); ! phi_ctrl = load_use->in(0)->in(u); ! assert(phi_ctrl->is_CFG(), "check"); ! assert(phi_ctrl != load, "check"); ! break; ! } ! } ! assert(load_use_is_phi, "must find"); ! assert(load_use_phi_index > 0, "sanity"); ! } ! ! // For each load use, see which catch projs dominates, create load clone lazily and reconnect ! bool found_dominating_catchproj = false; ! for (int c = 0; c < number_of_catch_projs; c++) { ! Node* catchproj = catch_node->raw_out(c); ! assert(catchproj != NULL && catchproj->isa_CatchProj(), "Sanity"); ! ! if (!phase->is_dominator(catchproj, load_use_control)) { ! if (load_use_is_phi && phase->is_dominator(catchproj, phi_ctrl)) { ! // The loads use is local to the catchproj. ! // fall out and replace load with catch-local load clone. ! } else { ! continue; } } - assert(!found_dominating_catchproj, "Max one should match"); - - // Clone loads to catch projs - Node* load_clone = proj_to_load_mapping[c]; - if (load_clone == NULL) { - load_clone = clone_load_to_catchproj(phase, load, catchproj); - proj_to_load_mapping[c] = load_clone; - } - phase->igvn().rehash_node_delayed(load_use); - - if (load_use_is_phi) { - // phis are special - the load is defined from a specific control flow - load_use->set_req(load_use_phi_index, load_clone); - } else { - // Multipe edges can be replaced at once - on calls for example - load_use->replace_edge(load, load_clone); - } - --i; // more than one edge can have been removed, but the next is in later iterations - - // We could break the for-loop after finding a dominating match. - // But keep iterating to catch any bad idom early. - found_dominating_catchproj = true; - } - - // We found no single catchproj that dominated the use - The use is at a point after - // where control flow from multiple catch projs have merged. We will have to create - // phi nodes before the use and tie the output from the cloned loads together. It - // can be a single phi or a number of chained phis, depending on control flow - if (!found_dominating_catchproj) { - - // Use phi-control if use is a phi - if (load_use_is_phi) { - load_use_control = phi_ctrl; - } - assert(phase->is_dominator(ctrl, load_use_control), "Common use but no dominator"); - - // Clone a load on all paths - for (int c = 0; c < number_of_catch_projs; c++) { - Node* catchproj = catch_node->raw_out(c); - Node* load_clone = proj_to_load_mapping[c]; - if (load_clone == NULL) { - load_clone = clone_load_to_catchproj(phase, load, catchproj); - proj_to_load_mapping[c] = load_clone; - } - } - - // Move up dominator tree from use until dom front is reached - Node* next_region = get_dominating_region(phase, load_use_control, ctrl); - while (phase->idom(next_region) != catch_node) { - next_region = phase->idom(next_region); - if (trace) tty->print_cr("Moving up idom to region ctrl %i", next_region->_idx); - } - assert(phase->is_dominator(catch_node, next_region), "Sanity"); - - // Create or reuse phi node that collect all cloned loads and feed it to the use. - Node* test_phi = phi_map[next_region->_idx]; - if ((test_phi != NULL) && test_phi->is_Phi()) { - // Reuse an already created phi - if (trace) tty->print_cr(" Using cached Phi %i on load_use %i", test_phi->_idx, load_use->_idx); - phase->igvn().rehash_node_delayed(load_use); - load_use->replace_edge(load, test_phi); - // Now this use is done - } else { - // Otherwise we need to create one or more phis - PhiNode* next_phi = new PhiNode(next_region, load->type()); - phi_map[next_region->_idx] = next_phi; // cache new phi - phase->igvn().rehash_node_delayed(load_use); - load_use->replace_edge(load, next_phi); - - int dominators_of_region = 0; - do { - // New phi, connect to region and add all loads as in. - Node* region = next_region; - assert(region->isa_Region() && region->req() > 2, "Catch dead region nodes"); - PhiNode* new_phi = next_phi; - - if (trace) tty->print_cr("Created Phi %i on load %i with control %i", new_phi->_idx, load->_idx, region->_idx); - - // Need to add all cloned loads to the phi, taking care that the right path is matched - dominators_of_region = 0; // reset for new region - for (unsigned int reg_i = 1; reg_i < region->req(); reg_i++) { - Node* region_pred = region->in(reg_i); - assert(region_pred->is_CFG(), "check"); - bool pred_has_dominator = false; - for (int c = 0; c < number_of_catch_projs; c++) { - Node* catchproj = catch_node->raw_out(c); - if (phase->is_dominator(catchproj, region_pred)) { - new_phi->set_req(reg_i, proj_to_load_mapping[c]); - if (trace) tty->print_cr(" - Phi in(%i) set to load %i", reg_i, proj_to_load_mapping[c]->_idx); - pred_has_dominator = true; - dominators_of_region++; - break; - } - } - - // Sometimes we need to chain several phis. - if (!pred_has_dominator) { - assert(dominators_of_region <= 1, "More than one region can't require extra phi"); - if (trace) tty->print_cr(" - Region %i pred %i not dominated by catch proj", region->_idx, region_pred->_idx); - // Continue search on on this region_pred - // - walk up to next region - // - create a new phi and connect to first new_phi - next_region = get_dominating_region(phase, region_pred, ctrl); - - // Lookup if there already is a phi, create a new otherwise - Node* test_phi = phi_map[next_region->_idx]; - if ((test_phi != NULL) && test_phi->is_Phi()) { - next_phi = test_phi->isa_Phi(); - dominators_of_region++; // record that a match was found and that we are done - if (trace) tty->print_cr(" Using cached phi Phi %i on control %i", next_phi->_idx, next_region->_idx); - } else { - next_phi = new PhiNode(next_region, load->type()); - phi_map[next_region->_idx] = next_phi; - } - new_phi->set_req(reg_i, next_phi); - } - } - - new_phi->set_req(0, region); - phase->igvn().register_new_node_with_optimizer(new_phi); - phase->set_ctrl(new_phi, region); - - assert(dominators_of_region != 0, "Must have found one this iteration"); - } while (dominators_of_region == 1); - } - --i; } - } // end of loop over uses - - assert(load->outcnt() == 0, "All uses should be handled"); - phase->igvn().remove_dead_node(load); - phase->C->print_method(PHASE_CALL_CATCH_CLEANUP, 4, load->_idx); - - // Now we should be home - phase->igvn().set_delay_transform(false); - } - - // Sort out the loads that are between a call ant its catch blocks - static void process_catch_cleanup_candidate(PhaseIdealLoop* phase, LoadNode* load) { - bool trace = phase->C->directive()->ZTraceLoadBarriersOption; - - Node* ctrl = phase->get_ctrl(load); - if (!ctrl->is_Proj() || (ctrl->in(0) == NULL) || !ctrl->in(0)->isa_Call()) { - return; } - - Node* catch_node = ctrl->isa_Proj()->raw_out(0); - if (catch_node->is_Catch()) { - if (catch_node->outcnt() > 1) { - call_catch_cleanup_one(phase, load, ctrl); - } else { - if (trace) tty->print_cr("Call catch cleanup with only one catch: load %i ", load->_idx); - } - } - } - - void ZBarrierSetC2::barrier_insertion_phase(Compile* C, PhaseIterGVN& igvn) const { - PhaseIdealLoop::optimize(igvn, LoopOptsZBarrierInsertion); - if (C->failing()) return; } ! bool ZBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { ! ! if (mode == LoopOptsZBarrierInsertion) { ! // First make sure all loads between call and catch are moved to the catch block ! clean_catch_blocks(phase); ! ! // Then expand barriers on all loads ! insert_load_barriers(phase); ! ! // Handle all Unsafe that need barriers. ! insert_barriers_on_unsafe(phase); ! ! phase->C->clear_major_progress(); ! return true; ! } else { ! return false; ! } ! } ! ! static bool can_simplify_cas(LoadStoreNode* node) { ! if (node->isa_LoadStoreConditional()) { ! Node *expected_in = node->as_LoadStoreConditional()->in(LoadStoreConditionalNode::ExpectedIn); ! return (expected_in->get_ptr_type() == TypePtr::NULL_PTR); ! } else { ! return false; ! } ! } ! ! static void insert_barrier_before_unsafe(PhaseIdealLoop* phase, LoadStoreNode* old_node) { ! ! Compile *C = phase->C; ! PhaseIterGVN &igvn = phase->igvn(); ! LoadStoreNode* zclone = NULL; ! bool is_weak = false; ! ! Node *in_ctrl = old_node->in(MemNode::Control); ! Node *in_mem = old_node->in(MemNode::Memory); ! Node *in_adr = old_node->in(MemNode::Address); ! Node *in_val = old_node->in(MemNode::ValueIn); ! const TypePtr *adr_type = old_node->adr_type(); ! const TypePtr* load_type = TypeOopPtr::BOTTOM; // The type for the load we are adding ! ! switch (old_node->Opcode()) { ! case Op_CompareAndExchangeP: { ! zclone = new ZCompareAndExchangePNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), ! adr_type, old_node->get_ptr_type(), ((CompareAndExchangeNode*)old_node)->order()); ! load_type = old_node->bottom_type()->is_ptr(); ! break; ! } ! case Op_WeakCompareAndSwapP: { ! if (can_simplify_cas(old_node)) { ! break; ! } ! is_weak = true; ! zclone = new ZWeakCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), ! ((CompareAndSwapNode*)old_node)->order()); ! adr_type = TypePtr::BOTTOM; ! break; ! } ! case Op_CompareAndSwapP: { ! if (can_simplify_cas(old_node)) { ! break; ! } ! zclone = new ZCompareAndSwapPNode(in_ctrl, in_mem, in_adr, in_val, old_node->in(LoadStoreConditionalNode::ExpectedIn), ! ((CompareAndSwapNode*)old_node)->order()); ! adr_type = TypePtr::BOTTOM; ! break; ! } ! case Op_GetAndSetP: { ! zclone = new ZGetAndSetPNode(in_ctrl, in_mem, in_adr, in_val, old_node->adr_type(), old_node->get_ptr_type()); ! load_type = old_node->bottom_type()->is_ptr(); ! break; ! } ! } ! if (zclone != NULL) { ! igvn.register_new_node_with_optimizer(zclone, old_node); ! ! // Make load ! LoadPNode *load = new LoadPNode(NULL, in_mem, in_adr, adr_type, load_type, MemNode::unordered, ! LoadNode::DependsOnlyOnTest); ! load_set_expanded_barrier(load); ! igvn.register_new_node_with_optimizer(load); ! igvn.replace_node(old_node, zclone); ! ! Node *barrier = new LoadBarrierNode(C, NULL, in_mem, load, in_adr, is_weak); ! Node *barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop); ! Node *barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control); ! ! igvn.register_new_node_with_optimizer(barrier); ! igvn.register_new_node_with_optimizer(barrier_val); ! igvn.register_new_node_with_optimizer(barrier_ctrl); ! ! // loop over all of in_ctrl usages and move to barrier_ctrl ! for (DUIterator_Last imin, i = in_ctrl->last_outs(imin); i >= imin; --i) { ! Node *use = in_ctrl->last_out(i); ! uint l; ! for (l = 0; use->in(l) != in_ctrl; l++) {} ! igvn.replace_input_of(use, l, barrier_ctrl); ! } ! ! load->set_req(MemNode::Control, in_ctrl); ! barrier->set_req(LoadBarrierNode::Control, in_ctrl); ! zclone->add_req(barrier_val); // add req as keep alive. ! ! C->print_method(PHASE_ADD_UNSAFE_BARRIER, 4, zclone->_idx); ! } ! } ! ! void ZBarrierSetC2::insert_barriers_on_unsafe(PhaseIdealLoop* phase) const { ! Compile *C = phase->C; ! PhaseIterGVN &igvn = phase->igvn(); ! uint new_ids = C->unique(); ! VectorSet visited(Thread::current()->resource_area()); ! GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL); ! nodeStack.push(C->root()); ! visited.test_set(C->root()->_idx); ! ! // Traverse all nodes, visit all unsafe ops that require a barrier ! while (nodeStack.length() > 0) { ! Node *n = nodeStack.pop(); ! ! bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup ! if (is_old_node) { ! if (n->is_LoadStore()) { ! LoadStoreNode* lsn = n->as_LoadStore(); ! if (lsn->has_barrier()) { ! BasicType bt = lsn->in(MemNode::Address)->bottom_type()->basic_type(); ! assert ((bt == T_OBJECT || bt == T_ARRAY), "Sanity test"); ! insert_barrier_before_unsafe(phase, lsn); ! } ! } ! } ! for (uint i = 0; i < n->len(); i++) { ! if (n->in(i)) { ! if (!visited.test_set(n->in(i)->_idx)) { ! nodeStack.push(n->in(i)); ! } ! } ! } ! } ! ! igvn.optimize(); ! C->print_method(PHASE_ADD_UNSAFE_BARRIER, 2); ! } ! ! // The purpose of ZBarrierSetC2::clean_catch_blocks is to prepare the IR for ! // splicing in load barrier nodes. ! // ! // The problem is that we might have instructions between a call and its catch nodes. ! // (This is usually handled in PhaseCFG:call_catch_cleanup, which clones mach nodes in ! // already scheduled blocks.) We can't have loads that require barriers there, ! // because we need to splice in new control flow, and that would violate the IR. ! // ! // clean_catch_blocks find all Loads that require a barrier and clone them and any ! // dependent instructions to each use. The loads must be in the beginning of the catch block ! // before any store. ! // ! // Sometimes the loads use will be at a place dominated by all catch blocks, then we need ! // a load in each catch block, and a Phi at the dominated use. ! ! void ZBarrierSetC2::clean_catch_blocks(PhaseIdealLoop* phase) const { ! ! Compile *C = phase->C; ! uint new_ids = C->unique(); ! PhaseIterGVN &igvn = phase->igvn(); ! VectorSet visited(Thread::current()->resource_area()); ! GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL); ! nodeStack.push(C->root()); ! visited.test_set(C->root()->_idx); ! ! // Traverse all nodes, visit all loads that require a barrier ! while(nodeStack.length() > 0) { ! Node *n = nodeStack.pop(); ! ! for (uint i = 0; i < n->len(); i++) { ! if (n->in(i)) { ! if (!visited.test_set(n->in(i)->_idx)) { ! nodeStack.push(n->in(i)); ! } ! } ! } ! ! bool is_old_node = (n->_idx < new_ids); // don't process nodes that were created during cleanup ! if (n->is_Load() && is_old_node) { ! LoadNode* load = n->isa_Load(); ! // only care about loads that will have a barrier ! if (load_require_barrier(load)) { ! process_catch_cleanup_candidate(phase, load); ! } ! } ! } ! ! C->print_method(PHASE_CALL_CATCH_CLEANUP, 2); ! } ! ! class DomDepthCompareClosure : public CompareClosure<LoadNode*> { ! PhaseIdealLoop* _phase; ! ! public: ! DomDepthCompareClosure(PhaseIdealLoop* phase) : _phase(phase) { } ! ! int do_compare(LoadNode* const &n1, LoadNode* const &n2) { ! int d1 = _phase->dom_depth(_phase->get_ctrl(n1)); ! int d2 = _phase->dom_depth(_phase->get_ctrl(n2)); ! if (d1 == d2) { ! // Compare index if the depth is the same, ensures all entries are unique. ! return n1->_idx - n2->_idx; ! } else { ! return d2 - d1; ! } ! } ! }; ! ! // Traverse graph and add all loadPs to list, sorted by dom depth ! void gather_loadnodes_sorted(PhaseIdealLoop* phase, GrowableArray<LoadNode*>* loadList) { ! ! VectorSet visited(Thread::current()->resource_area()); ! GrowableArray<Node *> nodeStack(Thread::current()->resource_area(), 0, 0, NULL); ! DomDepthCompareClosure ddcc(phase); ! ! nodeStack.push(phase->C->root()); ! while(nodeStack.length() > 0) { ! Node *n = nodeStack.pop(); ! if (visited.test(n->_idx)) { ! continue; ! } ! ! if (n->isa_Load()) { ! LoadNode *load = n->as_Load(); ! if (load_require_barrier(load)) { ! assert(phase->get_ctrl(load) != NULL, "sanity"); ! assert(phase->dom_depth(phase->get_ctrl(load)) != 0, "sanity"); ! loadList->insert_sorted(&ddcc, load); ! } ! } ! ! visited.set(n->_idx); ! for (uint i = 0; i < n->req(); i++) { ! if (n->in(i)) { ! if (!visited.test(n->in(i)->_idx)) { ! nodeStack.push(n->in(i)); ! } ! } ! } ! } ! } ! ! // Add LoadBarriers to all LoadPs ! void ZBarrierSetC2::insert_load_barriers(PhaseIdealLoop* phase) const { ! ! bool trace = phase->C->directive()->ZTraceLoadBarriersOption; ! GrowableArray<LoadNode *> loadList(Thread::current()->resource_area(), 0, 0, NULL); ! gather_loadnodes_sorted(phase, &loadList); ! ! PhaseIterGVN &igvn = phase->igvn(); ! int count = 0; ! ! for (GrowableArrayIterator<LoadNode *> loadIter = loadList.begin(); loadIter != loadList.end(); ++loadIter) { ! LoadNode *load = *loadIter; ! if (load_has_expanded_barrier(load)) { ! continue; ! } ! do { ! // Insert a barrier on a loadP ! // if another load is found that needs to be expanded first, retry on that one ! LoadNode* result = insert_one_loadbarrier(phase, load, phase->get_ctrl(load)); ! while (result != NULL) { ! result = insert_one_loadbarrier(phase, result, phase->get_ctrl(result)); } ! } while (!load_has_expanded_barrier(load)); } ! phase->C->print_method(PHASE_INSERT_BARRIER, 2); ! } ! ! void push_antidependent_stores(PhaseIdealLoop* phase, Node_Stack& nodestack, LoadNode* start_load) { ! // push all stores on the same mem, that can_alias ! // Any load found must be handled first ! PhaseIterGVN &igvn = phase->igvn(); ! int load_alias_idx = igvn.C->get_alias_index(start_load->adr_type()); ! ! Node *mem = start_load->in(1); ! for (DUIterator_Fast imax, u = mem->fast_outs(imax); u < imax; u++) { ! Node *mem_use = mem->fast_out(u); ! ! if (mem_use == start_load) continue; ! if (!mem_use->is_Store()) continue; ! if (!phase->has_ctrl(mem_use)) continue; ! if (phase->get_ctrl(mem_use) != phase->get_ctrl(start_load)) continue; ! ! // add any aliasing store in this block ! StoreNode *store = mem_use->isa_Store(); ! const TypePtr *adr_type = store->adr_type(); ! if (igvn.C->can_alias(adr_type, load_alias_idx)) { ! nodestack.push(store, 0); ! } ! } } ! LoadNode* ZBarrierSetC2::insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* start_load, Node* ctrl) const { ! bool trace = phase->C->directive()->ZTraceLoadBarriersOption; ! PhaseIterGVN &igvn = phase->igvn(); ! ! // Check for other loadPs at the same loop depth that is reachable by a DFS ! // - if found - return it. It needs to be inserted first ! // - otherwise proceed and insert barrier ! ! VectorSet visited(Thread::current()->resource_area()); ! Node_Stack nodestack(100); ! ! nodestack.push(start_load, 0); ! push_antidependent_stores(phase, nodestack, start_load); ! ! while(!nodestack.is_empty()) { ! Node* n = nodestack.node(); // peek ! nodestack.pop(); ! if (visited.test(n->_idx)) { ! continue; ! } ! ! if (n->is_Load() && n != start_load && load_require_barrier(n->as_Load()) && !load_has_expanded_barrier(n->as_Load())) { ! // Found another load that needs a barrier in the same block. Must expand later loads first. ! if (trace) tty->print_cr(" * Found LoadP %i on DFS", n->_idx); ! return n->as_Load(); // return node that should be expanded first ! } ! ! if (!phase->has_ctrl(n)) continue; ! if (phase->get_ctrl(n) != phase->get_ctrl(start_load)) continue; ! if (n->is_Phi()) continue; ! ! visited.set(n->_idx); ! // push all children ! for (DUIterator_Fast imax, ii = n->fast_outs(imax); ii < imax; ii++) { ! Node* c = n->fast_out(ii); ! if (c != NULL) { ! nodestack.push(c, 0); } ! } ! } ! ! insert_one_loadbarrier_inner(phase, start_load, ctrl, visited); ! return NULL; ! } ! ! void ZBarrierSetC2::insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited2) const { ! PhaseIterGVN &igvn = phase->igvn(); ! Compile* C = igvn.C; ! bool trace = C->directive()->ZTraceLoadBarriersOption; ! ! // create barrier ! Node* barrier = new LoadBarrierNode(C, NULL, load->in(LoadNode::Memory), NULL, load->in(LoadNode::Address), load_has_weak_barrier(load)); ! Node* barrier_val = new ProjNode(barrier, LoadBarrierNode::Oop); ! Node* barrier_ctrl = new ProjNode(barrier, LoadBarrierNode::Control); ! ! if (trace) tty->print_cr("Insert load %i with barrier: %i and ctrl : %i", load->_idx, barrier->_idx, ctrl->_idx); ! ! // Splice control ! // - insert barrier control diamond between loads ctrl and ctrl successor on path to block end. ! // - If control successor is a catch, step over to next. ! Node* ctrl_succ = NULL; ! for (DUIterator_Fast imax, j = ctrl->fast_outs(imax); j < imax; j++) { ! Node* tmp = ctrl->fast_out(j); ! ! // - CFG nodes is the ones we are going to splice (1 only!) ! // - Phi nodes will continue to hang from the region node! ! // - self loops should be skipped ! if (tmp->is_Phi() || tmp == ctrl) { ! continue; ! } ! ! if (tmp->is_CFG()) { ! assert(ctrl_succ == NULL, "There can be only one"); ! ctrl_succ = tmp; ! continue; ! } ! } ! ! // Now splice control ! assert(ctrl_succ != load, "sanity"); ! assert(ctrl_succ != NULL, "Broken IR"); ! bool found = false; ! for(uint k = 0; k < ctrl_succ->req(); k++) { ! if (ctrl_succ->in(k) == ctrl) { ! assert(!found, "sanity"); ! if (trace) tty->print_cr(" Move CFG ctrl_succ %i to barrier_ctrl", ctrl_succ->_idx); ! igvn.replace_input_of(ctrl_succ, k, barrier_ctrl); ! found = true; ! k--; ! } ! } ! ! // For all successors of ctrl - move all visited to become successors of barrier_ctrl instead ! for (DUIterator_Fast imax, r = ctrl->fast_outs(imax); r < imax; r++) { ! Node* tmp = ctrl->fast_out(r); ! if (visited2.test(tmp->_idx) && (tmp != load)) { ! if (trace) tty->print_cr(" Move ctrl_succ %i to barrier_ctrl", tmp->_idx); ! igvn.replace_input_of(tmp, 0, barrier_ctrl); ! --r; --imax; ! } ! } ! ! // Move the loads user to the barrier ! for (DUIterator_Fast imax, i = load->fast_outs(imax); i < imax; i++) { ! Node* u = load->fast_out(i); ! if (u->isa_LoadBarrier()) { ! continue; ! } ! // find correct input - replace with iterator? ! for(uint j = 0; j < u->req(); j++) { ! if (u->in(j) == load) { ! igvn.replace_input_of(u, j, barrier_val); ! --i; --imax; // Adjust the iterator of the *outer* loop ! break; // some nodes (calls) might have several uses from the same node } ! } } ! // Connect barrier to load and control ! barrier->set_req(LoadBarrierNode::Oop, load); ! barrier->set_req(LoadBarrierNode::Control, ctrl); ! ! igvn.rehash_node_delayed(load); ! igvn.register_new_node_with_optimizer(barrier); ! igvn.register_new_node_with_optimizer(barrier_val); ! igvn.register_new_node_with_optimizer(barrier_ctrl); ! load_set_expanded_barrier(load); ! ! C->print_method(PHASE_INSERT_BARRIER, 3, load->_idx); ! } ! ! // The bad_mask in the ThreadLocalData shouldn't have an anti-dep-check. ! // The bad_mask address if of type TypeRawPtr, but that will alias ! // InitializeNodes until the type system is expanded. ! bool ZBarrierSetC2::needs_anti_dependence_check(const Node* node) const { ! MachNode* mnode = node->as_Mach(); ! if (mnode != NULL) { ! intptr_t offset = 0; ! const TypePtr *adr_type2 = NULL; ! const Node* base = mnode->get_base_and_disp(offset, adr_type2); ! if ((base != NULL) && ! (base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadLocal) && ! (offset == in_bytes(ZThreadLocalData::address_bad_mask_offset()))) { ! return false; ! } ! } ! return true; } --- 1556,1644 ---- if (!n->in(LoadBarrierNode::Similar)->is_top()) { ResourceMark rm; Unique_Node_List wq; Node* other = n->in(LoadBarrierNode::Similar)->in(0); wq.push(n); + bool ok = true; + bool dom_found = false; for (uint next = 0; next < wq.size(); ++next) { ! Node *n = wq.at(next); ! assert(n->is_CFG(), ""); ! assert(!n->is_SafePoint(), ""); ! if (n == other) { continue; } ! if (n->is_Region()) { ! for (uint i = 1; i < n->req(); i++) { ! Node* m = n->in(i); if (m != NULL) { wq.push(m); } } } else { ! Node* m = n->in(0); if (m != NULL) { wq.push(m); } } } } ! if (ZVerifyLoadBarriers) { ! if ((n->is_Load() || n->is_LoadStore()) && n->bottom_type()->make_oopptr() != NULL) { ! visited.Clear(); ! bool found = look_for_barrier(n, post_parse, visited); ! if (!found) { ! n->dump(1); ! n->dump(-3); ! stringStream ss; ! C->method()->print_short_name(&ss); ! tty->print_cr("-%s-", ss.as_string()); ! assert(found, ""); } } } } } ! #endif ! bool ZBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const { ! switch (opcode) { ! case Op_LoadBarrierSlowReg: ! case Op_LoadBarrierWeakSlowReg: ! conn_graph->add_objload_to_connection_graph(n, delayed_worklist); ! return true; ! case Op_Proj: ! if (n->as_Proj()->_con != LoadBarrierNode::Oop || !n->in(0)->is_LoadBarrier()) { ! return false; } ! conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), delayed_worklist); ! return true; } ! return false; } ! bool ZBarrierSetC2::escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const { ! switch (opcode) { ! case Op_LoadBarrierSlowReg: ! case Op_LoadBarrierWeakSlowReg: ! if (gvn->type(n)->make_ptr() == NULL) { ! return false; } ! conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(MemNode::Address), NULL); ! return true; ! case Op_Proj: ! if (n->as_Proj()->_con != LoadBarrierNode::Oop || !n->in(0)->is_LoadBarrier()) { ! return false; } ! conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0)->in(LoadBarrierNode::Oop), NULL); ! return true; } ! return false; }
< prev index next >