< prev index next >

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

Print this page

        

*** 1014,1044 **** ctrl = new IfFalseNode(in_cset_fast_test_iff); phase->register_control(ctrl, loop, in_cset_fast_test_iff); } ! void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) { IdealLoopTree*loop = phase->get_loop(ctrl); const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst(); // The slow path stub consumes and produces raw memory in addition // to the existing memory edges Node* base = find_bottom_mem(ctrl, phase); MergeMemNode* mm = MergeMemNode::make(base); mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); phase->register_new_node(mm, ctrl); address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native) ! : CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier); const char* name = is_native ? "oop_load_from_native_barrier" : "load_reference_barrier"; Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM); call->init_req(TypeFunc::Control, ctrl); call->init_req(TypeFunc::I_O, phase->C->top()); call->init_req(TypeFunc::Memory, mm); call->init_req(TypeFunc::FramePtr, phase->C->top()); call->init_req(TypeFunc::ReturnAdr, phase->C->top()); call->init_req(TypeFunc::Parms, val); phase->register_control(call, loop, ctrl); ctrl = new ProjNode(call, TypeFunc::Control); phase->register_control(ctrl, loop, call); result_mem = new ProjNode(call, TypeFunc::Memory); phase->register_new_node(result_mem, call); --- 1014,1050 ---- ctrl = new IfFalseNode(in_cset_fast_test_iff); phase->register_control(ctrl, loop, in_cset_fast_test_iff); } ! void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase) { IdealLoopTree*loop = phase->get_loop(ctrl); const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr()->cast_to_nonconst(); // The slow path stub consumes and produces raw memory in addition // to the existing memory edges Node* base = find_bottom_mem(ctrl, phase); MergeMemNode* mm = MergeMemNode::make(base); mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); phase->register_new_node(mm, ctrl); + address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ? + CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup_narrow) : + CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_fixup); + address calladdr = is_native ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native) ! : target; const char* name = is_native ? "oop_load_from_native_barrier" : "load_reference_barrier"; Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM); + call->init_req(TypeFunc::Control, ctrl); call->init_req(TypeFunc::I_O, phase->C->top()); call->init_req(TypeFunc::Memory, mm); call->init_req(TypeFunc::FramePtr, phase->C->top()); call->init_req(TypeFunc::ReturnAdr, phase->C->top()); call->init_req(TypeFunc::Parms, val); + call->init_req(TypeFunc::Parms+1, load_addr); phase->register_control(call, loop, ctrl); ctrl = new ProjNode(call, TypeFunc::Control); phase->register_control(ctrl, loop, call); result_mem = new ProjNode(call, TypeFunc::Memory); phase->register_new_node(result_mem, call);
*** 1399,1409 **** Node* null_ctrl = NULL; assert(val->bottom_type()->make_oopptr(), "need oop"); assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); ! enum { _heap_stable = 1, _not_cset, _fwded, _evac_path, _null_path, PATH_LIMIT }; Node* region = new RegionNode(PATH_LIMIT); Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); // Stable path. --- 1405,1415 ---- Node* null_ctrl = NULL; assert(val->bottom_type()->make_oopptr(), "need oop"); assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); ! enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT }; Node* region = new RegionNode(PATH_LIMIT); Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); // Stable path.
*** 1449,1501 **** fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase); IfNode* iff = unc_ctrl->in(0)->as_If(); phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); } - Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(oopDesc::mark_offset_in_bytes())); - phase->register_new_node(addr, ctrl); - assert(new_val->bottom_type()->isa_oopptr(), "what else?"); - Node* markword = new LoadXNode(ctrl, raw_mem, addr, TypeRawPtr::BOTTOM, TypeX_X, MemNode::unordered); - phase->register_new_node(markword, ctrl); - - // Test if object is forwarded. This is the case if lowest two bits are set. - Node* masked = new AndXNode(markword, phase->igvn().MakeConX(markWord::lock_mask_in_place)); - phase->register_new_node(masked, ctrl); - Node* cmp = new CmpXNode(masked, phase->igvn().MakeConX(markWord::marked_value)); - phase->register_new_node(cmp, ctrl); - - // Only branch to LRB stub if object is not forwarded; otherwise reply with fwd ptr - Node* bol = new BoolNode(cmp, BoolTest::eq); // Equals 3 means it's forwarded - phase->register_new_node(bol, ctrl); - - IfNode* iff = new IfNode(ctrl, bol, PROB_LIKELY(0.999), COUNT_UNKNOWN); - phase->register_control(iff, loop, ctrl); - Node* if_fwd = new IfTrueNode(iff); - phase->register_control(if_fwd, loop, iff); - Node* if_not_fwd = new IfFalseNode(iff); - phase->register_control(if_not_fwd, loop, iff); - - // Decode forward pointer: since we already have the lowest bits, we can just subtract them - // from the mark word without the need for large immediate mask. - Node* masked2 = new SubXNode(markword, masked); - phase->register_new_node(masked2, if_fwd); - Node* fwdraw = new CastX2PNode(masked2); - fwdraw->init_req(0, if_fwd); - phase->register_new_node(fwdraw, if_fwd); - Node* fwd = new CheckCastPPNode(NULL, fwdraw, val->bottom_type()); - phase->register_new_node(fwd, if_fwd); - - // Wire up not-equal-path in slots 3. - region->init_req(_fwded, if_fwd); - val_phi->init_req(_fwded, fwd); - raw_mem_phi->init_req(_fwded, raw_mem); // Call lrb-stub and wire up that path in slots 4 Node* result_mem = NULL; ! ctrl = if_not_fwd; ! fwd = new_val; ! call_lrb_stub(ctrl, fwd, result_mem, raw_mem, lrb->is_native(), phase); region->init_req(_evac_path, ctrl); val_phi->init_req(_evac_path, fwd); raw_mem_phi->init_req(_evac_path, result_mem); phase->register_control(region, loop, heap_stable_iff); --- 1455,1502 ---- fix_null_check(unc, unc_ctrl, ctrl->in(0)->as_If()->proj_out(0), uses, phase); IfNode* iff = unc_ctrl->in(0)->as_If(); phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); } // Call lrb-stub and wire up that path in slots 4 Node* result_mem = NULL; ! ! Node* fwd = new_val; ! Node* addr; ! if (ShenandoahSelfFixing) { ! VectorSet visited(Thread::current()->resource_area()); ! addr = get_load_addr(phase, visited, lrb); ! } else { ! addr = phase->igvn().zerocon(T_OBJECT); ! } ! if (addr->Opcode() == Op_AddP) { ! Node* orig_base = addr->in(AddPNode::Base); ! Node* base = new CheckCastPPNode(ctrl, orig_base, orig_base->bottom_type(), true); ! phase->register_new_node(base, ctrl); ! if (addr->in(AddPNode::Base) == addr->in((AddPNode::Address))) { ! // Field access ! addr = addr->clone(); ! addr->set_req(AddPNode::Base, base); ! addr->set_req(AddPNode::Address, base); ! phase->register_new_node(addr, ctrl); ! } else { ! Node* addr2 = addr->in(AddPNode::Address); ! if (addr2->Opcode() == Op_AddP && addr2->in(AddPNode::Base) == addr2->in(AddPNode::Address) && ! addr2->in(AddPNode::Base) == orig_base) { ! addr2 = addr2->clone(); ! addr2->set_req(AddPNode::Base, base); ! addr2->set_req(AddPNode::Address, base); ! phase->register_new_node(addr2, ctrl); ! addr = addr->clone(); ! addr->set_req(AddPNode::Base, base); ! addr->set_req(AddPNode::Address, addr2); ! phase->register_new_node(addr, ctrl); ! } ! } ! } ! call_lrb_stub(ctrl, fwd, addr, result_mem, raw_mem, lrb->is_native(), phase); region->init_req(_evac_path, ctrl); val_phi->init_req(_evac_path, fwd); raw_mem_phi->init_req(_evac_path, result_mem); phase->register_control(region, loop, heap_stable_iff);
*** 1694,1703 **** --- 1695,1772 ---- } assert(state->enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced"); } + Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* in) { + if (visited.test_set(in->_idx)) { + return NULL; + } + switch (in->Opcode()) { + case Op_Proj: + return get_load_addr(phase, visited, in->in(0)); + case Op_CastPP: + case Op_CheckCastPP: + case Op_DecodeN: + case Op_EncodeP: + return get_load_addr(phase, visited, in->in(1)); + case Op_LoadN: + case Op_LoadP: + return in->in(MemNode::Address); + case Op_CompareAndExchangeN: + case Op_CompareAndExchangeP: + case Op_GetAndSetN: + case Op_GetAndSetP: + case Op_ShenandoahCompareAndExchangeP: + case Op_ShenandoahCompareAndExchangeN: + // Those instructions would just have stored a different + // value into the field. No use to attempt to fix it at this point. + return phase->igvn().zerocon(T_OBJECT); + case Op_CMoveP: + case Op_CMoveN: { + Node* t = get_load_addr(phase, visited, in->in(CMoveNode::IfTrue)); + Node* f = get_load_addr(phase, visited, in->in(CMoveNode::IfFalse)); + // Handle unambiguous cases: single address reported on both branches. + if (t != NULL && f == NULL) return t; + if (t == NULL && f != NULL) return f; + if (t != NULL && t == f) return t; + // Ambiguity. + return phase->igvn().zerocon(T_OBJECT); + } + case Op_Phi: { + Node* addr = NULL; + for (uint i = 1; i < in->req(); i++) { + Node* addr1 = get_load_addr(phase, visited, in->in(i)); + if (addr == NULL) { + addr = addr1; + } + if (addr != addr1) { + return phase->igvn().zerocon(T_OBJECT); + } + } + return addr; + } + case Op_ShenandoahLoadReferenceBarrier: + return get_load_addr(phase, visited, in->in(ShenandoahLoadReferenceBarrierNode::ValueIn)); + case Op_ShenandoahEnqueueBarrier: + return get_load_addr(phase, visited, in->in(1)); + case Op_CallDynamicJava: + case Op_CallLeaf: + case Op_CallStaticJava: + case Op_ConN: + case Op_ConP: + case Op_Parm: + return phase->igvn().zerocon(T_OBJECT); + default: + #ifdef ASSERT + fatal("Unknown node in get_load_addr: %s", NodeClassNames[in->Opcode()]); + #endif + return phase->igvn().zerocon(T_OBJECT); + } + + } + void ShenandoahBarrierC2Support::move_heap_stable_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { IdealLoopTree *loop = phase->get_loop(iff); Node* loop_head = loop->_head; Node* entry_c = loop_head->in(LoopNode::EntryControl);
< prev index next >