< prev index next >

src/hotspot/share/opto/arraycopynode.cpp

Print this page

        

*** 25,37 **** --- 25,41 ---- #include "precompiled.hpp" #include "gc/shared/barrierSet.hpp" #include "gc/shared/c2/barrierSetC2.hpp" #include "gc/shared/c2/cardTableBarrierSetC2.hpp" #include "opto/arraycopynode.hpp" + #include "opto/castnode.hpp" #include "opto/graphKit.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/macros.hpp" + #if INCLUDE_SHENANDOAHGC + #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" + #endif ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard) : CallNode(arraycopy_type(), NULL, TypeRawPtr::BOTTOM), _alloc_tightly_coupled(alloc_tightly_coupled), _has_negative_length_guard(has_negative_length_guard),
*** 146,155 **** --- 150,205 ---- } return get_length_if_constant(phase); } + #if INCLUDE_SHENANDOAHGC + Node* ArrayCopyNode::shenandoah_add_storeval_barrier(PhaseGVN *phase, bool can_reshape, Node* v, MergeMemNode* mem, Node*& ctl) { + if (ShenandoahStoreValReadBarrier) { + RegionNode* region = new RegionNode(3); + const Type* v_t = phase->type(v); + Node* phi = new PhiNode(region, v_t->isa_oopptr() ? v_t->is_oopptr()->cast_to_nonconst() : v_t); + Node* cmp = phase->transform(new CmpPNode(v, phase->zerocon(T_OBJECT))); + Node* bol = phase->transform(new BoolNode(cmp, BoolTest::ne)); + IfNode* iff = new IfNode(ctl, bol, PROB_LIKELY_MAG(3), COUNT_UNKNOWN); + + phase->transform(iff); + if (can_reshape) { + phase->is_IterGVN()->_worklist.push(iff); + } else { + phase->record_for_igvn(iff); + } + + Node* null_true = phase->transform(new IfFalseNode(iff)); + Node* null_false = phase->transform(new IfTrueNode(iff)); + region->init_req(1, null_true); + region->init_req(2, null_false); + phi->init_req(1, phase->zerocon(T_OBJECT)); + Node* cast = new CastPPNode(v, phase->type(v)->join_speculative(TypePtr::NOTNULL)); + cast->set_req(0, null_false); + cast = phase->transform(cast); + Node* rb = phase->transform(new ShenandoahReadBarrierNode(null_false, phase->C->immutable_memory(), cast, false)); + phi->init_req(2, rb); + ctl = phase->transform(region); + return phase->transform(phi); + } + if (ShenandoahStoreValEnqueueBarrier) { + const TypePtr* adr_type = ShenandoahBarrierNode::brooks_pointer_type(phase->type(v)); + int alias = phase->C->get_alias_index(adr_type); + Node* wb = new ShenandoahWriteBarrierNode(phase->C, ctl, mem->memory_at(alias), v); + Node* wb_transformed = phase->transform(wb); + Node* enqueue = phase->transform(new ShenandoahEnqueueBarrierNode(wb_transformed)); + if (wb_transformed == wb) { + Node* proj = phase->transform(new ShenandoahWBMemProjNode(wb)); + mem->set_memory_at(alias, proj); + } + return enqueue; + } + return v; + } + #endif + Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) { if (!is_clonebasic()) { return NULL; }
*** 203,212 **** --- 253,267 ---- type = Type::get_const_basic_type(bt); } Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered); v = phase->transform(v); + #if INCLUDE_SHENANDOAHGC + if (UseShenandoahGC && bt == T_OBJECT) { + v = shenandoah_add_storeval_barrier(phase, can_reshape, v, mem, ctl); + } + #endif Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered); s = phase->transform(s); mem->set_memory_at(fieldidx, s); }
*** 256,266 **** return false; } BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); if (dest_elem == T_OBJECT && (!is_alloc_tightly_coupled() || ! bs->array_copy_requires_gc_barriers(T_OBJECT))) { // It's an object array copy but we can't emit the card marking // that is needed return false; } --- 311,321 ---- return false; } BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); if (dest_elem == T_OBJECT && (!is_alloc_tightly_coupled() || ! (bs->array_copy_requires_gc_barriers(T_OBJECT) SHENANDOAHGC_ONLY(&& !ShenandoahStoreValEnqueueBarrier)))) { // It's an object array copy but we can't emit the card marking // that is needed return false; }
*** 348,443 **** } } Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase, bool can_reshape, ! Node* forward_ctl, ! Node* start_mem_src, ! Node* start_mem_dest, const TypePtr* atp_src, const TypePtr* atp_dest, Node* adr_src, Node* base_src, Node* adr_dest, Node* base_dest, BasicType copy_type, const Type* value_type, int count) { - Node* mem = phase->C->top(); if (!forward_ctl->is_top()) { // copy forward ! mem = start_mem_dest; uint alias_idx_src = phase->C->get_alias_index(atp_src); uint alias_idx_dest = phase->C->get_alias_index(atp_dest); bool same_alias = (alias_idx_src == alias_idx_dest); if (count > 0) { Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered); v = phase->transform(v); mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered); mem = phase->transform(mem); for (int i = 1; i < count; i++) { Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered); v = phase->transform(v); mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered); mem = phase->transform(mem); } } else if(can_reshape) { PhaseIterGVN* igvn = phase->is_IterGVN(); igvn->_worklist.push(adr_src); igvn->_worklist.push(adr_dest); } } ! return mem; } Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase, bool can_reshape, ! Node* backward_ctl, ! Node* start_mem_src, ! Node* start_mem_dest, const TypePtr* atp_src, const TypePtr* atp_dest, Node* adr_src, Node* base_src, Node* adr_dest, Node* base_dest, BasicType copy_type, const Type* value_type, int count) { - Node* mem = phase->C->top(); if (!backward_ctl->is_top()) { // copy backward ! mem = start_mem_dest; uint alias_idx_src = phase->C->get_alias_index(atp_src); uint alias_idx_dest = phase->C->get_alias_index(atp_dest); bool same_alias = (alias_idx_src == alias_idx_dest); if (count > 0) { for (int i = count-1; i >= 1; i--) { Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered); v = phase->transform(v); mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered); mem = phase->transform(mem); } Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered); v = phase->transform(v); mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered); mem = phase->transform(mem); } else if(can_reshape) { PhaseIterGVN* igvn = phase->is_IterGVN(); igvn->_worklist.push(adr_src); igvn->_worklist.push(adr_dest); } } ! return mem; } bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape, Node* ctl, Node *mem) { if (can_reshape) { --- 403,526 ---- } } Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase, bool can_reshape, ! Node*& forward_ctl, ! MergeMemNode* mm, const TypePtr* atp_src, const TypePtr* atp_dest, Node* adr_src, Node* base_src, Node* adr_dest, Node* base_dest, BasicType copy_type, const Type* value_type, int count) { if (!forward_ctl->is_top()) { // copy forward ! mm = mm->clone()->as_MergeMem(); uint alias_idx_src = phase->C->get_alias_index(atp_src); uint alias_idx_dest = phase->C->get_alias_index(atp_dest); + Node *start_mem_src = mm->memory_at(alias_idx_src); + Node *start_mem_dest = mm->memory_at(alias_idx_dest); + Node* mem = start_mem_dest; bool same_alias = (alias_idx_src == alias_idx_dest); if (count > 0) { Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered); v = phase->transform(v); + #if INCLUDE_SHENANDOAHGC + if (UseShenandoahGC && copy_type == T_OBJECT) { + v = shenandoah_add_storeval_barrier(phase, can_reshape, v, mm, forward_ctl); + } + #endif mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered); mem = phase->transform(mem); for (int i = 1; i < count; i++) { Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); v = LoadNode::make(*phase, forward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered); v = phase->transform(v); + #if INCLUDE_SHENANDOAHGC + if (UseShenandoahGC && copy_type == T_OBJECT) { + v = shenandoah_add_storeval_barrier(phase, can_reshape, v, mm, forward_ctl); + } + #endif mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered); mem = phase->transform(mem); } + mm->set_memory_at(alias_idx_dest, mem); } else if(can_reshape) { PhaseIterGVN* igvn = phase->is_IterGVN(); igvn->_worklist.push(adr_src); igvn->_worklist.push(adr_dest); } + return mm; } ! return phase->C->top(); } Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase, bool can_reshape, ! Node*& backward_ctl, ! MergeMemNode* mm, const TypePtr* atp_src, const TypePtr* atp_dest, Node* adr_src, Node* base_src, Node* adr_dest, Node* base_dest, BasicType copy_type, const Type* value_type, int count) { if (!backward_ctl->is_top()) { // copy backward ! mm = mm->clone()->as_MergeMem(); uint alias_idx_src = phase->C->get_alias_index(atp_src); uint alias_idx_dest = phase->C->get_alias_index(atp_dest); + Node *start_mem_src = mm->memory_at(alias_idx_src); + Node *start_mem_dest = mm->memory_at(alias_idx_dest); + Node* mem = start_mem_dest; + + assert(copy_type != T_OBJECT SHENANDOAHGC_ONLY(|| ShenandoahStoreValEnqueueBarrier), "only tightly coupled allocations for object arrays"); bool same_alias = (alias_idx_src == alias_idx_dest); if (count > 0) { for (int i = count-1; i >= 1; i--) { Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, next_src, atp_src, value_type, copy_type, MemNode::unordered); v = phase->transform(v); + #if INCLUDE_SHENANDOAHGC + if (UseShenandoahGC && copy_type == T_OBJECT) { + v = shenandoah_add_storeval_barrier(phase, can_reshape, v, mm, backward_ctl); + } + #endif mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered); mem = phase->transform(mem); } Node* v = LoadNode::make(*phase, backward_ctl, same_alias ? mem : start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered); v = phase->transform(v); + #if INCLUDE_SHENANDOAHGC + if (UseShenandoahGC && copy_type == T_OBJECT) { + v = shenandoah_add_storeval_barrier(phase, can_reshape, v, mm, backward_ctl); + } + #endif mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered); mem = phase->transform(mem); + mm->set_memory_at(alias_idx_dest, mem); } else if(can_reshape) { PhaseIterGVN* igvn = phase->is_IterGVN(); igvn->_worklist.push(adr_src); igvn->_worklist.push(adr_dest); } + return phase->transform(mm); } ! return phase->C->top(); } bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape, Node* ctl, Node *mem) { if (can_reshape) {
*** 483,493 **** remove_dead_region(phase, can_reshape); } } else { if (in(TypeFunc::Control) != ctl) { // we can't return new memory and control from Ideal at parse time ! assert(!is_clonebasic(), "added control for clone?"); return false; } } return true; } --- 566,577 ---- remove_dead_region(phase, can_reshape); } } else { if (in(TypeFunc::Control) != ctl) { // we can't return new memory and control from Ideal at parse time ! assert(!is_clonebasic() || UseShenandoahGC, "added control for clone?"); ! phase->record_for_igvn(this); return false; } } return true; }
*** 557,575 **** Node* src = in(ArrayCopyNode::Src); Node* dest = in(ArrayCopyNode::Dest); const TypePtr* atp_src = get_address_type(phase, src); const TypePtr* atp_dest = get_address_type(phase, dest); - uint alias_idx_src = phase->C->get_alias_index(atp_src); - uint alias_idx_dest = phase->C->get_alias_index(atp_dest); Node *in_mem = in(TypeFunc::Memory); ! Node *start_mem_src = in_mem; ! Node *start_mem_dest = in_mem; ! if (in_mem->is_MergeMem()) { ! start_mem_src = in_mem->as_MergeMem()->memory_at(alias_idx_src); ! start_mem_dest = in_mem->as_MergeMem()->memory_at(alias_idx_dest); } if (can_reshape) { assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms"); --- 641,654 ---- Node* src = in(ArrayCopyNode::Src); Node* dest = in(ArrayCopyNode::Dest); const TypePtr* atp_src = get_address_type(phase, src); const TypePtr* atp_dest = get_address_type(phase, dest); Node *in_mem = in(TypeFunc::Memory); ! if (!in_mem->is_MergeMem()) { ! in_mem = MergeMemNode::make(in_mem); } if (can_reshape) { assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms");
*** 579,609 **** Node* backward_ctl = phase->C->top(); Node* forward_ctl = phase->C->top(); array_copy_test_overlap(phase, can_reshape, disjoint_bases, count, forward_ctl, backward_ctl); Node* forward_mem = array_copy_forward(phase, can_reshape, forward_ctl, ! start_mem_src, start_mem_dest, atp_src, atp_dest, adr_src, base_src, adr_dest, base_dest, copy_type, value_type, count); Node* backward_mem = array_copy_backward(phase, can_reshape, backward_ctl, ! start_mem_src, start_mem_dest, atp_src, atp_dest, adr_src, base_src, adr_dest, base_dest, copy_type, value_type, count); Node* ctl = NULL; if (!forward_ctl->is_top() && !backward_ctl->is_top()) { ctl = new RegionNode(3); - mem = new PhiNode(ctl, Type::MEMORY, atp_dest); ctl->init_req(1, forward_ctl); - mem->init_req(1, forward_mem); ctl->init_req(2, backward_ctl); - mem->init_req(2, backward_mem); ctl = phase->transform(ctl); ! mem = phase->transform(mem); } else if (!forward_ctl->is_top()) { ctl = forward_ctl; mem = forward_mem; } else { assert(!backward_ctl->is_top(), "no copy?"); --- 658,696 ---- Node* backward_ctl = phase->C->top(); Node* forward_ctl = phase->C->top(); array_copy_test_overlap(phase, can_reshape, disjoint_bases, count, forward_ctl, backward_ctl); Node* forward_mem = array_copy_forward(phase, can_reshape, forward_ctl, ! in_mem->as_MergeMem(), atp_src, atp_dest, adr_src, base_src, adr_dest, base_dest, copy_type, value_type, count); Node* backward_mem = array_copy_backward(phase, can_reshape, backward_ctl, ! in_mem->as_MergeMem(), atp_src, atp_dest, adr_src, base_src, adr_dest, base_dest, copy_type, value_type, count); Node* ctl = NULL; if (!forward_ctl->is_top() && !backward_ctl->is_top()) { ctl = new RegionNode(3); ctl->init_req(1, forward_ctl); ctl->init_req(2, backward_ctl); ctl = phase->transform(ctl); ! MergeMemNode* forward_mm = forward_mem->as_MergeMem(); ! MergeMemNode* backward_mm = backward_mem->as_MergeMem(); ! for (MergeMemStream mms(forward_mm, backward_mm); mms.next_non_empty2(); ) { ! if (mms.memory() != mms.memory2()) { ! Node* phi = new PhiNode(ctl, Type::MEMORY, phase->C->get_adr_type(mms.alias_idx())); ! phi->init_req(1, mms.memory()); ! phi->init_req(2, mms.memory2()); ! phi = phase->transform(phi); ! mms.set_memory(phi); ! } ! } ! mem = forward_mem; } else if (!forward_ctl->is_top()) { ctl = forward_ctl; mem = forward_mem; } else { assert(!backward_ctl->is_top(), "no copy?");
*** 614,627 **** if (can_reshape) { assert(phase->is_IterGVN()->delay_transform(), "should be delaying transforms"); phase->is_IterGVN()->set_delay_transform(false); } - MergeMemNode* out_mem = MergeMemNode::make(in_mem); - out_mem->set_memory_at(alias_idx_dest, mem); - mem = out_mem; - if (!finish_transform(phase, can_reshape, ctl, mem)) { return NULL; } return mem; --- 701,710 ----
< prev index next >