< prev index next >

src/hotspot/share/opto/arraycopynode.cpp

Print this page

        

*** 26,35 **** --- 26,36 ---- #include "gc/shared/barrierSet.hpp" #include "gc/shared/c2/barrierSetC2.hpp" #include "gc/shared/c2/cardTableBarrierSetC2.hpp" #include "opto/arraycopynode.hpp" #include "opto/graphKit.hpp" + #include "opto/valuetypenode.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/macros.hpp" ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard) : CallNode(arraycopy_type(), NULL, TypePtr::BOTTOM),
*** 110,123 **** return is_clonebasic() ? length->find_intptr_t_con(-1) : length->find_int_con(-1); } int ArrayCopyNode::get_count(PhaseGVN *phase) const { - Node* src = in(ArrayCopyNode::Src); - const Type* src_type = phase->type(src); - if (is_clonebasic()) { if (src_type->isa_instptr()) { const TypeInstPtr* inst_src = src_type->is_instptr(); ciInstanceKlass* ik = inst_src->klass()->as_instance_klass(); // ciInstanceKlass::nof_nonstatic_fields() doesn't take injected // fields into account. They are rare anyway so easier to simply --- 111,128 ---- return is_clonebasic() ? length->find_intptr_t_con(-1) : length->find_int_con(-1); } int ArrayCopyNode::get_count(PhaseGVN *phase) const { if (is_clonebasic()) { + Node* src = in(ArrayCopyNode::Src); + const Type* src_type = phase->type(src); + + if (src_type == Type::TOP) { + return -1; + } + if (src_type->isa_instptr()) { const TypeInstPtr* inst_src = src_type->is_instptr(); ciInstanceKlass* ik = inst_src->klass()->as_instance_klass(); // ciInstanceKlass::nof_nonstatic_fields() doesn't take injected // fields into account. They are rare anyway so easier to simply
*** 134,144 **** // cloning an array we'll do it element by element. If the // length input to ArrayCopyNode is constant, length of input // array must be too. assert((get_length_if_constant(phase) == -1) == !ary_src->size()->is_con() || ! phase->is_IterGVN(), "inconsistent"); if (ary_src->size()->is_con()) { return ary_src->size()->get_con(); } return -1; --- 139,150 ---- // cloning an array we'll do it element by element. If the // length input to ArrayCopyNode is constant, length of input // array must be too. assert((get_length_if_constant(phase) == -1) == !ary_src->size()->is_con() || ! (ValueArrayFlatten && ary_src->elem()->make_oopptr() != NULL && ary_src->elem()->make_oopptr()->can_be_value_type()) || ! phase->is_IterGVN() || phase->C->inlining_incrementally(), "inconsistent"); if (ary_src->size()->is_con()) { return ary_src->size()->get_con(); } return -1;
*** 266,277 **** return false; } BasicType src_elem = ary_src->klass()->as_array_klass()->element_type()->basic_type(); BasicType dest_elem = ary_dest->klass()->as_array_klass()->element_type()->basic_type(); ! if (src_elem == T_ARRAY) src_elem = T_OBJECT; ! if (dest_elem == T_ARRAY) dest_elem = T_OBJECT; if (src_elem != dest_elem || dest_elem == T_VOID) { // We don't know if arguments are arrays of the same type return false; } --- 272,289 ---- return false; } BasicType src_elem = ary_src->klass()->as_array_klass()->element_type()->basic_type(); BasicType dest_elem = ary_dest->klass()->as_array_klass()->element_type()->basic_type(); ! if (src_elem == T_ARRAY || ! (src_elem == T_VALUETYPE && ary_src->klass()->is_obj_array_klass())) { ! src_elem = T_OBJECT; ! } ! if (dest_elem == T_ARRAY || ! (dest_elem == T_VALUETYPE && ary_dest->klass()->is_obj_array_klass())) { ! dest_elem = T_OBJECT; ! } if (src_elem != dest_elem || dest_elem == T_VOID) { // We don't know if arguments are arrays of the same type return false; }
*** 287,296 **** --- 299,312 ---- base_src = src; base_dest = dest; uint shift = exact_log2(type2aelembytes(dest_elem)); + if (dest_elem == T_VALUETYPE) { + ciValueArrayKlass* vak = ary_src->klass()->as_value_array_klass(); + shift = vak->log2_element_size(); + } uint header = arrayOopDesc::base_offset_in_bytes(dest_elem); adr_src = src; adr_dest = dest;
*** 302,320 **** } Node* src_scale = phase->transform(new LShiftXNode(src_offset, phase->intcon(shift))); Node* dest_scale = phase->transform(new LShiftXNode(dest_offset, phase->intcon(shift))); adr_src = phase->transform(new AddPNode(base_src, adr_src, src_scale)); adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, dest_scale)); - adr_src = new AddPNode(base_src, adr_src, phase->MakeConX(header)); - adr_dest = new AddPNode(base_dest, adr_dest, phase->MakeConX(header)); - - adr_src = phase->transform(adr_src); - adr_dest = phase->transform(adr_dest); - copy_type = dest_elem; } else { assert(ary_src != NULL, "should be a clone"); assert(is_clonebasic(), "should be"); --- 318,333 ---- } Node* src_scale = phase->transform(new LShiftXNode(src_offset, phase->intcon(shift))); Node* dest_scale = phase->transform(new LShiftXNode(dest_offset, phase->intcon(shift))); + adr_src = phase->transform(new AddPNode(base_src, adr_src, phase->MakeConX(header))); + adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, phase->MakeConX(header))); + adr_src = phase->transform(new AddPNode(base_src, adr_src, src_scale)); adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, dest_scale)); copy_type = dest_elem; } else { assert(ary_src != NULL, "should be a clone"); assert(is_clonebasic(), "should be");
*** 325,336 **** base_src = src->in(AddPNode::Base); adr_dest = dest; base_dest = dest->in(AddPNode::Base); assert(phase->type(src->in(AddPNode::Offset))->is_intptr_t()->get_con() == phase->type(dest->in(AddPNode::Offset))->is_intptr_t()->get_con(), "same start offset?"); BasicType elem = ary_src->klass()->as_array_klass()->element_type()->basic_type(); ! if (elem == T_ARRAY) elem = T_OBJECT; BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); if (bs->array_copy_requires_gc_barriers(true, elem, true, BarrierSetC2::Optimization)) { return false; } --- 338,358 ---- base_src = src->in(AddPNode::Base); adr_dest = dest; base_dest = dest->in(AddPNode::Base); assert(phase->type(src->in(AddPNode::Offset))->is_intptr_t()->get_con() == phase->type(dest->in(AddPNode::Offset))->is_intptr_t()->get_con(), "same start offset?"); + + if (ary_src->elem()->make_oopptr() != NULL && + ary_src->elem()->make_oopptr()->can_be_value_type()) { + return false; + } + BasicType elem = ary_src->klass()->as_array_klass()->element_type()->basic_type(); ! if (elem == T_ARRAY || ! (elem == T_VALUETYPE && ary_src->klass()->is_obj_array_klass())) { ! elem = T_OBJECT; ! } BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); if (bs->array_copy_requires_gc_barriers(true, elem, true, BarrierSetC2::Optimization)) { return false; }
*** 346,459 **** value_type = ary_src->elem(); } return true; } ! const TypePtr* ArrayCopyNode::get_address_type(PhaseGVN *phase, Node* n) { const Type* at = phase->type(n); assert(at != Type::TOP, "unexpected type"); ! const TypePtr* atp = at->isa_ptr(); // adjust atp to be the correct array element address type ! atp = atp->add_offset(Type::OffsetBot); return atp; } ! void ArrayCopyNode::array_copy_test_overlap(PhaseGVN *phase, bool can_reshape, bool disjoint_bases, int count, Node*& forward_ctl, Node*& backward_ctl) { ! Node* ctl = in(TypeFunc::Control); if (!disjoint_bases && count > 1) { Node* src_offset = in(ArrayCopyNode::SrcPos); Node* dest_offset = in(ArrayCopyNode::DestPos); assert(src_offset != NULL && dest_offset != NULL, "should be"); ! Node* cmp = phase->transform(new CmpINode(src_offset, dest_offset)); ! Node *bol = phase->transform(new BoolNode(cmp, BoolTest::lt)); IfNode *iff = new IfNode(ctl, bol, PROB_FAIR, COUNT_UNKNOWN); ! phase->transform(iff); ! forward_ctl = phase->transform(new IfFalseNode(iff)); ! backward_ctl = phase->transform(new IfTrueNode(iff)); } else { ! forward_ctl = ctl; } } ! Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase, bool can_reshape, ! Node*& forward_ctl, ! MergeMemNode* mm, ! const TypePtr* atp_src, ! const TypePtr* atp_dest, Node* adr_src, Node* base_src, Node* adr_dest, Node* base_dest, BasicType copy_type, const Type* value_type, int count) { ! if (!forward_ctl->is_top()) { ! // copy forward ! mm = mm->clone()->as_MergeMem(); ! ! if (count > 0) { ! BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); ! Node* v = load(bs, phase, forward_ctl, mm, adr_src, atp_src, value_type, copy_type); ! store(bs, phase, forward_ctl, mm, adr_dest, atp_dest, v, value_type, copy_type); ! for (int i = 1; i < count; i++) { ! Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); ! Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); ! Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); ! v = load(bs, phase, forward_ctl, mm, next_src, atp_src, value_type, copy_type); ! store(bs, phase, forward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type); ! } ! } else if(can_reshape) { ! PhaseIterGVN* igvn = phase->is_IterGVN(); ! igvn->_worklist.push(adr_src); ! igvn->_worklist.push(adr_dest); ! } ! return mm; ! } ! return phase->C->top(); ! } ! ! Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase, ! bool can_reshape, ! Node*& backward_ctl, ! MergeMemNode* mm, ! const TypePtr* atp_src, ! const TypePtr* atp_dest, ! Node* adr_src, ! Node* base_src, ! Node* adr_dest, ! Node* base_dest, ! BasicType copy_type, ! const Type* value_type, ! int count) { ! if (!backward_ctl->is_top()) { // copy backward ! mm = mm->clone()->as_MergeMem(); ! ! BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); ! assert(copy_type != T_OBJECT || !bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization), "only tightly coupled allocations for object arrays"); if (count > 0) { ! for (int i = count-1; i >= 1; i--) { ! Node* off = phase->MakeConX(type2aelembytes(copy_type) * i); ! Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off)); ! Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off)); ! Node* v = load(bs, phase, backward_ctl, mm, next_src, atp_src, value_type, copy_type); ! store(bs, phase, backward_ctl, mm, next_dest, atp_dest, v, value_type, copy_type); } - Node* v = load(bs, phase, backward_ctl, mm, adr_src, atp_src, value_type, copy_type); - store(bs, phase, backward_ctl, mm, adr_dest, atp_dest, v, value_type, copy_type); } else if(can_reshape) { ! PhaseIterGVN* igvn = phase->is_IterGVN(); ! igvn->_worklist.push(adr_src); ! igvn->_worklist.push(adr_dest); } - return phase->transform(mm); } - return phase->C->top(); } bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape, Node* ctl, Node *mem) { if (can_reshape) { --- 368,510 ---- value_type = ary_src->elem(); } return true; } ! const TypeAryPtr* ArrayCopyNode::get_address_type(PhaseGVN *phase, Node* n) { const Type* at = phase->type(n); assert(at != Type::TOP, "unexpected type"); ! const TypeAryPtr* atp = at->is_aryptr(); // adjust atp to be the correct array element address type ! atp = atp->add_offset(Type::OffsetBot)->is_aryptr(); return atp; } ! void ArrayCopyNode::array_copy_test_overlap(GraphKit& kit, bool disjoint_bases, int count, Node*& backward_ctl) { ! Node* ctl = kit.control(); if (!disjoint_bases && count > 1) { + PhaseGVN& gvn = kit.gvn(); Node* src_offset = in(ArrayCopyNode::SrcPos); Node* dest_offset = in(ArrayCopyNode::DestPos); assert(src_offset != NULL && dest_offset != NULL, "should be"); ! Node* cmp = gvn.transform(new CmpINode(src_offset, dest_offset)); ! Node *bol = gvn.transform(new BoolNode(cmp, BoolTest::lt)); IfNode *iff = new IfNode(ctl, bol, PROB_FAIR, COUNT_UNKNOWN); ! gvn.transform(iff); ! kit.set_control(gvn.transform(new IfFalseNode(iff))); ! backward_ctl = gvn.transform(new IfTrueNode(iff)); ! } ! } ! ! void ArrayCopyNode::copy(GraphKit& kit, ! const TypeAryPtr* atp_src, ! const TypeAryPtr* atp_dest, ! int i, ! Node* base_src, ! Node* base_dest, ! Node* adr_src, ! Node* adr_dest, ! BasicType copy_type, ! const Type* value_type) { ! if (copy_type == T_VALUETYPE) { ! ciValueArrayKlass* vak = atp_src->klass()->as_value_array_klass(); ! ciValueKlass* vk = vak->element_klass()->as_value_klass(); ! for (int j = 0; j < vk->nof_nonstatic_fields(); j++) { ! ciField* field = vk->nonstatic_field_at(j); ! int off_in_vt = field->offset() - vk->first_field_offset(); ! Node* off = kit.MakeConX(off_in_vt + i * vak->element_byte_size()); ! ciType* ft = field->type(); ! BasicType bt = type2field[ft->basic_type()]; ! assert(!field->is_flattened(), "flattened field encountered"); ! if (bt == T_VALUETYPE) { ! bt = T_OBJECT; ! } ! const Type* rt = Type::get_const_type(ft); ! const TypePtr* adr_type = atp_src->with_field_offset(off_in_vt)->add_offset(Type::OffsetBot); ! Node* next_src = kit.gvn().transform(new AddPNode(base_src, adr_src, off)); ! Node* v = kit.make_load(kit.control(), next_src, rt, bt, adr_type, MemNode::unordered); ! ! Node* next_dest = kit.gvn().transform(new AddPNode(base_dest, adr_dest, off)); ! if (is_java_primitive(bt)) { ! kit.store_to_memory(kit.control(), next_dest, v, bt, adr_type, MemNode::unordered); ! } else { ! const TypeOopPtr* val_type = Type::get_const_type(ft)->is_oopptr(); ! kit.access_store_at(base_dest, next_dest, adr_type, v, ! val_type, bt, StoreNode::release_if_reference(T_OBJECT)); ! } ! } } else { ! Node* off = kit.MakeConX(type2aelembytes(copy_type) * i); ! Node* next_src = kit.gvn().transform(new AddPNode(base_src, adr_src, off)); ! Node* v = kit.make_load(kit.control(), next_src, value_type, copy_type, atp_src, MemNode::unordered); ! Node* next_dest = kit.gvn().transform(new AddPNode(base_dest, adr_dest, off)); ! BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); ! if (copy_type == T_OBJECT && (bs->array_copy_requires_gc_barriers(false, T_OBJECT, false, BarrierSetC2::Optimization))) { ! kit.access_store_at(base_dest, next_dest, atp_dest, v, ! value_type->make_ptr()->is_oopptr(), copy_type, ! StoreNode::release_if_reference(T_OBJECT)); ! } else { ! kit.store_to_memory(kit.control(), next_dest, v, copy_type, atp_dest, MemNode::unordered); ! } ! } ! } ! ! ! void ArrayCopyNode::array_copy_forward(GraphKit& kit, ! bool can_reshape, ! const TypeAryPtr* atp_src, ! const TypeAryPtr* atp_dest, ! Node* adr_src, ! Node* base_src, ! Node* adr_dest, ! Node* base_dest, ! BasicType copy_type, ! const Type* value_type, ! int count) { ! if (!kit.stopped()) { ! // copy forward ! if (count > 0) { ! for (int i = 0; i < count; i++) { ! copy(kit, atp_src, atp_dest, i, base_src, base_dest, adr_src, adr_dest, copy_type, value_type); ! } ! } else if(can_reshape) { ! PhaseGVN& gvn = kit.gvn(); ! assert(gvn.is_IterGVN(), ""); ! gvn.record_for_igvn(adr_src); ! gvn.record_for_igvn(adr_dest); ! } } } ! void ArrayCopyNode::array_copy_backward(GraphKit& kit, bool can_reshape, ! const TypeAryPtr* atp_src, ! const TypeAryPtr* atp_dest, Node* adr_src, Node* base_src, Node* adr_dest, Node* base_dest, BasicType copy_type, const Type* value_type, int count) { ! if (!kit.stopped()) { // copy backward ! PhaseGVN& gvn = kit.gvn(); if (count > 0) { ! for (int i = count-1; i >= 0; i--) { ! copy(kit, atp_src, atp_dest, i, base_src, base_dest, adr_src, adr_dest, copy_type, value_type); } } else if(can_reshape) { ! PhaseGVN& gvn = kit.gvn(); ! assert(gvn.is_IterGVN(), ""); ! gvn.record_for_igvn(adr_src); ! gvn.record_for_igvn(adr_dest); } } } bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape, Node* ctl, Node *mem) { if (can_reshape) {
*** 474,494 **** Node* out_ctl = proj_out(TypeFunc::Control); igvn->replace_node(out_ctl, ctl); } else { // replace fallthrough projections of the ArrayCopyNode by the // new memory, control and the input IO. ! CallProjections callprojs; ! extract_projections(&callprojs, true, false); ! if (callprojs.fallthrough_ioproj != NULL) { ! igvn->replace_node(callprojs.fallthrough_ioproj, in(TypeFunc::I_O)); } ! if (callprojs.fallthrough_memproj != NULL) { ! igvn->replace_node(callprojs.fallthrough_memproj, mem); } ! if (callprojs.fallthrough_catchproj != NULL) { ! igvn->replace_node(callprojs.fallthrough_catchproj, ctl); } // The ArrayCopyNode is not disconnected. It still has the // projections for the exception case. Replace current // ArrayCopyNode with a dummy new one with a top() control so --- 525,544 ---- Node* out_ctl = proj_out(TypeFunc::Control); igvn->replace_node(out_ctl, ctl); } else { // replace fallthrough projections of the ArrayCopyNode by the // new memory, control and the input IO. ! CallProjections* callprojs = extract_projections(true, false); ! if (callprojs->fallthrough_ioproj != NULL) { ! igvn->replace_node(callprojs->fallthrough_ioproj, in(TypeFunc::I_O)); } ! if (callprojs->fallthrough_memproj != NULL) { ! igvn->replace_node(callprojs->fallthrough_memproj, mem); } ! if (callprojs->fallthrough_catchproj != NULL) { ! igvn->replace_node(callprojs->fallthrough_catchproj, ctl); } // The ArrayCopyNode is not disconnected. It still has the // projections for the exception case. Replace current // ArrayCopyNode with a dummy new one with a top() control so
*** 499,519 **** remove_dead_region(phase, can_reshape); } } else { if (in(TypeFunc::Control) != ctl) { // we can't return new memory and control from Ideal at parse time assert(!is_clonebasic() || UseShenandoahGC, "added control for clone?"); phase->record_for_igvn(this); return false; } } return true; } Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) { ! if (remove_dead_region(phase, can_reshape)) return this; if (StressArrayCopyMacroNode && !can_reshape) { phase->record_for_igvn(this); return NULL; } --- 549,582 ---- remove_dead_region(phase, can_reshape); } } else { if (in(TypeFunc::Control) != ctl) { // we can't return new memory and control from Ideal at parse time + #ifdef ASSERT + Node* src = in(ArrayCopyNode::Src); + const Type* src_type = phase->type(src); + const TypeAryPtr* ary_src = src_type->isa_aryptr(); + BasicType elem = ary_src != NULL ? ary_src->klass()->as_array_klass()->element_type()->basic_type() : T_CONFLICT; + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); + assert(!is_clonebasic() || bs->array_copy_requires_gc_barriers(true, T_OBJECT, true, BarrierSetC2::Optimization) || + (ary_src != NULL && elem == T_VALUETYPE && ary_src->klass()->is_obj_array_klass()), "added control for clone?"); + #endif assert(!is_clonebasic() || UseShenandoahGC, "added control for clone?"); phase->record_for_igvn(this); return false; } } return true; } Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) { ! // Perform any generic optimizations first ! Node* result = SafePointNode::Ideal(phase, can_reshape); ! if (result != NULL) { ! return result; ! } if (StressArrayCopyMacroNode && !can_reshape) { phase->record_for_igvn(this); return NULL; }
*** 551,560 **** --- 614,634 ---- if (count < 0 || count > ArrayCopyLoadStoreMaxElem) { return NULL; } + Node* src = in(ArrayCopyNode::Src); + Node* dest = in(ArrayCopyNode::Dest); + const Type* src_type = phase->type(src); + const Type* dest_type = phase->type(dest); + + if (src_type->isa_aryptr() && dest_type->isa_instptr()) { + // clone used for load of unknown value type can't be optimized at + // this point + return NULL; + } + Node* mem = try_clone_instance(phase, can_reshape, count); if (mem != NULL) { return (mem == NodeSentinel) ? NULL : mem; }
*** 570,644 **** adr_src, base_src, adr_dest, base_dest, copy_type, value_type, disjoint_bases)) { return NULL; } ! Node* src = in(ArrayCopyNode::Src); ! Node* dest = in(ArrayCopyNode::Dest); ! const TypePtr* atp_src = get_address_type(phase, src); ! const TypePtr* atp_dest = get_address_type(phase, dest); ! ! Node *in_mem = in(TypeFunc::Memory); ! if (!in_mem->is_MergeMem()) { ! in_mem = MergeMemNode::make(in_mem); ! } ! if (can_reshape) { assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms"); phase->is_IterGVN()->set_delay_transform(true); } Node* backward_ctl = phase->C->top(); - Node* forward_ctl = phase->C->top(); - array_copy_test_overlap(phase, can_reshape, disjoint_bases, count, forward_ctl, backward_ctl); ! Node* forward_mem = array_copy_forward(phase, can_reshape, forward_ctl, ! in_mem->as_MergeMem(), ! atp_src, atp_dest, ! adr_src, base_src, adr_dest, base_dest, ! copy_type, value_type, count); ! ! Node* backward_mem = array_copy_backward(phase, can_reshape, backward_ctl, ! in_mem->as_MergeMem(), ! atp_src, atp_dest, ! adr_src, base_src, adr_dest, base_dest, ! copy_type, value_type, count); ! ! Node* ctl = NULL; ! if (!forward_ctl->is_top() && !backward_ctl->is_top()) { ! ctl = new RegionNode(3); ! ctl->init_req(1, forward_ctl); ! ctl->init_req(2, backward_ctl); ! ctl = phase->transform(ctl); ! MergeMemNode* forward_mm = forward_mem->as_MergeMem(); ! MergeMemNode* backward_mm = backward_mem->as_MergeMem(); ! for (MergeMemStream mms(forward_mm, backward_mm); mms.next_non_empty2(); ) { ! if (mms.memory() != mms.memory2()) { ! Node* phi = new PhiNode(ctl, Type::MEMORY, phase->C->get_adr_type(mms.alias_idx())); ! phi->init_req(1, mms.memory()); ! phi->init_req(2, mms.memory2()); ! phi = phase->transform(phi); ! mms.set_memory(phi); ! } ! } ! mem = forward_mem; ! } else if (!forward_ctl->is_top()) { ! ctl = forward_ctl; ! mem = forward_mem; } else { ! assert(!backward_ctl->is_top(), "no copy?"); ! ctl = backward_ctl; ! mem = backward_mem; } if (can_reshape) { assert(phase->is_IterGVN()->delay_transform(), "should be delaying transforms"); phase->is_IterGVN()->set_delay_transform(false); } ! if (!finish_transform(phase, can_reshape, ctl, mem)) { return NULL; } return mem; } --- 644,738 ---- adr_src, base_src, adr_dest, base_dest, copy_type, value_type, disjoint_bases)) { return NULL; } ! JVMState* new_jvms = NULL; ! SafePointNode* new_map = NULL; ! if (!is_clonebasic()) { ! new_jvms = jvms()->clone_shallow(phase->C); ! new_map = new SafePointNode(req(), new_jvms); ! for (uint i = TypeFunc::FramePtr; i < req(); i++) { ! new_map->init_req(i, in(i)); ! } ! new_jvms->set_map(new_map); ! } else { ! new_jvms = new (phase->C) JVMState(0); ! new_map = new SafePointNode(TypeFunc::Parms, new_jvms); ! new_jvms->set_map(new_map); ! } ! new_map->set_control(in(TypeFunc::Control)); ! new_map->set_memory(MergeMemNode::make(in(TypeFunc::Memory))); ! new_map->set_i_o(in(TypeFunc::I_O)); ! phase->record_for_igvn(new_map); ! ! const TypeAryPtr* atp_src = get_address_type(phase, src); ! const TypeAryPtr* atp_dest = get_address_type(phase, dest); ! uint alias_idx_src = phase->C->get_alias_index(atp_src); ! uint alias_idx_dest = phase->C->get_alias_index(atp_dest); if (can_reshape) { assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms"); phase->is_IterGVN()->set_delay_transform(true); } + GraphKit kit(new_jvms, phase); + + SafePointNode* backward_map = NULL; + SafePointNode* forward_map = NULL; Node* backward_ctl = phase->C->top(); ! array_copy_test_overlap(kit, disjoint_bases, count, backward_ctl); ! ! { ! PreserveJVMState pjvms(&kit); ! ! array_copy_forward(kit, can_reshape, ! atp_src, atp_dest, ! adr_src, base_src, adr_dest, base_dest, ! copy_type, value_type, count); ! ! forward_map = kit.stop(); ! } ! ! kit.set_control(backward_ctl); ! array_copy_backward(kit, can_reshape, ! atp_src, atp_dest, ! adr_src, base_src, adr_dest, base_dest, ! copy_type, value_type, count); ! ! backward_map = kit.stop(); ! ! if (!forward_map->control()->is_top() && !backward_map->control()->is_top()) { ! assert(forward_map->i_o() == backward_map->i_o(), "need a phi on IO?"); ! Node* ctl = new RegionNode(3); ! Node* mem = new PhiNode(ctl, Type::MEMORY, TypePtr::BOTTOM); ! kit.set_map(forward_map); ! ctl->init_req(1, kit.control()); ! mem->init_req(1, kit.reset_memory()); ! kit.set_map(backward_map); ! ctl->init_req(2, kit.control()); ! mem->init_req(2, kit.reset_memory()); ! kit.set_control(phase->transform(ctl)); ! kit.set_all_memory(phase->transform(mem)); ! } else if (!forward_map->control()->is_top()) { ! kit.set_map(forward_map); } else { ! assert(!backward_map->control()->is_top(), "no copy?"); ! kit.set_map(backward_map); } if (can_reshape) { assert(phase->is_IterGVN()->delay_transform(), "should be delaying transforms"); phase->is_IterGVN()->set_delay_transform(false); } ! mem = kit.map()->memory(); ! if (!finish_transform(phase, can_reshape, kit.control(), mem)) { ! if (!can_reshape) { ! phase->record_for_igvn(this); ! } return NULL; } return mem; }
*** 725,737 **** if (dest_pos_t == NULL || len_t == NULL || ary_t == NULL) { return !must_modify; } ! BasicType ary_elem = ary_t->klass()->as_array_klass()->element_type()->basic_type(); uint header = arrayOopDesc::base_offset_in_bytes(ary_elem); uint elemsize = type2aelembytes(ary_elem); jlong dest_pos_plus_len_lo = (((jlong)dest_pos_t->_lo) + len_t->_lo) * elemsize + header; jlong dest_pos_plus_len_hi = (((jlong)dest_pos_t->_hi) + len_t->_hi) * elemsize + header; jlong dest_pos_lo = ((jlong)dest_pos_t->_lo) * elemsize + header; jlong dest_pos_hi = ((jlong)dest_pos_t->_hi) * elemsize + header; --- 819,835 ---- if (dest_pos_t == NULL || len_t == NULL || ary_t == NULL) { return !must_modify; } ! ciArrayKlass* klass = ary_t->klass()->as_array_klass(); ! BasicType ary_elem = klass->element_type()->basic_type(); uint header = arrayOopDesc::base_offset_in_bytes(ary_elem); uint elemsize = type2aelembytes(ary_elem); + if (klass->is_value_array_klass()) { + elemsize = klass->as_value_array_klass()->element_byte_size(); + } jlong dest_pos_plus_len_lo = (((jlong)dest_pos_t->_lo) + len_t->_lo) * elemsize + header; jlong dest_pos_plus_len_hi = (((jlong)dest_pos_t->_hi) + len_t->_hi) * elemsize + header; jlong dest_pos_lo = ((jlong)dest_pos_t->_lo) * elemsize + header; jlong dest_pos_hi = ((jlong)dest_pos_t->_hi) * elemsize + header;
< prev index next >