1 /* 2 * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "ci/ciFlatArrayKlass.hpp" 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shared/c2/barrierSetC2.hpp" 29 #include "gc/shared/c2/cardTableBarrierSetC2.hpp" 30 #include "gc/shared/gc_globals.hpp" 31 #include "opto/arraycopynode.hpp" 32 #include "opto/graphKit.hpp" 33 #include "opto/inlinetypenode.hpp" 34 #include "runtime/sharedRuntime.hpp" 35 #include "utilities/macros.hpp" 36 #include "utilities/powerOfTwo.hpp" 37 38 ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard) 39 : CallNode(arraycopy_type(), nullptr, TypePtr::BOTTOM), 40 _kind(None), 41 _alloc_tightly_coupled(alloc_tightly_coupled), 42 _has_negative_length_guard(has_negative_length_guard), 43 _arguments_validated(false), 44 _src_type(TypeOopPtr::BOTTOM), 45 _dest_type(TypeOopPtr::BOTTOM) { 46 init_class_id(Class_ArrayCopy); 47 init_flags(Flag_is_macro); 48 C->add_macro_node(this); 49 } 50 51 uint ArrayCopyNode::size_of() const { return sizeof(*this); } 52 53 ArrayCopyNode* ArrayCopyNode::make(GraphKit* kit, bool may_throw, 54 Node* src, Node* src_offset, 55 Node* dest, Node* dest_offset, 56 Node* length, 57 bool alloc_tightly_coupled, 58 bool has_negative_length_guard, 59 Node* src_klass, Node* dest_klass, 60 Node* src_length, Node* dest_length) { 61 62 ArrayCopyNode* ac = new ArrayCopyNode(kit->C, alloc_tightly_coupled, has_negative_length_guard); 63 kit->set_predefined_input_for_runtime_call(ac); 64 65 ac->init_req(ArrayCopyNode::Src, src); 66 ac->init_req(ArrayCopyNode::SrcPos, src_offset); 67 ac->init_req(ArrayCopyNode::Dest, dest); 68 ac->init_req(ArrayCopyNode::DestPos, dest_offset); 69 ac->init_req(ArrayCopyNode::Length, length); 70 ac->init_req(ArrayCopyNode::SrcLen, src_length); 71 ac->init_req(ArrayCopyNode::DestLen, dest_length); 72 ac->init_req(ArrayCopyNode::SrcKlass, src_klass); 73 ac->init_req(ArrayCopyNode::DestKlass, dest_klass); 74 75 if (may_throw) { 76 ac->set_req(TypeFunc::I_O , kit->i_o()); 77 kit->add_safepoint_edges(ac, false); 78 } 79 80 return ac; 81 } 82 83 void ArrayCopyNode::connect_outputs(GraphKit* kit, bool deoptimize_on_exception) { 84 kit->set_all_memory_call(this, true); 85 kit->set_control(kit->gvn().transform(new ProjNode(this,TypeFunc::Control))); 86 kit->set_i_o(kit->gvn().transform(new ProjNode(this, TypeFunc::I_O))); 87 kit->make_slow_call_ex(this, kit->env()->Throwable_klass(), true, deoptimize_on_exception); 88 kit->set_all_memory_call(this); 89 } 90 91 #ifndef PRODUCT 92 const char* ArrayCopyNode::_kind_names[] = {"arraycopy", "arraycopy, validated arguments", "clone", "oop array clone", "CopyOf", "CopyOfRange"}; 93 94 void ArrayCopyNode::dump_spec(outputStream *st) const { 95 CallNode::dump_spec(st); 96 st->print(" (%s%s)", _kind_names[_kind], _alloc_tightly_coupled ? ", tightly coupled allocation" : ""); 97 } 98 99 void ArrayCopyNode::dump_compact_spec(outputStream* st) const { 100 st->print("%s%s", _kind_names[_kind], _alloc_tightly_coupled ? ",tight" : ""); 101 } 102 #endif 103 104 intptr_t ArrayCopyNode::get_length_if_constant(PhaseGVN *phase) const { 105 // check that length is constant 106 Node* length = in(ArrayCopyNode::Length); 107 const Type* length_type = phase->type(length); 108 109 if (length_type == Type::TOP) { 110 return -1; 111 } 112 113 assert(is_clonebasic() || is_arraycopy() || is_copyof() || is_copyofrange(), "unexpected array copy type"); 114 115 return is_clonebasic() ? length->find_intptr_t_con(-1) : length->find_int_con(-1); 116 } 117 118 int ArrayCopyNode::get_count(PhaseGVN *phase) const { 119 if (is_clonebasic()) { 120 Node* src = in(ArrayCopyNode::Src); 121 const Type* src_type = phase->type(src); 122 123 if (src_type == Type::TOP) { 124 return -1; 125 } 126 127 if (src_type->isa_instptr()) { 128 const TypeInstPtr* inst_src = src_type->is_instptr(); 129 ciInstanceKlass* ik = inst_src->instance_klass(); 130 // ciInstanceKlass::nof_nonstatic_fields() doesn't take injected 131 // fields into account. They are rare anyway so easier to simply 132 // skip instances with injected fields. 133 if ((!inst_src->klass_is_exact() && (ik->is_interface() || ik->has_subklass())) || ik->has_injected_fields()) { 134 return -1; 135 } 136 int nb_fields = ik->nof_nonstatic_fields(); 137 return nb_fields; 138 } else { 139 const TypeAryPtr* ary_src = src_type->isa_aryptr(); 140 assert (ary_src != nullptr, "not an array or instance?"); 141 // clone passes a length as a rounded number of longs. If we're 142 // cloning an array we'll do it element by element. If the 143 // length of the input array is constant, ArrayCopyNode::Length 144 // must be too. Note that the opposite does not need to hold, 145 // because different input array lengths (e.g. int arrays with 146 // 3 or 4 elements) might lead to the same length input 147 // (e.g. 2 double-words). 148 assert(!ary_src->size()->is_con() || (get_length_if_constant(phase) >= 0) || 149 (UseFlatArray && ary_src->elem()->make_oopptr() != nullptr && ary_src->elem()->make_oopptr()->can_be_inline_type()) || 150 phase->is_IterGVN() || phase->C->inlining_incrementally() || StressReflectiveCode, "inconsistent"); 151 if (ary_src->size()->is_con()) { 152 return ary_src->size()->get_con(); 153 } 154 return -1; 155 } 156 } 157 158 return get_length_if_constant(phase); 159 } 160 161 Node* ArrayCopyNode::load(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, const Type *type, BasicType bt) { 162 DecoratorSet decorators = C2_READ_ACCESS | C2_CONTROL_DEPENDENT_LOAD | IN_HEAP | C2_ARRAY_COPY; 163 C2AccessValuePtr addr(adr, adr_type); 164 C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr); 165 Node* res = bs->load_at(access, type); 166 ctl = access.ctl(); 167 return res; 168 } 169 170 void ArrayCopyNode::store(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, Node* val, const Type *type, BasicType bt) { 171 DecoratorSet decorators = C2_WRITE_ACCESS | IN_HEAP | C2_ARRAY_COPY; 172 if (is_alloc_tightly_coupled()) { 173 decorators |= C2_TIGHTLY_COUPLED_ALLOC; 174 } 175 C2AccessValuePtr addr(adr, adr_type); 176 C2AccessValue value(val, type); 177 C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr); 178 bs->store_at(access, value); 179 ctl = access.ctl(); 180 } 181 182 183 Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) { 184 if (!is_clonebasic()) { 185 return nullptr; 186 } 187 188 Node* base_src = in(ArrayCopyNode::Src); 189 Node* base_dest = in(ArrayCopyNode::Dest); 190 Node* ctl = in(TypeFunc::Control); 191 Node* in_mem = in(TypeFunc::Memory); 192 193 const Type* src_type = phase->type(base_src); 194 const TypeInstPtr* inst_src = src_type->isa_instptr(); 195 if (inst_src == nullptr) { 196 return nullptr; 197 } 198 199 MergeMemNode* mem = phase->transform(MergeMemNode::make(in_mem))->as_MergeMem(); 200 phase->record_for_igvn(mem); 201 if (can_reshape) { 202 phase->is_IterGVN()->_worklist.push(mem); 203 } 204 205 206 ciInstanceKlass* ik = inst_src->instance_klass(); 207 208 if (!inst_src->klass_is_exact()) { 209 assert(!ik->is_interface(), "inconsistent klass hierarchy"); 210 if (ik->has_subklass()) { 211 // Concurrent class loading. 212 // Fail fast and return NodeSentinel to indicate that the transform failed. 213 return NodeSentinel; 214 } else { 215 phase->C->dependencies()->assert_leaf_type(ik); 216 } 217 } 218 219 assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields"); 220 221 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 222 for (int i = 0; i < count; i++) { 223 ciField* field = ik->nonstatic_field_at(i); 224 const TypePtr* adr_type = phase->C->alias_type(field)->adr_type(); 225 Node* off = phase->MakeConX(field->offset_in_bytes()); 226 Node* next_src = phase->transform(new AddPNode(base_src,base_src,off)); 227 Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off)); 228 BasicType bt = field->layout_type(); 229 230 const Type *type; 231 if (bt == T_OBJECT) { 232 if (!field->type()->is_loaded()) { 233 type = TypeInstPtr::BOTTOM; 234 } else { 235 ciType* field_klass = field->type(); 236 type = TypeOopPtr::make_from_klass(field_klass->as_klass()); 237 } 238 } else { 239 type = Type::get_const_basic_type(bt); 240 } 241 242 Node* v = load(bs, phase, ctl, mem, next_src, adr_type, type, bt); 243 store(bs, phase, ctl, mem, next_dest, adr_type, v, type, bt); 244 } 245 246 if (!finish_transform(phase, can_reshape, ctl, mem)) { 247 // Return NodeSentinel to indicate that the transform failed 248 return NodeSentinel; 249 } 250 251 return mem; 252 } 253 254 bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape, 255 Node*& adr_src, 256 Node*& base_src, 257 Node*& adr_dest, 258 Node*& base_dest, 259 BasicType& copy_type, 260 const Type*& value_type, 261 bool& disjoint_bases) { 262 base_src = in(ArrayCopyNode::Src); 263 base_dest = in(ArrayCopyNode::Dest); 264 const Type* src_type = phase->type(base_src); 265 const TypeAryPtr* ary_src = src_type->isa_aryptr(); 266 267 Node* src_offset = in(ArrayCopyNode::SrcPos); 268 Node* dest_offset = in(ArrayCopyNode::DestPos); 269 270 if (is_arraycopy() || is_copyofrange() || is_copyof()) { 271 const Type* dest_type = phase->type(base_dest); 272 const TypeAryPtr* ary_dest = dest_type->isa_aryptr(); 273 274 // newly allocated object is guaranteed to not overlap with source object 275 disjoint_bases = is_alloc_tightly_coupled(); 276 if (ary_src == nullptr || ary_src->elem() == Type::BOTTOM || 277 ary_dest == nullptr || ary_dest->elem() == Type::BOTTOM) { 278 // We don't know if arguments are arrays 279 return false; 280 } 281 282 BasicType src_elem = ary_src->elem()->array_element_basic_type(); 283 BasicType dest_elem = ary_dest->elem()->array_element_basic_type(); 284 if (is_reference_type(src_elem, true)) src_elem = T_OBJECT; 285 if (is_reference_type(dest_elem, true)) dest_elem = T_OBJECT; 286 287 if (src_elem != dest_elem || ary_src->is_flat() != ary_dest->is_flat() || dest_elem == T_VOID) { 288 // We don't know if arguments are arrays of the same type 289 return false; 290 } 291 292 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 293 if ((!ary_dest->is_flat() && bs->array_copy_requires_gc_barriers(is_alloc_tightly_coupled(), dest_elem, false, false, BarrierSetC2::Optimization)) || 294 (ary_dest->is_flat() && ary_src->elem()->inline_klass()->contains_oops() && 295 bs->array_copy_requires_gc_barriers(is_alloc_tightly_coupled(), T_OBJECT, false, false, BarrierSetC2::Optimization))) { 296 // It's an object array copy but we can't emit the card marking that is needed 297 return false; 298 } 299 300 value_type = ary_src->elem(); 301 302 uint shift = exact_log2(type2aelembytes(dest_elem)); 303 if (ary_dest->is_flat()) { 304 shift = ary_src->flat_log_elem_size(); 305 } 306 uint header = arrayOopDesc::base_offset_in_bytes(dest_elem); 307 308 src_offset = Compile::conv_I2X_index(phase, src_offset, ary_src->size()); 309 if (src_offset->is_top()) { 310 // Offset is out of bounds (the ArrayCopyNode will be removed) 311 return false; 312 } 313 dest_offset = Compile::conv_I2X_index(phase, dest_offset, ary_dest->size()); 314 if (dest_offset->is_top()) { 315 // Offset is out of bounds (the ArrayCopyNode will be removed) 316 if (can_reshape) { 317 // record src_offset, so it can be deleted later (if it is dead) 318 phase->is_IterGVN()->_worklist.push(src_offset); 319 } 320 return false; 321 } 322 323 Node* hook = new Node(1); 324 hook->init_req(0, dest_offset); 325 326 Node* src_scale = phase->transform(new LShiftXNode(src_offset, phase->intcon(shift))); 327 328 hook->destruct(phase); 329 330 Node* dest_scale = phase->transform(new LShiftXNode(dest_offset, phase->intcon(shift))); 331 332 adr_src = phase->transform(new AddPNode(base_src, base_src, src_scale)); 333 adr_dest = phase->transform(new AddPNode(base_dest, base_dest, dest_scale)); 334 335 adr_src = phase->transform(new AddPNode(base_src, adr_src, phase->MakeConX(header))); 336 adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, phase->MakeConX(header))); 337 338 copy_type = dest_elem; 339 } else { 340 assert(ary_src != nullptr, "should be a clone"); 341 assert(is_clonebasic(), "should be"); 342 343 disjoint_bases = true; 344 345 if (ary_src->elem()->make_oopptr() != nullptr && 346 ary_src->elem()->make_oopptr()->can_be_inline_type()) { 347 return false; 348 } 349 350 BasicType elem = ary_src->isa_aryptr()->elem()->array_element_basic_type(); 351 if (is_reference_type(elem, true)) { 352 elem = T_OBJECT; 353 } 354 355 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 356 if ((!ary_src->is_flat() && bs->array_copy_requires_gc_barriers(true, elem, true, is_clone_inst(), BarrierSetC2::Optimization)) || 357 (ary_src->is_flat() && ary_src->elem()->inline_klass()->contains_oops() && 358 bs->array_copy_requires_gc_barriers(true, T_OBJECT, true, is_clone_inst(), BarrierSetC2::Optimization))) { 359 // It's an object array copy but we can't emit the card marking that is needed 360 return false; 361 } 362 363 adr_src = phase->transform(new AddPNode(base_src, base_src, src_offset)); 364 adr_dest = phase->transform(new AddPNode(base_dest, base_dest, dest_offset)); 365 366 // The address is offsetted to an aligned address where a raw copy would start. 367 // If the clone copy is decomposed into load-stores - the address is adjusted to 368 // point at where the array starts. 369 const Type* toff = phase->type(src_offset); 370 int offset = toff->isa_long() ? (int) toff->is_long()->get_con() : (int) toff->is_int()->get_con(); 371 int diff = arrayOopDesc::base_offset_in_bytes(elem) - offset; 372 assert(diff >= 0, "clone should not start after 1st array element"); 373 if (diff > 0) { 374 adr_src = phase->transform(new AddPNode(base_src, adr_src, phase->MakeConX(diff))); 375 adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, phase->MakeConX(diff))); 376 } 377 copy_type = elem; 378 value_type = ary_src->elem(); 379 } 380 return true; 381 } 382 383 const TypeAryPtr* ArrayCopyNode::get_address_type(PhaseGVN* phase, const TypePtr* atp, Node* n) { 384 if (atp == TypeOopPtr::BOTTOM) { 385 atp = phase->type(n)->isa_ptr(); 386 } 387 // adjust atp to be the correct array element address type 388 return atp->add_offset(Type::OffsetBot)->is_aryptr(); 389 } 390 391 void ArrayCopyNode::array_copy_test_overlap(GraphKit& kit, bool disjoint_bases, int count, Node*& backward_ctl) { 392 Node* ctl = kit.control(); 393 if (!disjoint_bases && count > 1) { 394 PhaseGVN& gvn = kit.gvn(); 395 Node* src_offset = in(ArrayCopyNode::SrcPos); 396 Node* dest_offset = in(ArrayCopyNode::DestPos); 397 assert(src_offset != nullptr && dest_offset != nullptr, "should be"); 398 Node* cmp = gvn.transform(new CmpINode(src_offset, dest_offset)); 399 Node *bol = gvn.transform(new BoolNode(cmp, BoolTest::lt)); 400 IfNode *iff = new IfNode(ctl, bol, PROB_FAIR, COUNT_UNKNOWN); 401 402 gvn.transform(iff); 403 404 kit.set_control(gvn.transform(new IfFalseNode(iff))); 405 backward_ctl = gvn.transform(new IfTrueNode(iff)); 406 } 407 } 408 409 void ArrayCopyNode::copy(GraphKit& kit, 410 const TypeAryPtr* atp_src, 411 const TypeAryPtr* atp_dest, 412 int i, 413 Node* base_src, 414 Node* base_dest, 415 Node* adr_src, 416 Node* adr_dest, 417 BasicType copy_type, 418 const Type* value_type) { 419 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 420 Node* ctl = kit.control(); 421 if (atp_dest->is_flat()) { 422 ciInlineKlass* vk = atp_src->elem()->inline_klass(); 423 for (int j = 0; j < vk->nof_nonstatic_fields(); j++) { 424 ciField* field = vk->nonstatic_field_at(j); 425 int off_in_vt = field->offset_in_bytes() - vk->first_field_offset(); 426 Node* off = kit.MakeConX(off_in_vt + i * atp_src->flat_elem_size()); 427 ciType* ft = field->type(); 428 BasicType bt = type2field[ft->basic_type()]; 429 assert(!field->is_flat(), "flat field encountered"); 430 const Type* rt = Type::get_const_type(ft); 431 const TypePtr* adr_type = atp_src->with_field_offset(off_in_vt)->add_offset(Type::OffsetBot); 432 assert(!bs->array_copy_requires_gc_barriers(is_alloc_tightly_coupled(), bt, false, false, BarrierSetC2::Optimization), "GC barriers required"); 433 Node* next_src = kit.gvn().transform(new AddPNode(base_src, adr_src, off)); 434 Node* next_dest = kit.gvn().transform(new AddPNode(base_dest, adr_dest, off)); 435 Node* v = load(bs, &kit.gvn(), ctl, kit.merged_memory(), next_src, adr_type, rt, bt); 436 store(bs, &kit.gvn(), ctl, kit.merged_memory(), next_dest, adr_type, v, rt, bt); 437 } 438 } else { 439 Node* off = kit.MakeConX(type2aelembytes(copy_type) * i); 440 Node* next_src = kit.gvn().transform(new AddPNode(base_src, adr_src, off)); 441 Node* next_dest = kit.gvn().transform(new AddPNode(base_dest, adr_dest, off)); 442 Node* v = load(bs, &kit.gvn(), ctl, kit.merged_memory(), next_src, atp_src, value_type, copy_type); 443 store(bs, &kit.gvn(), ctl, kit.merged_memory(), next_dest, atp_dest, v, value_type, copy_type); 444 } 445 kit.set_control(ctl); 446 } 447 448 449 void ArrayCopyNode::array_copy_forward(GraphKit& kit, 450 bool can_reshape, 451 const TypeAryPtr* atp_src, 452 const TypeAryPtr* atp_dest, 453 Node* adr_src, 454 Node* base_src, 455 Node* adr_dest, 456 Node* base_dest, 457 BasicType copy_type, 458 const Type* value_type, 459 int count) { 460 if (!kit.stopped()) { 461 // copy forward 462 if (count > 0) { 463 for (int i = 0; i < count; i++) { 464 copy(kit, atp_src, atp_dest, i, base_src, base_dest, adr_src, adr_dest, copy_type, value_type); 465 } 466 } else if (can_reshape) { 467 PhaseGVN& gvn = kit.gvn(); 468 assert(gvn.is_IterGVN(), ""); 469 gvn.record_for_igvn(adr_src); 470 gvn.record_for_igvn(adr_dest); 471 } 472 } 473 } 474 475 void ArrayCopyNode::array_copy_backward(GraphKit& kit, 476 bool can_reshape, 477 const TypeAryPtr* atp_src, 478 const TypeAryPtr* atp_dest, 479 Node* adr_src, 480 Node* base_src, 481 Node* adr_dest, 482 Node* base_dest, 483 BasicType copy_type, 484 const Type* value_type, 485 int count) { 486 if (!kit.stopped()) { 487 // copy backward 488 PhaseGVN& gvn = kit.gvn(); 489 490 if (count > 0) { 491 for (int i = count-1; i >= 0; i--) { 492 copy(kit, atp_src, atp_dest, i, base_src, base_dest, adr_src, adr_dest, copy_type, value_type); 493 } 494 } else if(can_reshape) { 495 PhaseGVN& gvn = kit.gvn(); 496 assert(gvn.is_IterGVN(), ""); 497 gvn.record_for_igvn(adr_src); 498 gvn.record_for_igvn(adr_dest); 499 } 500 } 501 } 502 503 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape, 504 Node* ctl, Node *mem) { 505 if (can_reshape) { 506 PhaseIterGVN* igvn = phase->is_IterGVN(); 507 igvn->set_delay_transform(false); 508 if (is_clonebasic()) { 509 Node* out_mem = proj_out(TypeFunc::Memory); 510 511 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 512 if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() || 513 out_mem->raw_out(0)->outcnt() != 1 || !out_mem->raw_out(0)->raw_out(0)->is_MemBar()) { 514 assert(bs->array_copy_requires_gc_barriers(true, T_OBJECT, true, is_clone_inst(), BarrierSetC2::Optimization), "can only happen with card marking"); 515 return false; 516 } 517 518 igvn->replace_node(out_mem->raw_out(0), mem); 519 520 Node* out_ctl = proj_out(TypeFunc::Control); 521 igvn->replace_node(out_ctl, ctl); 522 } else { 523 // replace fallthrough projections of the ArrayCopyNode by the 524 // new memory, control and the input IO. 525 CallProjections* callprojs = extract_projections(true, false); 526 527 if (callprojs->fallthrough_ioproj != nullptr) { 528 igvn->replace_node(callprojs->fallthrough_ioproj, in(TypeFunc::I_O)); 529 } 530 if (callprojs->fallthrough_memproj != nullptr) { 531 igvn->replace_node(callprojs->fallthrough_memproj, mem); 532 } 533 if (callprojs->fallthrough_catchproj != nullptr) { 534 igvn->replace_node(callprojs->fallthrough_catchproj, ctl); 535 } 536 537 // The ArrayCopyNode is not disconnected. It still has the 538 // projections for the exception case. Replace current 539 // ArrayCopyNode with a dummy new one with a top() control so 540 // that this part of the graph stays consistent but is 541 // eventually removed. 542 543 set_req(0, phase->C->top()); 544 remove_dead_region(phase, can_reshape); 545 } 546 } else { 547 if (in(TypeFunc::Control) != ctl) { 548 // we can't return new memory and control from Ideal at parse time 549 assert(!is_clonebasic() || UseShenandoahGC, "added control for clone?"); 550 phase->record_for_igvn(this); 551 return false; 552 } 553 } 554 return true; 555 } 556 557 558 Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) { 559 // Perform any generic optimizations first 560 Node* result = SafePointNode::Ideal(phase, can_reshape); 561 if (result != nullptr) { 562 return result; 563 } 564 565 if (StressArrayCopyMacroNode && !can_reshape) { 566 phase->record_for_igvn(this); 567 return nullptr; 568 } 569 570 // See if it's a small array copy and we can inline it as 571 // loads/stores 572 // Here we can only do: 573 // - arraycopy if all arguments were validated before and we don't 574 // need card marking 575 // - clone for which we don't need to do card marking 576 577 if (!is_clonebasic() && !is_arraycopy_validated() && 578 !is_copyofrange_validated() && !is_copyof_validated()) { 579 return nullptr; 580 } 581 582 assert(in(TypeFunc::Control) != nullptr && 583 in(TypeFunc::Memory) != nullptr && 584 in(ArrayCopyNode::Src) != nullptr && 585 in(ArrayCopyNode::Dest) != nullptr && 586 in(ArrayCopyNode::Length) != nullptr && 587 in(ArrayCopyNode::SrcPos) != nullptr && 588 in(ArrayCopyNode::DestPos) != nullptr, "broken inputs"); 589 590 if (in(TypeFunc::Control)->is_top() || 591 in(TypeFunc::Memory)->is_top() || 592 phase->type(in(ArrayCopyNode::Src)) == Type::TOP || 593 phase->type(in(ArrayCopyNode::Dest)) == Type::TOP || 594 (in(ArrayCopyNode::SrcPos) != nullptr && in(ArrayCopyNode::SrcPos)->is_top()) || 595 (in(ArrayCopyNode::DestPos) != nullptr && in(ArrayCopyNode::DestPos)->is_top())) { 596 return nullptr; 597 } 598 599 int count = get_count(phase); 600 601 if (count < 0 || count > ArrayCopyLoadStoreMaxElem) { 602 return nullptr; 603 } 604 605 Node* src = in(ArrayCopyNode::Src); 606 Node* dest = in(ArrayCopyNode::Dest); 607 const Type* src_type = phase->type(src); 608 const Type* dest_type = phase->type(dest); 609 610 if (src_type->isa_aryptr() && dest_type->isa_instptr()) { 611 // clone used for load of unknown inline type can't be optimized at 612 // this point 613 return nullptr; 614 } 615 616 Node* mem = try_clone_instance(phase, can_reshape, count); 617 if (mem != nullptr) { 618 return (mem == NodeSentinel) ? nullptr : mem; 619 } 620 621 Node* adr_src = nullptr; 622 Node* base_src = nullptr; 623 Node* adr_dest = nullptr; 624 Node* base_dest = nullptr; 625 BasicType copy_type = T_ILLEGAL; 626 const Type* value_type = nullptr; 627 bool disjoint_bases = false; 628 629 if (!prepare_array_copy(phase, can_reshape, 630 adr_src, base_src, adr_dest, base_dest, 631 copy_type, value_type, disjoint_bases)) { 632 assert(adr_src == nullptr, "no node can be left behind"); 633 assert(adr_dest == nullptr, "no node can be left behind"); 634 return nullptr; 635 } 636 637 JVMState* new_jvms = nullptr; 638 SafePointNode* new_map = nullptr; 639 if (!is_clonebasic()) { 640 new_jvms = jvms()->clone_shallow(phase->C); 641 new_map = new SafePointNode(req(), new_jvms); 642 for (uint i = TypeFunc::FramePtr; i < req(); i++) { 643 new_map->init_req(i, in(i)); 644 } 645 new_jvms->set_map(new_map); 646 } else { 647 new_jvms = new (phase->C) JVMState(0); 648 new_map = new SafePointNode(TypeFunc::Parms, new_jvms); 649 new_jvms->set_map(new_map); 650 } 651 new_map->set_control(in(TypeFunc::Control)); 652 new_map->set_memory(MergeMemNode::make(in(TypeFunc::Memory))); 653 new_map->set_i_o(in(TypeFunc::I_O)); 654 phase->record_for_igvn(new_map); 655 656 const TypeAryPtr* atp_src = get_address_type(phase, _src_type, src); 657 const TypeAryPtr* atp_dest = get_address_type(phase, _dest_type, dest); 658 659 if (can_reshape) { 660 assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms"); 661 phase->is_IterGVN()->set_delay_transform(true); 662 } 663 664 GraphKit kit(new_jvms, phase); 665 666 SafePointNode* backward_map = nullptr; 667 SafePointNode* forward_map = nullptr; 668 Node* backward_ctl = phase->C->top(); 669 670 array_copy_test_overlap(kit, disjoint_bases, count, backward_ctl); 671 672 { 673 PreserveJVMState pjvms(&kit); 674 675 array_copy_forward(kit, can_reshape, 676 atp_src, atp_dest, 677 adr_src, base_src, adr_dest, base_dest, 678 copy_type, value_type, count); 679 680 forward_map = kit.stop(); 681 } 682 683 kit.set_control(backward_ctl); 684 array_copy_backward(kit, can_reshape, 685 atp_src, atp_dest, 686 adr_src, base_src, adr_dest, base_dest, 687 copy_type, value_type, count); 688 689 backward_map = kit.stop(); 690 691 if (!forward_map->control()->is_top() && !backward_map->control()->is_top()) { 692 assert(forward_map->i_o() == backward_map->i_o(), "need a phi on IO?"); 693 Node* ctl = new RegionNode(3); 694 Node* mem = new PhiNode(ctl, Type::MEMORY, TypePtr::BOTTOM); 695 kit.set_map(forward_map); 696 ctl->init_req(1, kit.control()); 697 mem->init_req(1, kit.reset_memory()); 698 kit.set_map(backward_map); 699 ctl->init_req(2, kit.control()); 700 mem->init_req(2, kit.reset_memory()); 701 kit.set_control(phase->transform(ctl)); 702 kit.set_all_memory(phase->transform(mem)); 703 } else if (!forward_map->control()->is_top()) { 704 kit.set_map(forward_map); 705 } else { 706 assert(!backward_map->control()->is_top(), "no copy?"); 707 kit.set_map(backward_map); 708 } 709 710 if (can_reshape) { 711 assert(phase->is_IterGVN()->delay_transform(), "should be delaying transforms"); 712 phase->is_IterGVN()->set_delay_transform(false); 713 } 714 715 mem = kit.map()->memory(); 716 if (!finish_transform(phase, can_reshape, kit.control(), mem)) { 717 if (!can_reshape) { 718 phase->record_for_igvn(this); 719 } else { 720 // put in worklist, so that if it happens to be dead it is removed 721 phase->is_IterGVN()->_worklist.push(mem); 722 } 723 return nullptr; 724 } 725 726 return mem; 727 } 728 729 bool ArrayCopyNode::may_modify(const TypeOopPtr* t_oop, PhaseValues* phase) { 730 Node* dest = in(ArrayCopyNode::Dest); 731 if (dest->is_top()) { 732 return false; 733 } 734 const TypeOopPtr* dest_t = phase->type(dest)->is_oopptr(); 735 assert(!dest_t->is_known_instance() || _dest_type->is_known_instance(), "result of EA not recorded"); 736 assert(in(ArrayCopyNode::Src)->is_top() || !phase->type(in(ArrayCopyNode::Src))->is_oopptr()->is_known_instance() || 737 _src_type->is_known_instance(), "result of EA not recorded"); 738 739 if (_dest_type != TypeOopPtr::BOTTOM || t_oop->is_known_instance()) { 740 assert(_dest_type == TypeOopPtr::BOTTOM || _dest_type->is_known_instance(), "result of EA is known instance"); 741 return t_oop->instance_id() == _dest_type->instance_id(); 742 } 743 744 return CallNode::may_modify_arraycopy_helper(dest_t, t_oop, phase); 745 } 746 747 bool ArrayCopyNode::may_modify_helper(const TypeOopPtr* t_oop, Node* n, PhaseValues* phase, CallNode*& call) { 748 if (n != nullptr && 749 n->is_Call() && 750 n->as_Call()->may_modify(t_oop, phase) && 751 (n->as_Call()->is_ArrayCopy() || n->as_Call()->is_call_to_arraycopystub())) { 752 call = n->as_Call(); 753 return true; 754 } 755 return false; 756 } 757 758 bool ArrayCopyNode::may_modify(const TypeOopPtr* t_oop, MemBarNode* mb, PhaseValues* phase, ArrayCopyNode*& ac) { 759 Node* c = mb->in(0); 760 761 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 762 // step over g1 gc barrier if we're at e.g. a clone with ReduceInitialCardMarks off 763 c = bs->step_over_gc_barrier(c); 764 765 CallNode* call = nullptr; 766 guarantee(c != nullptr, "step_over_gc_barrier failed, there must be something to step to."); 767 if (c->is_Region()) { 768 for (uint i = 1; i < c->req(); i++) { 769 if (c->in(i) != nullptr) { 770 Node* n = c->in(i)->in(0); 771 if (may_modify_helper(t_oop, n, phase, call)) { 772 ac = call->isa_ArrayCopy(); 773 assert(c == mb->in(0), "only for clone"); 774 return true; 775 } 776 } 777 } 778 } else if (may_modify_helper(t_oop, c->in(0), phase, call)) { 779 ac = call->isa_ArrayCopy(); 780 #ifdef ASSERT 781 bool use_ReduceInitialCardMarks = BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) && 782 static_cast<CardTableBarrierSetC2*>(bs)->use_ReduceInitialCardMarks(); 783 assert(c == mb->in(0) || (ac != nullptr && ac->is_clonebasic() && !use_ReduceInitialCardMarks), "only for clone"); 784 #endif 785 return true; 786 } else if (mb->trailing_partial_array_copy()) { 787 return true; 788 } 789 790 return false; 791 } 792 793 // Does this array copy modify offsets between offset_lo and offset_hi 794 // in the destination array 795 // if must_modify is false, return true if the copy could write 796 // between offset_lo and offset_hi 797 // if must_modify is true, return true if the copy is guaranteed to 798 // write between offset_lo and offset_hi 799 bool ArrayCopyNode::modifies(intptr_t offset_lo, intptr_t offset_hi, PhaseValues* phase, bool must_modify) const { 800 assert(_kind == ArrayCopy || _kind == CopyOf || _kind == CopyOfRange, "only for real array copies"); 801 802 Node* dest = in(Dest); 803 Node* dest_pos = in(DestPos); 804 Node* len = in(Length); 805 806 const TypeInt *dest_pos_t = phase->type(dest_pos)->isa_int(); 807 const TypeInt *len_t = phase->type(len)->isa_int(); 808 const TypeAryPtr* ary_t = phase->type(dest)->isa_aryptr(); 809 810 if (dest_pos_t == nullptr || len_t == nullptr || ary_t == nullptr) { 811 return !must_modify; 812 } 813 814 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type(); 815 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT; 816 817 uint header = arrayOopDesc::base_offset_in_bytes(ary_elem); 818 uint elemsize = ary_t->is_flat() ? ary_t->flat_elem_size() : type2aelembytes(ary_elem); 819 820 jlong dest_pos_plus_len_lo = (((jlong)dest_pos_t->_lo) + len_t->_lo) * elemsize + header; 821 jlong dest_pos_plus_len_hi = (((jlong)dest_pos_t->_hi) + len_t->_hi) * elemsize + header; 822 jlong dest_pos_lo = ((jlong)dest_pos_t->_lo) * elemsize + header; 823 jlong dest_pos_hi = ((jlong)dest_pos_t->_hi) * elemsize + header; 824 825 if (must_modify) { 826 if (offset_lo >= dest_pos_hi && offset_hi < dest_pos_plus_len_lo) { 827 return true; 828 } 829 } else { 830 if (offset_hi >= dest_pos_lo && offset_lo < dest_pos_plus_len_hi) { 831 return true; 832 } 833 } 834 return false; 835 } 836 837 // As an optimization, choose optimum vector size for copy length known at compile time. 838 int ArrayCopyNode::get_partial_inline_vector_lane_count(BasicType type, int const_len) { 839 int lane_count = ArrayOperationPartialInlineSize/type2aelembytes(type); 840 if (const_len > 0) { 841 int size_in_bytes = const_len * type2aelembytes(type); 842 if (size_in_bytes <= 16) 843 lane_count = 16/type2aelembytes(type); 844 else if (size_in_bytes > 16 && size_in_bytes <= 32) 845 lane_count = 32/type2aelembytes(type); 846 } 847 return lane_count; 848 }