1 /* 2 * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "ci/ciFlatArrayKlass.hpp" 26 #include "gc/shared/barrierSet.hpp" 27 #include "gc/shared/c2/barrierSetC2.hpp" 28 #include "gc/shared/c2/cardTableBarrierSetC2.hpp" 29 #include "gc/shared/gc_globals.hpp" 30 #include "opto/arraycopynode.hpp" 31 #include "opto/graphKit.hpp" 32 #include "opto/inlinetypenode.hpp" 33 #include "runtime/sharedRuntime.hpp" 34 #include "utilities/macros.hpp" 35 #include "utilities/powerOfTwo.hpp" 36 37 const TypeFunc* ArrayCopyNode::_arraycopy_type_Type = nullptr; 38 39 ArrayCopyNode::ArrayCopyNode(Compile* C, bool alloc_tightly_coupled, bool has_negative_length_guard) 40 : CallNode(arraycopy_type(), nullptr, TypePtr::BOTTOM), 41 _kind(None), 42 _alloc_tightly_coupled(alloc_tightly_coupled), 43 _has_negative_length_guard(has_negative_length_guard), 44 _arguments_validated(false), 45 _src_type(TypeOopPtr::BOTTOM), 46 _dest_type(TypeOopPtr::BOTTOM) { 47 init_class_id(Class_ArrayCopy); 48 init_flags(Flag_is_macro); 49 C->add_macro_node(this); 50 } 51 52 uint ArrayCopyNode::size_of() const { return sizeof(*this); } 53 54 ArrayCopyNode* ArrayCopyNode::make(GraphKit* kit, bool may_throw, 55 Node* src, Node* src_offset, 56 Node* dest, Node* dest_offset, 57 Node* length, 58 bool alloc_tightly_coupled, 59 bool has_negative_length_guard, 60 Node* src_klass, Node* dest_klass, 61 Node* src_length, Node* dest_length) { 62 63 ArrayCopyNode* ac = new ArrayCopyNode(kit->C, alloc_tightly_coupled, has_negative_length_guard); 64 kit->set_predefined_input_for_runtime_call(ac); 65 66 ac->init_req(ArrayCopyNode::Src, src); 67 ac->init_req(ArrayCopyNode::SrcPos, src_offset); 68 ac->init_req(ArrayCopyNode::Dest, dest); 69 ac->init_req(ArrayCopyNode::DestPos, dest_offset); 70 ac->init_req(ArrayCopyNode::Length, length); 71 ac->init_req(ArrayCopyNode::SrcLen, src_length); 72 ac->init_req(ArrayCopyNode::DestLen, dest_length); 73 ac->init_req(ArrayCopyNode::SrcKlass, src_klass); 74 ac->init_req(ArrayCopyNode::DestKlass, dest_klass); 75 76 if (may_throw) { 77 ac->set_req(TypeFunc::I_O , kit->i_o()); 78 kit->add_safepoint_edges(ac, false); 79 } 80 81 return ac; 82 } 83 84 void ArrayCopyNode::connect_outputs(GraphKit* kit, bool deoptimize_on_exception) { 85 kit->set_all_memory_call(this, true); 86 kit->set_control(kit->gvn().transform(new ProjNode(this,TypeFunc::Control))); 87 kit->set_i_o(kit->gvn().transform(new ProjNode(this, TypeFunc::I_O))); 88 kit->make_slow_call_ex(this, kit->env()->Throwable_klass(), true, deoptimize_on_exception); 89 kit->set_all_memory_call(this); 90 } 91 92 #ifndef PRODUCT 93 const char* ArrayCopyNode::_kind_names[] = {"arraycopy", "arraycopy, validated arguments", "clone", "oop array clone", "CopyOf", "CopyOfRange"}; 94 95 void ArrayCopyNode::dump_spec(outputStream *st) const { 96 CallNode::dump_spec(st); 97 st->print(" (%s%s)", _kind_names[_kind], _alloc_tightly_coupled ? ", tightly coupled allocation" : ""); 98 } 99 100 void ArrayCopyNode::dump_compact_spec(outputStream* st) const { 101 st->print("%s%s", _kind_names[_kind], _alloc_tightly_coupled ? ",tight" : ""); 102 } 103 #endif 104 105 intptr_t ArrayCopyNode::get_length_if_constant(PhaseGVN *phase) const { 106 // check that length is constant 107 Node* length = in(ArrayCopyNode::Length); 108 const Type* length_type = phase->type(length); 109 110 if (length_type == Type::TOP) { 111 return -1; 112 } 113 114 assert(is_clonebasic() || is_arraycopy() || is_copyof() || is_copyofrange(), "unexpected array copy type"); 115 116 return is_clonebasic() ? length->find_intptr_t_con(-1) : length->find_int_con(-1); 117 } 118 119 int ArrayCopyNode::get_count(PhaseGVN *phase) const { 120 if (is_clonebasic()) { 121 Node* src = in(ArrayCopyNode::Src); 122 const Type* src_type = phase->type(src); 123 124 if (src_type == Type::TOP) { 125 return -1; 126 } 127 128 if (src_type->isa_instptr()) { 129 const TypeInstPtr* inst_src = src_type->is_instptr(); 130 ciInstanceKlass* ik = inst_src->instance_klass(); 131 // ciInstanceKlass::nof_nonstatic_fields() doesn't take injected 132 // fields into account. They are rare anyway so easier to simply 133 // skip instances with injected fields. 134 if ((!inst_src->klass_is_exact() && (ik->is_interface() || ik->has_subklass())) || ik->has_injected_fields()) { 135 return -1; 136 } 137 int nb_fields = ik->nof_nonstatic_fields(); 138 return nb_fields; 139 } else { 140 const TypeAryPtr* ary_src = src_type->isa_aryptr(); 141 assert (ary_src != nullptr, "not an array or instance?"); 142 // clone passes a length as a rounded number of longs. If we're 143 // cloning an array we'll do it element by element. If the 144 // length of the input array is constant, ArrayCopyNode::Length 145 // must be too. Note that the opposite does not need to hold, 146 // because different input array lengths (e.g. int arrays with 147 // 3 or 4 elements) might lead to the same length input 148 // (e.g. 2 double-words). 149 assert(!ary_src->size()->is_con() || (get_length_if_constant(phase) >= 0) || 150 (UseArrayFlattening && ary_src->elem()->make_oopptr() != nullptr && ary_src->elem()->make_oopptr()->can_be_inline_type()) || 151 phase->is_IterGVN() || phase->C->inlining_incrementally() || StressReflectiveCode, "inconsistent"); 152 if (ary_src->size()->is_con()) { 153 return ary_src->size()->get_con(); 154 } 155 return -1; 156 } 157 } 158 159 return get_length_if_constant(phase); 160 } 161 162 Node* ArrayCopyNode::load(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, const Type *type, BasicType bt) { 163 // Pin the load: if this is an array load, it's going to be dependent on a condition that's not a range check for that 164 // access. If that condition is replaced by an identical dominating one, then an unpinned load would risk floating 165 // above runtime checks that guarantee it is within bounds. 166 DecoratorSet decorators = C2_READ_ACCESS | C2_CONTROL_DEPENDENT_LOAD | IN_HEAP | C2_ARRAY_COPY | C2_UNKNOWN_CONTROL_LOAD; 167 C2AccessValuePtr addr(adr, adr_type); 168 C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr); 169 Node* res = bs->load_at(access, type); 170 ctl = access.ctl(); 171 return res; 172 } 173 174 void ArrayCopyNode::store(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, Node* val, const Type *type, BasicType bt) { 175 DecoratorSet decorators = C2_WRITE_ACCESS | IN_HEAP | C2_ARRAY_COPY; 176 if (is_alloc_tightly_coupled()) { 177 decorators |= C2_TIGHTLY_COUPLED_ALLOC; 178 } 179 C2AccessValuePtr addr(adr, adr_type); 180 C2AccessValue value(val, type); 181 C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr); 182 bs->store_at(access, value); 183 ctl = access.ctl(); 184 } 185 186 187 Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) { 188 if (!is_clonebasic()) { 189 return nullptr; 190 } 191 192 Node* base_src = in(ArrayCopyNode::Src); 193 Node* base_dest = in(ArrayCopyNode::Dest); 194 Node* ctl = in(TypeFunc::Control); 195 Node* in_mem = in(TypeFunc::Memory); 196 197 const Type* src_type = phase->type(base_src); 198 const TypeInstPtr* inst_src = src_type->isa_instptr(); 199 if (inst_src == nullptr) { 200 return nullptr; 201 } 202 203 MergeMemNode* mem = phase->transform(MergeMemNode::make(in_mem))->as_MergeMem(); 204 phase->record_for_igvn(mem); 205 if (can_reshape) { 206 phase->is_IterGVN()->_worklist.push(mem); 207 } 208 209 210 ciInstanceKlass* ik = inst_src->instance_klass(); 211 212 if (!inst_src->klass_is_exact()) { 213 assert(!ik->is_interface(), "inconsistent klass hierarchy"); 214 if (ik->has_subklass()) { 215 // Concurrent class loading. 216 // Fail fast and return NodeSentinel to indicate that the transform failed. 217 return NodeSentinel; 218 } else { 219 phase->C->dependencies()->assert_leaf_type(ik); 220 } 221 } 222 223 assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields"); 224 225 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 226 for (int i = 0; i < count; i++) { 227 ciField* field = ik->nonstatic_field_at(i); 228 const TypePtr* adr_type = phase->C->alias_type(field)->adr_type(); 229 Node* off = phase->MakeConX(field->offset_in_bytes()); 230 Node* next_src = phase->transform(new AddPNode(base_src,base_src,off)); 231 Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off)); 232 assert(phase->C->get_alias_index(adr_type) == phase->C->get_alias_index(phase->type(next_src)->isa_ptr()), 233 "slice of address and input slice don't match"); 234 assert(phase->C->get_alias_index(adr_type) == phase->C->get_alias_index(phase->type(next_dest)->isa_ptr()), 235 "slice of address and input slice don't match"); 236 BasicType bt = field->layout_type(); 237 238 const Type *type; 239 if (bt == T_OBJECT) { 240 if (!field->type()->is_loaded()) { 241 type = TypeInstPtr::BOTTOM; 242 } else { 243 ciType* field_klass = field->type(); 244 type = TypeOopPtr::make_from_klass(field_klass->as_klass()); 245 } 246 } else { 247 type = Type::get_const_basic_type(bt); 248 } 249 250 Node* v = load(bs, phase, ctl, mem, next_src, adr_type, type, bt); 251 store(bs, phase, ctl, mem, next_dest, adr_type, v, type, bt); 252 } 253 254 if (!finish_transform(phase, can_reshape, ctl, mem)) { 255 // Return NodeSentinel to indicate that the transform failed 256 return NodeSentinel; 257 } 258 259 return mem; 260 } 261 262 bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape, 263 Node*& adr_src, 264 Node*& base_src, 265 Node*& adr_dest, 266 Node*& base_dest, 267 BasicType& copy_type, 268 const Type*& value_type, 269 bool& disjoint_bases) { 270 base_src = in(ArrayCopyNode::Src); 271 base_dest = in(ArrayCopyNode::Dest); 272 const Type* src_type = phase->type(base_src); 273 const TypeAryPtr* ary_src = src_type->isa_aryptr(); 274 275 Node* src_offset = in(ArrayCopyNode::SrcPos); 276 Node* dest_offset = in(ArrayCopyNode::DestPos); 277 278 if (is_arraycopy() || is_copyofrange() || is_copyof()) { 279 const Type* dest_type = phase->type(base_dest); 280 const TypeAryPtr* ary_dest = dest_type->isa_aryptr(); 281 282 // newly allocated object is guaranteed to not overlap with source object 283 disjoint_bases = is_alloc_tightly_coupled(); 284 if (ary_src == nullptr || ary_src->elem() == Type::BOTTOM || 285 ary_dest == nullptr || ary_dest->elem() == Type::BOTTOM) { 286 // We don't know if arguments are arrays 287 return false; 288 } 289 290 BasicType src_elem = ary_src->elem()->array_element_basic_type(); 291 BasicType dest_elem = ary_dest->elem()->array_element_basic_type(); 292 if (is_reference_type(src_elem, true)) src_elem = T_OBJECT; 293 if (is_reference_type(dest_elem, true)) dest_elem = T_OBJECT; 294 295 // TODO 8350865 What about atomicity? 296 if (src_elem != dest_elem || ary_src->is_null_free() != ary_dest->is_null_free() || ary_src->is_flat() != ary_dest->is_flat() || dest_elem == T_VOID) { 297 // We don't know if arguments are arrays of the same type 298 return false; 299 } 300 301 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 302 if ((!ary_dest->is_flat() && bs->array_copy_requires_gc_barriers(is_alloc_tightly_coupled(), dest_elem, false, false, BarrierSetC2::Optimization)) || 303 (ary_dest->is_flat() && ary_src->elem()->inline_klass()->contains_oops() && 304 bs->array_copy_requires_gc_barriers(is_alloc_tightly_coupled(), T_OBJECT, false, false, BarrierSetC2::Optimization))) { 305 // It's an object array copy but we can't emit the card marking that is needed 306 return false; 307 } 308 309 value_type = ary_src->elem(); 310 311 uint shift = exact_log2(type2aelembytes(dest_elem)); 312 if (ary_dest->is_flat()) { 313 assert(ary_src->is_flat(), "src and dest must be flat"); 314 shift = ary_src->flat_log_elem_size(); 315 src_elem = T_FLAT_ELEMENT; 316 dest_elem = T_FLAT_ELEMENT; 317 } 318 319 const uint header = arrayOopDesc::base_offset_in_bytes(dest_elem); 320 321 src_offset = Compile::conv_I2X_index(phase, src_offset, ary_src->size()); 322 if (src_offset->is_top()) { 323 // Offset is out of bounds (the ArrayCopyNode will be removed) 324 return false; 325 } 326 dest_offset = Compile::conv_I2X_index(phase, dest_offset, ary_dest->size()); 327 if (dest_offset->is_top()) { 328 // Offset is out of bounds (the ArrayCopyNode will be removed) 329 if (can_reshape) { 330 // record src_offset, so it can be deleted later (if it is dead) 331 phase->is_IterGVN()->_worklist.push(src_offset); 332 } 333 return false; 334 } 335 336 Node* hook = new Node(1); 337 hook->init_req(0, dest_offset); 338 339 Node* src_scale = phase->transform(new LShiftXNode(src_offset, phase->intcon(shift))); 340 341 hook->destruct(phase); 342 343 Node* dest_scale = phase->transform(new LShiftXNode(dest_offset, phase->intcon(shift))); 344 345 adr_src = phase->transform(new AddPNode(base_src, base_src, src_scale)); 346 adr_dest = phase->transform(new AddPNode(base_dest, base_dest, dest_scale)); 347 348 adr_src = phase->transform(new AddPNode(base_src, adr_src, phase->MakeConX(header))); 349 adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, phase->MakeConX(header))); 350 351 copy_type = dest_elem; 352 } else { 353 assert(ary_src != nullptr, "should be a clone"); 354 assert(is_clonebasic(), "should be"); 355 356 disjoint_bases = true; 357 358 if (ary_src->elem()->make_oopptr() != nullptr && 359 ary_src->elem()->make_oopptr()->can_be_inline_type()) { 360 return false; 361 } 362 363 BasicType elem = ary_src->isa_aryptr()->elem()->array_element_basic_type(); 364 if (is_reference_type(elem, true)) { 365 elem = T_OBJECT; 366 } 367 368 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 369 if ((!ary_src->is_flat() && bs->array_copy_requires_gc_barriers(true, elem, true, is_clone_inst(), BarrierSetC2::Optimization)) || 370 (ary_src->is_flat() && ary_src->elem()->inline_klass()->contains_oops() && 371 bs->array_copy_requires_gc_barriers(true, T_OBJECT, true, is_clone_inst(), BarrierSetC2::Optimization))) { 372 // It's an object array copy but we can't emit the card marking that is needed 373 return false; 374 } 375 376 adr_src = phase->transform(new AddPNode(base_src, base_src, src_offset)); 377 adr_dest = phase->transform(new AddPNode(base_dest, base_dest, dest_offset)); 378 379 // The address is offsetted to an aligned address where a raw copy would start. 380 // If the clone copy is decomposed into load-stores - the address is adjusted to 381 // point at where the array starts. 382 const Type* toff = phase->type(src_offset); 383 int offset = toff->isa_long() ? (int) toff->is_long()->get_con() : (int) toff->is_int()->get_con(); 384 int diff = arrayOopDesc::base_offset_in_bytes(elem) - offset; 385 assert(diff >= 0, "clone should not start after 1st array element"); 386 if (diff > 0) { 387 adr_src = phase->transform(new AddPNode(base_src, adr_src, phase->MakeConX(diff))); 388 adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, phase->MakeConX(diff))); 389 } 390 copy_type = elem; 391 value_type = ary_src->elem(); 392 } 393 return true; 394 } 395 396 const TypeAryPtr* ArrayCopyNode::get_address_type(PhaseGVN* phase, const TypePtr* atp, Node* n) { 397 if (atp == TypeOopPtr::BOTTOM) { 398 atp = phase->type(n)->isa_ptr(); 399 } 400 // adjust atp to be the correct array element address type 401 return atp->add_offset(Type::OffsetBot)->is_aryptr(); 402 } 403 404 void ArrayCopyNode::array_copy_test_overlap(GraphKit& kit, bool disjoint_bases, int count, Node*& backward_ctl) { 405 Node* ctl = kit.control(); 406 if (!disjoint_bases && count > 1) { 407 PhaseGVN& gvn = kit.gvn(); 408 Node* src_offset = in(ArrayCopyNode::SrcPos); 409 Node* dest_offset = in(ArrayCopyNode::DestPos); 410 assert(src_offset != nullptr && dest_offset != nullptr, "should be"); 411 Node* cmp = gvn.transform(new CmpINode(src_offset, dest_offset)); 412 Node *bol = gvn.transform(new BoolNode(cmp, BoolTest::lt)); 413 IfNode *iff = new IfNode(ctl, bol, PROB_FAIR, COUNT_UNKNOWN); 414 415 gvn.transform(iff); 416 417 kit.set_control(gvn.transform(new IfFalseNode(iff))); 418 backward_ctl = gvn.transform(new IfTrueNode(iff)); 419 } 420 } 421 422 void ArrayCopyNode::copy(GraphKit& kit, 423 const TypeAryPtr* atp_src, 424 const TypeAryPtr* atp_dest, 425 int i, 426 Node* base_src, 427 Node* base_dest, 428 Node* adr_src, 429 Node* adr_dest, 430 BasicType copy_type, 431 const Type* value_type) { 432 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 433 Node* ctl = kit.control(); 434 if (atp_dest->is_flat()) { 435 ciInlineKlass* vk = atp_src->elem()->inline_klass(); 436 for (int j = 0; j < vk->nof_nonstatic_fields(); j++) { 437 ciField* field = vk->nonstatic_field_at(j); 438 int off_in_vt = field->offset_in_bytes() - vk->payload_offset(); 439 Node* off = kit.MakeConX(off_in_vt + i * atp_src->flat_elem_size()); 440 ciType* ft = field->type(); 441 BasicType bt = type2field[ft->basic_type()]; 442 assert(!field->is_flat(), "flat field encountered"); 443 const Type* rt = Type::get_const_type(ft); 444 const TypePtr* adr_type = atp_src->with_field_offset(off_in_vt)->add_offset(Type::OffsetBot); 445 assert(!bs->array_copy_requires_gc_barriers(is_alloc_tightly_coupled(), bt, false, false, BarrierSetC2::Optimization), "GC barriers required"); 446 Node* next_src = kit.gvn().transform(new AddPNode(base_src, adr_src, off)); 447 Node* next_dest = kit.gvn().transform(new AddPNode(base_dest, adr_dest, off)); 448 Node* v = load(bs, &kit.gvn(), ctl, kit.merged_memory(), next_src, adr_type, rt, bt); 449 store(bs, &kit.gvn(), ctl, kit.merged_memory(), next_dest, adr_type, v, rt, bt); 450 } 451 } else { 452 Node* off = kit.MakeConX(type2aelembytes(copy_type) * i); 453 Node* next_src = kit.gvn().transform(new AddPNode(base_src, adr_src, off)); 454 Node* next_dest = kit.gvn().transform(new AddPNode(base_dest, adr_dest, off)); 455 Node* v = load(bs, &kit.gvn(), ctl, kit.merged_memory(), next_src, atp_src, value_type, copy_type); 456 store(bs, &kit.gvn(), ctl, kit.merged_memory(), next_dest, atp_dest, v, value_type, copy_type); 457 } 458 kit.set_control(ctl); 459 } 460 461 462 void ArrayCopyNode::array_copy_forward(GraphKit& kit, 463 bool can_reshape, 464 const TypeAryPtr* atp_src, 465 const TypeAryPtr* atp_dest, 466 Node* adr_src, 467 Node* base_src, 468 Node* adr_dest, 469 Node* base_dest, 470 BasicType copy_type, 471 const Type* value_type, 472 int count) { 473 if (!kit.stopped()) { 474 // copy forward 475 if (count > 0) { 476 for (int i = 0; i < count; i++) { 477 copy(kit, atp_src, atp_dest, i, base_src, base_dest, adr_src, adr_dest, copy_type, value_type); 478 } 479 } else if (can_reshape) { 480 PhaseGVN& gvn = kit.gvn(); 481 assert(gvn.is_IterGVN(), ""); 482 gvn.record_for_igvn(adr_src); 483 gvn.record_for_igvn(adr_dest); 484 } 485 } 486 } 487 488 void ArrayCopyNode::array_copy_backward(GraphKit& kit, 489 bool can_reshape, 490 const TypeAryPtr* atp_src, 491 const TypeAryPtr* atp_dest, 492 Node* adr_src, 493 Node* base_src, 494 Node* adr_dest, 495 Node* base_dest, 496 BasicType copy_type, 497 const Type* value_type, 498 int count) { 499 if (!kit.stopped()) { 500 // copy backward 501 PhaseGVN& gvn = kit.gvn(); 502 503 if (count > 0) { 504 for (int i = count-1; i >= 0; i--) { 505 copy(kit, atp_src, atp_dest, i, base_src, base_dest, adr_src, adr_dest, copy_type, value_type); 506 } 507 } else if(can_reshape) { 508 PhaseGVN& gvn = kit.gvn(); 509 assert(gvn.is_IterGVN(), ""); 510 gvn.record_for_igvn(adr_src); 511 gvn.record_for_igvn(adr_dest); 512 } 513 } 514 } 515 516 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape, 517 Node* ctl, Node *mem) { 518 if (can_reshape) { 519 PhaseIterGVN* igvn = phase->is_IterGVN(); 520 igvn->set_delay_transform(false); 521 if (is_clonebasic()) { 522 Node* out_mem = proj_out(TypeFunc::Memory); 523 524 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 525 if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() || 526 out_mem->raw_out(0)->outcnt() != 1 || !out_mem->raw_out(0)->raw_out(0)->is_MemBar()) { 527 assert(bs->array_copy_requires_gc_barriers(true, T_OBJECT, true, is_clone_inst(), BarrierSetC2::Optimization), "can only happen with card marking"); 528 return false; 529 } 530 531 igvn->replace_node(out_mem->raw_out(0), mem); 532 533 Node* out_ctl = proj_out(TypeFunc::Control); 534 igvn->replace_node(out_ctl, ctl); 535 } else { 536 // replace fallthrough projections of the ArrayCopyNode by the 537 // new memory, control and the input IO. 538 CallProjections* callprojs = extract_projections(true, false); 539 540 if (callprojs->fallthrough_ioproj != nullptr) { 541 igvn->replace_node(callprojs->fallthrough_ioproj, in(TypeFunc::I_O)); 542 } 543 if (callprojs->fallthrough_memproj != nullptr) { 544 igvn->replace_node(callprojs->fallthrough_memproj, mem); 545 } 546 if (callprojs->fallthrough_catchproj != nullptr) { 547 igvn->replace_node(callprojs->fallthrough_catchproj, ctl); 548 } 549 550 // The ArrayCopyNode is not disconnected. It still has the 551 // projections for the exception case. Replace current 552 // ArrayCopyNode with a dummy new one with a top() control so 553 // that this part of the graph stays consistent but is 554 // eventually removed. 555 556 set_req(0, phase->C->top()); 557 remove_dead_region(phase, can_reshape); 558 } 559 } else { 560 if (in(TypeFunc::Control) != ctl) { 561 // we can't return new memory and control from Ideal at parse time 562 assert(!is_clonebasic() || UseShenandoahGC, "added control for clone?"); 563 phase->record_for_igvn(this); 564 return false; 565 } 566 } 567 return true; 568 } 569 570 571 Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) { 572 // Perform any generic optimizations first 573 Node* result = SafePointNode::Ideal(phase, can_reshape); 574 if (result != nullptr) { 575 return result; 576 } 577 578 if (StressArrayCopyMacroNode && !can_reshape) { 579 phase->record_for_igvn(this); 580 return nullptr; 581 } 582 583 // See if it's a small array copy and we can inline it as 584 // loads/stores 585 // Here we can only do: 586 // - arraycopy if all arguments were validated before and we don't 587 // need card marking 588 // - clone for which we don't need to do card marking 589 590 if (!is_clonebasic() && !is_arraycopy_validated() && 591 !is_copyofrange_validated() && !is_copyof_validated()) { 592 return nullptr; 593 } 594 595 assert(in(TypeFunc::Control) != nullptr && 596 in(TypeFunc::Memory) != nullptr && 597 in(ArrayCopyNode::Src) != nullptr && 598 in(ArrayCopyNode::Dest) != nullptr && 599 in(ArrayCopyNode::Length) != nullptr && 600 in(ArrayCopyNode::SrcPos) != nullptr && 601 in(ArrayCopyNode::DestPos) != nullptr, "broken inputs"); 602 603 if (in(TypeFunc::Control)->is_top() || 604 in(TypeFunc::Memory)->is_top() || 605 phase->type(in(ArrayCopyNode::Src)) == Type::TOP || 606 phase->type(in(ArrayCopyNode::Dest)) == Type::TOP || 607 (in(ArrayCopyNode::SrcPos) != nullptr && in(ArrayCopyNode::SrcPos)->is_top()) || 608 (in(ArrayCopyNode::DestPos) != nullptr && in(ArrayCopyNode::DestPos)->is_top())) { 609 return nullptr; 610 } 611 612 int count = get_count(phase); 613 614 if (count < 0 || count > ArrayCopyLoadStoreMaxElem) { 615 return nullptr; 616 } 617 618 Node* src = in(ArrayCopyNode::Src); 619 Node* dest = in(ArrayCopyNode::Dest); 620 const Type* src_type = phase->type(src); 621 const Type* dest_type = phase->type(dest); 622 623 if (src_type->isa_aryptr() && dest_type->isa_instptr()) { 624 // clone used for load of unknown inline type can't be optimized at 625 // this point 626 return nullptr; 627 } 628 629 Node* mem = try_clone_instance(phase, can_reshape, count); 630 if (mem != nullptr) { 631 return (mem == NodeSentinel) ? nullptr : mem; 632 } 633 634 Node* adr_src = nullptr; 635 Node* base_src = nullptr; 636 Node* adr_dest = nullptr; 637 Node* base_dest = nullptr; 638 BasicType copy_type = T_ILLEGAL; 639 const Type* value_type = nullptr; 640 bool disjoint_bases = false; 641 642 if (!prepare_array_copy(phase, can_reshape, 643 adr_src, base_src, adr_dest, base_dest, 644 copy_type, value_type, disjoint_bases)) { 645 assert(adr_src == nullptr, "no node can be left behind"); 646 assert(adr_dest == nullptr, "no node can be left behind"); 647 return nullptr; 648 } 649 650 JVMState* new_jvms = nullptr; 651 SafePointNode* new_map = nullptr; 652 if (!is_clonebasic()) { 653 new_jvms = jvms()->clone_shallow(phase->C); 654 new_map = new SafePointNode(req(), new_jvms); 655 for (uint i = TypeFunc::FramePtr; i < req(); i++) { 656 new_map->init_req(i, in(i)); 657 } 658 new_jvms->set_map(new_map); 659 } else { 660 new_jvms = new (phase->C) JVMState(0); 661 new_map = new SafePointNode(TypeFunc::Parms, new_jvms); 662 new_jvms->set_map(new_map); 663 } 664 new_map->set_control(in(TypeFunc::Control)); 665 new_map->set_memory(MergeMemNode::make(in(TypeFunc::Memory))); 666 new_map->set_i_o(in(TypeFunc::I_O)); 667 phase->record_for_igvn(new_map); 668 669 const TypeAryPtr* atp_src = get_address_type(phase, _src_type, src); 670 const TypeAryPtr* atp_dest = get_address_type(phase, _dest_type, dest); 671 672 if (can_reshape) { 673 assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms"); 674 phase->is_IterGVN()->set_delay_transform(true); 675 } 676 677 GraphKit kit(new_jvms, phase); 678 679 SafePointNode* backward_map = nullptr; 680 SafePointNode* forward_map = nullptr; 681 Node* backward_ctl = phase->C->top(); 682 683 array_copy_test_overlap(kit, disjoint_bases, count, backward_ctl); 684 685 { 686 PreserveJVMState pjvms(&kit); 687 688 array_copy_forward(kit, can_reshape, 689 atp_src, atp_dest, 690 adr_src, base_src, adr_dest, base_dest, 691 copy_type, value_type, count); 692 693 forward_map = kit.stop(); 694 } 695 696 kit.set_control(backward_ctl); 697 array_copy_backward(kit, can_reshape, 698 atp_src, atp_dest, 699 adr_src, base_src, adr_dest, base_dest, 700 copy_type, value_type, count); 701 702 backward_map = kit.stop(); 703 704 if (!forward_map->control()->is_top() && !backward_map->control()->is_top()) { 705 assert(forward_map->i_o() == backward_map->i_o(), "need a phi on IO?"); 706 Node* ctl = new RegionNode(3); 707 Node* mem = new PhiNode(ctl, Type::MEMORY, TypePtr::BOTTOM); 708 kit.set_map(forward_map); 709 ctl->init_req(1, kit.control()); 710 mem->init_req(1, kit.reset_memory()); 711 kit.set_map(backward_map); 712 ctl->init_req(2, kit.control()); 713 mem->init_req(2, kit.reset_memory()); 714 kit.set_control(phase->transform(ctl)); 715 kit.set_all_memory(phase->transform(mem)); 716 } else if (!forward_map->control()->is_top()) { 717 kit.set_map(forward_map); 718 } else { 719 assert(!backward_map->control()->is_top(), "no copy?"); 720 kit.set_map(backward_map); 721 } 722 723 if (can_reshape) { 724 assert(phase->is_IterGVN()->delay_transform(), "should be delaying transforms"); 725 phase->is_IterGVN()->set_delay_transform(false); 726 } 727 728 mem = kit.map()->memory(); 729 if (!finish_transform(phase, can_reshape, kit.control(), mem)) { 730 if (!can_reshape) { 731 phase->record_for_igvn(this); 732 } else { 733 // put in worklist, so that if it happens to be dead it is removed 734 phase->is_IterGVN()->_worklist.push(mem); 735 } 736 return nullptr; 737 } 738 739 return mem; 740 } 741 742 bool ArrayCopyNode::may_modify(const TypeOopPtr* t_oop, PhaseValues* phase) { 743 Node* dest = in(ArrayCopyNode::Dest); 744 if (dest->is_top()) { 745 return false; 746 } 747 const TypeOopPtr* dest_t = phase->type(dest)->is_oopptr(); 748 assert(!dest_t->is_known_instance() || _dest_type->is_known_instance(), "result of EA not recorded"); 749 assert(in(ArrayCopyNode::Src)->is_top() || !phase->type(in(ArrayCopyNode::Src))->is_oopptr()->is_known_instance() || 750 _src_type->is_known_instance(), "result of EA not recorded"); 751 752 if (_dest_type != TypeOopPtr::BOTTOM || t_oop->is_known_instance()) { 753 assert(_dest_type == TypeOopPtr::BOTTOM || _dest_type->is_known_instance(), "result of EA is known instance"); 754 return t_oop->instance_id() == _dest_type->instance_id(); 755 } 756 757 return CallNode::may_modify_arraycopy_helper(dest_t, t_oop, phase); 758 } 759 760 bool ArrayCopyNode::may_modify_helper(const TypeOopPtr* t_oop, Node* n, PhaseValues* phase, ArrayCopyNode*& ac) { 761 if (n != nullptr && 762 n->is_ArrayCopy() && 763 n->as_ArrayCopy()->may_modify(t_oop, phase)) { 764 ac = n->as_ArrayCopy(); 765 return true; 766 } 767 return false; 768 } 769 770 bool ArrayCopyNode::may_modify(const TypeOopPtr* t_oop, MemBarNode* mb, PhaseValues* phase, ArrayCopyNode*& ac) { 771 if (mb->trailing_expanded_array_copy()) { 772 return true; 773 } 774 775 Node* c = mb->in(0); 776 777 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 778 // step over g1 gc barrier if we're at e.g. a clone with ReduceInitialCardMarks off 779 c = bs->step_over_gc_barrier(c); 780 781 CallNode* call = nullptr; 782 guarantee(c != nullptr, "step_over_gc_barrier failed, there must be something to step to."); 783 if (c->is_Region()) { 784 for (uint i = 1; i < c->req(); i++) { 785 if (c->in(i) != nullptr) { 786 Node* n = c->in(i)->in(0); 787 if (may_modify_helper(t_oop, n, phase, ac)) { 788 assert(c == mb->in(0), "only for clone"); 789 return true; 790 } 791 } 792 } 793 } else if (may_modify_helper(t_oop, c->in(0), phase, ac)) { 794 #ifdef ASSERT 795 bool use_ReduceInitialCardMarks = BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) && 796 static_cast<CardTableBarrierSetC2*>(bs)->use_ReduceInitialCardMarks(); 797 assert(c == mb->in(0) || (ac->is_clonebasic() && !use_ReduceInitialCardMarks), "only for clone"); 798 #endif 799 return true; 800 } 801 802 return false; 803 } 804 805 // Does this array copy modify offsets between offset_lo and offset_hi 806 // in the destination array 807 // if must_modify is false, return true if the copy could write 808 // between offset_lo and offset_hi 809 // if must_modify is true, return true if the copy is guaranteed to 810 // write between offset_lo and offset_hi 811 bool ArrayCopyNode::modifies(intptr_t offset_lo, intptr_t offset_hi, PhaseValues* phase, bool must_modify) const { 812 assert(_kind == ArrayCopy || _kind == CopyOf || _kind == CopyOfRange, "only for real array copies"); 813 814 Node* dest = in(Dest); 815 Node* dest_pos = in(DestPos); 816 Node* len = in(Length); 817 818 const TypeInt *dest_pos_t = phase->type(dest_pos)->isa_int(); 819 const TypeInt *len_t = phase->type(len)->isa_int(); 820 const TypeAryPtr* ary_t = phase->type(dest)->isa_aryptr(); 821 822 if (dest_pos_t == nullptr || len_t == nullptr || ary_t == nullptr) { 823 return !must_modify; 824 } 825 826 BasicType ary_elem = ary_t->isa_aryptr()->elem()->array_element_basic_type(); 827 if (is_reference_type(ary_elem, true)) ary_elem = T_OBJECT; 828 829 uint header; 830 uint elem_size; 831 if (ary_t->is_flat()) { 832 header = arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT); 833 elem_size = ary_t->flat_elem_size(); 834 } else { 835 header = arrayOopDesc::base_offset_in_bytes(ary_elem); 836 elem_size = type2aelembytes(ary_elem); 837 } 838 839 jlong dest_pos_plus_len_lo = (((jlong)dest_pos_t->_lo) + len_t->_lo) * elem_size + header; 840 jlong dest_pos_plus_len_hi = (((jlong)dest_pos_t->_hi) + len_t->_hi) * elem_size + header; 841 jlong dest_pos_lo = ((jlong)dest_pos_t->_lo) * elem_size + header; 842 jlong dest_pos_hi = ((jlong)dest_pos_t->_hi) * elem_size + header; 843 844 if (must_modify) { 845 if (offset_lo >= dest_pos_hi && offset_hi < dest_pos_plus_len_lo) { 846 return true; 847 } 848 } else { 849 if (offset_hi >= dest_pos_lo && offset_lo < dest_pos_plus_len_hi) { 850 return true; 851 } 852 } 853 return false; 854 } 855 856 // As an optimization, choose optimum vector size for copy length known at compile time. 857 int ArrayCopyNode::get_partial_inline_vector_lane_count(BasicType type, int const_len) { 858 int lane_count = ArrayOperationPartialInlineSize/type2aelembytes(type); 859 if (const_len > 0) { 860 int size_in_bytes = const_len * type2aelembytes(type); 861 if (size_in_bytes <= 16) 862 lane_count = 16/type2aelembytes(type); 863 else if (size_in_bytes > 16 && size_in_bytes <= 32) 864 lane_count = 32/type2aelembytes(type); 865 } 866 return lane_count; 867 }