1 /* 2 * Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "ci/ciFlatArrayKlass.hpp" 26 #include "gc/shared/barrierSet.hpp" 27 #include "gc/shared/tlab_globals.hpp" 28 #include "oops/objArrayKlass.hpp" 29 #include "opto/arraycopynode.hpp" 30 #include "opto/castnode.hpp" 31 #include "opto/convertnode.hpp" 32 #include "opto/graphKit.hpp" 33 #include "opto/macro.hpp" 34 #include "opto/runtime.hpp" 35 #include "opto/vectornode.hpp" 36 #include "runtime/stubRoutines.hpp" 37 #include "utilities/align.hpp" 38 #include "utilities/powerOfTwo.hpp" 39 40 void PhaseMacroExpand::insert_mem_bar(Node** ctrl, Node** mem, int opcode, int alias_idx, Node* precedent) { 41 MemBarNode* mb = MemBarNode::make(C, opcode, alias_idx, precedent); 42 mb->init_req(TypeFunc::Control, *ctrl); 43 mb->init_req(TypeFunc::Memory, *mem); 44 transform_later(mb); 45 *ctrl = new ProjNode(mb,TypeFunc::Control); 46 transform_later(*ctrl); 47 Node* mem_proj = new ProjNode(mb,TypeFunc::Memory); 48 transform_later(mem_proj); 49 if (alias_idx == Compile::AliasIdxBot) { 50 *mem = mem_proj; 51 } else { 52 MergeMemNode* mm = (*mem)->clone()->as_MergeMem(); 53 mm->set_memory_at(alias_idx, mem_proj); 54 transform_later(mm); 55 *mem = mm; 56 } 57 } 58 59 Node* PhaseMacroExpand::array_element_address(Node* ary, Node* idx, BasicType elembt) { 60 uint shift = exact_log2(type2aelembytes(elembt)); 61 const TypeAryPtr* array_type = _igvn.type(ary)->isa_aryptr(); 62 if (array_type != nullptr && array_type->is_aryptr()->is_flat()) { 63 // Use T_FLAT_ELEMENT to get proper alignment with COH when fetching the array element address. 64 elembt = T_FLAT_ELEMENT; 65 } 66 uint header = arrayOopDesc::base_offset_in_bytes(elembt); 67 Node* base = basic_plus_adr(ary, header); 68 #ifdef _LP64 69 // see comment in GraphKit::array_element_address 70 int index_max = max_jint - 1; // array size is max_jint, index is one less 71 const TypeLong* lidxtype = TypeLong::make(CONST64(0), index_max, Type::WidenMax); 72 idx = transform_later( new ConvI2LNode(idx, lidxtype) ); 73 #endif 74 Node* scale = new LShiftXNode(idx, intcon(shift)); 75 transform_later(scale); 76 return basic_plus_adr(ary, base, scale); 77 } 78 79 Node* PhaseMacroExpand::ConvI2L(Node* offset) { 80 return transform_later(new ConvI2LNode(offset)); 81 } 82 83 Node* PhaseMacroExpand::make_leaf_call(Node* ctrl, Node* mem, 84 const TypeFunc* call_type, address call_addr, 85 const char* call_name, 86 const TypePtr* adr_type, 87 Node* parm0, Node* parm1, 88 Node* parm2, Node* parm3, 89 Node* parm4, Node* parm5, 90 Node* parm6, Node* parm7) { 91 Node* call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type); 92 call->init_req(TypeFunc::Control, ctrl); 93 call->init_req(TypeFunc::I_O , top()); 94 call->init_req(TypeFunc::Memory , mem); 95 call->init_req(TypeFunc::ReturnAdr, top()); 96 call->init_req(TypeFunc::FramePtr, top()); 97 98 // Hook each parm in order. Stop looking at the first null. 99 if (parm0 != nullptr) { call->init_req(TypeFunc::Parms+0, parm0); 100 if (parm1 != nullptr) { call->init_req(TypeFunc::Parms+1, parm1); 101 if (parm2 != nullptr) { call->init_req(TypeFunc::Parms+2, parm2); 102 if (parm3 != nullptr) { call->init_req(TypeFunc::Parms+3, parm3); 103 if (parm4 != nullptr) { call->init_req(TypeFunc::Parms+4, parm4); 104 if (parm5 != nullptr) { call->init_req(TypeFunc::Parms+5, parm5); 105 if (parm6 != nullptr) { call->init_req(TypeFunc::Parms+6, parm6); 106 if (parm7 != nullptr) { call->init_req(TypeFunc::Parms+7, parm7); 107 /* close each nested if ===> */ } } } } } } } } 108 assert(call->in(call->req()-1) != nullptr, "must initialize all parms"); 109 110 return call; 111 } 112 113 114 //------------------------------generate_guard--------------------------- 115 // Helper function for generating guarded fast-slow graph structures. 116 // The given 'test', if true, guards a slow path. If the test fails 117 // then a fast path can be taken. (We generally hope it fails.) 118 // In all cases, GraphKit::control() is updated to the fast path. 119 // The returned value represents the control for the slow path. 120 // The return value is never 'top'; it is either a valid control 121 // or null if it is obvious that the slow path can never be taken. 122 // Also, if region and the slow control are not null, the slow edge 123 // is appended to the region. 124 Node* PhaseMacroExpand::generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob) { 125 if ((*ctrl)->is_top()) { 126 // Already short circuited. 127 return nullptr; 128 } 129 // Build an if node and its projections. 130 // If test is true we take the slow path, which we assume is uncommon. 131 if (_igvn.type(test) == TypeInt::ZERO) { 132 // The slow branch is never taken. No need to build this guard. 133 return nullptr; 134 } 135 136 IfNode* iff = new IfNode(*ctrl, test, true_prob, COUNT_UNKNOWN); 137 transform_later(iff); 138 139 Node* if_slow = new IfTrueNode(iff); 140 transform_later(if_slow); 141 142 if (region != nullptr) { 143 region->add_req(if_slow); 144 } 145 146 Node* if_fast = new IfFalseNode(iff); 147 transform_later(if_fast); 148 149 *ctrl = if_fast; 150 151 return if_slow; 152 } 153 154 Node* PhaseMacroExpand::generate_slow_guard(Node** ctrl, Node* test, RegionNode* region) { 155 return generate_guard(ctrl, test, region, PROB_UNLIKELY_MAG(3)); 156 } 157 158 inline Node* PhaseMacroExpand::generate_fair_guard(Node** ctrl, Node* test, RegionNode* region) { 159 return generate_guard(ctrl, test, region, PROB_FAIR); 160 } 161 162 void PhaseMacroExpand::generate_negative_guard(Node** ctrl, Node* index, RegionNode* region) { 163 if ((*ctrl)->is_top()) 164 return; // already stopped 165 if (_igvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint] 166 return; // index is already adequately typed 167 Node* cmp_lt = new CmpINode(index, intcon(0)); 168 transform_later(cmp_lt); 169 Node* bol_lt = new BoolNode(cmp_lt, BoolTest::lt); 170 transform_later(bol_lt); 171 generate_guard(ctrl, bol_lt, region, PROB_MIN); 172 } 173 174 void PhaseMacroExpand::generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region) { 175 if ((*ctrl)->is_top()) 176 return; // already stopped 177 bool zero_offset = _igvn.type(offset) == TypeInt::ZERO; 178 if (zero_offset && subseq_length->eqv_uncast(array_length)) 179 return; // common case of whole-array copy 180 Node* last = subseq_length; 181 if (!zero_offset) { // last += offset 182 last = new AddINode(last, offset); 183 transform_later(last); 184 } 185 Node* cmp_lt = new CmpUNode(array_length, last); 186 transform_later(cmp_lt); 187 Node* bol_lt = new BoolNode(cmp_lt, BoolTest::lt); 188 transform_later(bol_lt); 189 generate_guard(ctrl, bol_lt, region, PROB_MIN); 190 } 191 192 // 193 // Partial in-lining handling for smaller conjoint/disjoint array copies having 194 // length(in bytes) less than ArrayOperationPartialInlineSize. 195 // if (length <= ArrayOperationPartialInlineSize) { 196 // partial_inlining_block: 197 // mask = Mask_Gen 198 // vload = LoadVectorMasked src , mask 199 // StoreVectorMasked dst, mask, vload 200 // } else { 201 // stub_block: 202 // callstub array_copy 203 // } 204 // exit_block: 205 // Phi = label partial_inlining_block:mem , label stub_block:mem (filled by caller) 206 // mem = MergeMem (Phi) 207 // control = stub_block 208 // 209 // Exit_block and associated phi(memory) are partially initialized for partial_in-lining_block 210 // edges. Remaining edges for exit_block coming from stub_block are connected by the caller 211 // post stub nodes creation. 212 // 213 214 void PhaseMacroExpand::generate_partial_inlining_block(Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type, 215 RegionNode** exit_block, Node** result_memory, Node* length, 216 Node* src_start, Node* dst_start, BasicType type) { 217 const TypePtr *src_adr_type = _igvn.type(src_start)->isa_ptr(); 218 Node* inline_block = nullptr; 219 Node* stub_block = nullptr; 220 221 int const_len = -1; 222 const TypeInt* lty = nullptr; 223 uint shift = exact_log2(type2aelembytes(type)); 224 if (length->Opcode() == Op_ConvI2L) { 225 lty = _igvn.type(length->in(1))->isa_int(); 226 } else { 227 lty = _igvn.type(length)->isa_int(); 228 } 229 if (lty && lty->is_con()) { 230 const_len = lty->get_con() << shift; 231 } 232 233 // Return if copy length is greater than partial inline size limit or 234 // target does not supports masked load/stores. 235 int lane_count = ArrayCopyNode::get_partial_inline_vector_lane_count(type, const_len); 236 if ( const_len > ArrayOperationPartialInlineSize || 237 !Matcher::match_rule_supported_vector(Op_LoadVectorMasked, lane_count, type) || 238 !Matcher::match_rule_supported_vector(Op_StoreVectorMasked, lane_count, type) || 239 !Matcher::match_rule_supported_vector(Op_VectorMaskGen, lane_count, type)) { 240 return; 241 } 242 243 int inline_limit = ArrayOperationPartialInlineSize / type2aelembytes(type); 244 Node* casted_length = new CastLLNode(*ctrl, length, TypeLong::make(0, inline_limit, Type::WidenMin)); 245 transform_later(casted_length); 246 Node* copy_bytes = new LShiftXNode(length, intcon(shift)); 247 transform_later(copy_bytes); 248 249 Node* cmp_le = new CmpULNode(copy_bytes, longcon(ArrayOperationPartialInlineSize)); 250 transform_later(cmp_le); 251 Node* bol_le = new BoolNode(cmp_le, BoolTest::le); 252 transform_later(bol_le); 253 inline_block = generate_guard(ctrl, bol_le, nullptr, PROB_FAIR); 254 stub_block = *ctrl; 255 256 Node* mask_gen = VectorMaskGenNode::make(casted_length, type); 257 transform_later(mask_gen); 258 259 unsigned vec_size = lane_count * type2aelembytes(type); 260 if (C->max_vector_size() < vec_size) { 261 C->set_max_vector_size(vec_size); 262 } 263 264 const TypeVect * vt = TypeVect::make(type, lane_count); 265 Node* mm = (*mem)->memory_at(C->get_alias_index(src_adr_type)); 266 Node* masked_load = new LoadVectorMaskedNode(inline_block, mm, src_start, 267 src_adr_type, vt, mask_gen); 268 transform_later(masked_load); 269 270 mm = (*mem)->memory_at(C->get_alias_index(adr_type)); 271 Node* masked_store = new StoreVectorMaskedNode(inline_block, mm, dst_start, 272 masked_load, adr_type, mask_gen); 273 transform_later(masked_store); 274 275 // Convergence region for inline_block and stub_block. 276 *exit_block = new RegionNode(3); 277 transform_later(*exit_block); 278 (*exit_block)->init_req(1, inline_block); 279 *result_memory = new PhiNode(*exit_block, Type::MEMORY, adr_type); 280 transform_later(*result_memory); 281 (*result_memory)->init_req(1, masked_store); 282 283 *ctrl = stub_block; 284 } 285 286 287 Node* PhaseMacroExpand::generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative) { 288 if ((*ctrl)->is_top()) return nullptr; 289 290 if (_igvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint] 291 return nullptr; // index is already adequately typed 292 Node* cmp_le = new CmpINode(index, intcon(0)); 293 transform_later(cmp_le); 294 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le); 295 Node* bol_le = new BoolNode(cmp_le, le_or_eq); 296 transform_later(bol_le); 297 Node* is_notp = generate_guard(ctrl, bol_le, nullptr, PROB_MIN); 298 299 return is_notp; 300 } 301 302 Node* PhaseMacroExpand::mark_word_test(Node** ctrl, Node* obj, MergeMemNode* mem, uintptr_t mask_val, RegionNode* region) { 303 // Load markword and check if obj is locked 304 Node* mark = make_load(nullptr, mem->memory_at(Compile::AliasIdxRaw), obj, oopDesc::mark_offset_in_bytes(), TypeX_X, TypeX_X->basic_type()); 305 Node* locked_bit = MakeConX(markWord::unlocked_value); 306 locked_bit = transform_later(new AndXNode(locked_bit, mark)); 307 Node* cmp = transform_later(new CmpXNode(locked_bit, MakeConX(0))); 308 Node* is_unlocked = transform_later(new BoolNode(cmp, BoolTest::ne)); 309 IfNode* iff = transform_later(new IfNode(*ctrl, is_unlocked, PROB_MAX, COUNT_UNKNOWN))->as_If(); 310 Node* locked_region = transform_later(new RegionNode(3)); 311 Node* mark_phi = transform_later(new PhiNode(locked_region, TypeX_X)); 312 313 // Unlocked: Use bits from mark word 314 locked_region->init_req(1, transform_later(new IfTrueNode(iff))); 315 mark_phi->init_req(1, mark); 316 317 // Locked: Load prototype header from klass 318 *ctrl = transform_later(new IfFalseNode(iff)); 319 // Make loads control dependent to make sure they are only executed if array is locked 320 Node* klass_adr = basic_plus_adr(obj, oopDesc::klass_offset_in_bytes()); 321 Node* klass = transform_later(LoadKlassNode::make(_igvn, C->immutable_memory(), klass_adr, TypeInstPtr::KLASS, TypeInstKlassPtr::OBJECT)); 322 Node* proto_adr = basic_plus_adr(klass, in_bytes(Klass::prototype_header_offset())); 323 Node* proto = transform_later(LoadNode::make(_igvn, *ctrl, C->immutable_memory(), proto_adr, proto_adr->bottom_type()->is_ptr(), TypeX_X, TypeX_X->basic_type(), MemNode::unordered)); 324 325 locked_region->init_req(2, *ctrl); 326 mark_phi->init_req(2, proto); 327 *ctrl = locked_region; 328 329 // Now check if mark word bits are set 330 Node* mask = MakeConX(mask_val); 331 Node* masked = transform_later(new AndXNode(mark_phi, mask)); 332 cmp = transform_later(new CmpXNode(masked, mask)); 333 Node* bol = transform_later(new BoolNode(cmp, BoolTest::eq)); 334 return generate_fair_guard(ctrl, bol, region); 335 } 336 337 Node* PhaseMacroExpand::generate_flat_array_guard(Node** ctrl, Node* array, MergeMemNode* mem, RegionNode* region) { 338 return mark_word_test(ctrl, array, mem, markWord::flat_array_bit_in_place, region); 339 } 340 341 Node* PhaseMacroExpand::generate_null_free_array_guard(Node** ctrl, Node* array, MergeMemNode* mem, RegionNode* region) { 342 return mark_word_test(ctrl, array, mem, markWord::null_free_array_bit_in_place, region); 343 } 344 345 void PhaseMacroExpand::finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type) { 346 transform_later(call); 347 348 *ctrl = new ProjNode(call,TypeFunc::Control); 349 transform_later(*ctrl); 350 Node* newmem = new ProjNode(call, TypeFunc::Memory); 351 transform_later(newmem); 352 353 uint alias_idx = C->get_alias_index(adr_type); 354 if (alias_idx != Compile::AliasIdxBot) { 355 *mem = MergeMemNode::make(*mem); 356 (*mem)->set_memory_at(alias_idx, newmem); 357 } else { 358 *mem = MergeMemNode::make(newmem); 359 } 360 transform_later(*mem); 361 } 362 363 address PhaseMacroExpand::basictype2arraycopy(BasicType t, 364 Node* src_offset, 365 Node* dest_offset, 366 bool disjoint_bases, 367 const char* &name, 368 bool dest_uninitialized) { 369 const TypeInt* src_offset_inttype = _igvn.find_int_type(src_offset); 370 const TypeInt* dest_offset_inttype = _igvn.find_int_type(dest_offset); 371 372 bool aligned = false; 373 bool disjoint = disjoint_bases; 374 375 // if the offsets are the same, we can treat the memory regions as 376 // disjoint, because either the memory regions are in different arrays, 377 // or they are identical (which we can treat as disjoint.) We can also 378 // treat a copy with a destination index less that the source index 379 // as disjoint since a low->high copy will work correctly in this case. 380 if (src_offset_inttype != nullptr && src_offset_inttype->is_con() && 381 dest_offset_inttype != nullptr && dest_offset_inttype->is_con()) { 382 // both indices are constants 383 int s_offs = src_offset_inttype->get_con(); 384 int d_offs = dest_offset_inttype->get_con(); 385 int element_size = type2aelembytes(t); 386 aligned = ((arrayOopDesc::base_offset_in_bytes(t) + (uint)s_offs * element_size) % HeapWordSize == 0) && 387 ((arrayOopDesc::base_offset_in_bytes(t) + (uint)d_offs * element_size) % HeapWordSize == 0); 388 if (s_offs >= d_offs) disjoint = true; 389 } else if (src_offset == dest_offset && src_offset != nullptr) { 390 // This can occur if the offsets are identical non-constants. 391 disjoint = true; 392 } 393 394 return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized); 395 } 396 397 #define XTOP LP64_ONLY(COMMA top()) 398 399 // Generate an optimized call to arraycopy. 400 // Caller must guard against non-arrays. 401 // Caller must determine a common array basic-type for both arrays. 402 // Caller must validate offsets against array bounds. 403 // The slow_region has already collected guard failure paths 404 // (such as out of bounds length or non-conformable array types). 405 // The generated code has this shape, in general: 406 // 407 // if (length == 0) return // via zero_path 408 // slowval = -1 409 // if (types unknown) { 410 // slowval = call generic copy loop 411 // if (slowval == 0) return // via checked_path 412 // } else if (indexes in bounds) { 413 // if ((is object array) && !(array type check)) { 414 // slowval = call checked copy loop 415 // if (slowval == 0) return // via checked_path 416 // } else { 417 // call bulk copy loop 418 // return // via fast_path 419 // } 420 // } 421 // // adjust params for remaining work: 422 // if (slowval != -1) { 423 // n = -1^slowval; src_offset += n; dest_offset += n; length -= n 424 // } 425 // slow_region: 426 // call slow arraycopy(src, src_offset, dest, dest_offset, length) 427 // return // via slow_call_path 428 // 429 // This routine is used from several intrinsics: System.arraycopy, 430 // Object.clone (the array subcase), and Arrays.copyOf[Range]. 431 // 432 Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode* alloc, 433 Node** ctrl, MergeMemNode* mem, Node** io, 434 const TypePtr* adr_type, 435 BasicType basic_elem_type, 436 Node* src, Node* src_offset, 437 Node* dest, Node* dest_offset, 438 Node* copy_length, 439 Node* dest_length, 440 bool disjoint_bases, 441 bool length_never_negative, 442 RegionNode* slow_region) { 443 if (slow_region == nullptr) { 444 slow_region = new RegionNode(1); 445 transform_later(slow_region); 446 } 447 448 Node* original_dest = dest; 449 bool dest_needs_zeroing = false; 450 bool acopy_to_uninitialized = false; 451 Node* init_value = nullptr; 452 Node* raw_init_value = nullptr; 453 454 // See if this is the initialization of a newly-allocated array. 455 // If so, we will take responsibility here for initializing it to zero. 456 // (Note: Because tightly_coupled_allocation performs checks on the 457 // out-edges of the dest, we need to avoid making derived pointers 458 // from it until we have checked its uses.) 459 if (ReduceBulkZeroing 460 && !(UseTLAB && ZeroTLAB) // pointless if already zeroed 461 && basic_elem_type != T_CONFLICT // avoid corner case 462 && !src->eqv_uncast(dest) 463 && alloc != nullptr 464 && _igvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0) { 465 assert(ac->is_alloc_tightly_coupled(), "sanity"); 466 // acopy to uninitialized tightly coupled allocations 467 // needs zeroing outside the copy range 468 // and the acopy itself will be to uninitialized memory 469 acopy_to_uninitialized = true; 470 if (alloc->maybe_set_complete(&_igvn)) { 471 // "You break it, you buy it." 472 InitializeNode* init = alloc->initialization(); 473 assert(init->is_complete(), "we just did this"); 474 init->set_complete_with_arraycopy(); 475 assert(dest->is_CheckCastPP(), "sanity"); 476 assert(dest->in(0)->in(0) == init, "dest pinned"); 477 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory 478 // From this point on, every exit path is responsible for 479 // initializing any non-copied parts of the object to zero. 480 // Also, if this flag is set we make sure that arraycopy interacts properly 481 // with G1, eliding pre-barriers. See CR 6627983. 482 dest_needs_zeroing = true; 483 init_value = alloc->in(AllocateNode::InitValue); 484 raw_init_value = alloc->in(AllocateNode::RawInitValue); 485 } else { 486 // dest_need_zeroing = false; 487 } 488 } else { 489 // No zeroing elimination needed here. 490 alloc = nullptr; 491 acopy_to_uninitialized = false; 492 //original_dest = dest; 493 //dest_needs_zeroing = false; 494 } 495 496 uint alias_idx = C->get_alias_index(adr_type); 497 498 // Results are placed here: 499 enum { fast_path = 1, // normal void-returning assembly stub 500 checked_path = 2, // special assembly stub with cleanup 501 slow_call_path = 3, // something went wrong; call the VM 502 zero_path = 4, // bypass when length of copy is zero 503 bcopy_path = 5, // copy primitive array by 64-bit blocks 504 PATH_LIMIT = 6 505 }; 506 RegionNode* result_region = new RegionNode(PATH_LIMIT); 507 PhiNode* result_i_o = new PhiNode(result_region, Type::ABIO); 508 PhiNode* result_memory = new PhiNode(result_region, Type::MEMORY, adr_type); 509 assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice"); 510 transform_later(result_region); 511 transform_later(result_i_o); 512 transform_later(result_memory); 513 514 // The slow_control path: 515 Node* slow_control; 516 Node* slow_i_o = *io; 517 Node* slow_mem = mem->memory_at(alias_idx); 518 DEBUG_ONLY(slow_control = (Node*) badAddress); 519 520 // Checked control path: 521 Node* checked_control = top(); 522 Node* checked_mem = nullptr; 523 Node* checked_i_o = nullptr; 524 Node* checked_value = nullptr; 525 526 if (basic_elem_type == T_CONFLICT) { 527 assert(!dest_needs_zeroing, ""); 528 Node* cv = generate_generic_arraycopy(ctrl, &mem, 529 adr_type, 530 src, src_offset, dest, dest_offset, 531 copy_length, acopy_to_uninitialized); 532 if (cv == nullptr) cv = intcon(-1); // failure (no stub available) 533 checked_control = *ctrl; 534 checked_i_o = *io; 535 checked_mem = mem->memory_at(alias_idx); 536 checked_value = cv; 537 *ctrl = top(); 538 } 539 540 Node* not_pos = generate_nonpositive_guard(ctrl, copy_length, length_never_negative); 541 if (not_pos != nullptr) { 542 Node* local_ctrl = not_pos, *local_io = *io; 543 MergeMemNode* local_mem = MergeMemNode::make(mem); 544 transform_later(local_mem); 545 546 // (6) length must not be negative. 547 if (!length_never_negative) { 548 generate_negative_guard(&local_ctrl, copy_length, slow_region); 549 } 550 551 // copy_length is 0. 552 if (dest_needs_zeroing) { 553 assert(!local_ctrl->is_top(), "no ctrl?"); 554 if (copy_length->eqv_uncast(dest_length) 555 || _igvn.find_int_con(dest_length, 1) <= 0) { 556 // There is no zeroing to do. No need for a secondary raw memory barrier. 557 } else { 558 // Clear the whole thing since there are no source elements to copy. 559 generate_clear_array(local_ctrl, local_mem, 560 adr_type, dest, 561 init_value, raw_init_value, 562 basic_elem_type, 563 intcon(0), nullptr, 564 alloc->in(AllocateNode::AllocSize)); 565 // Use a secondary InitializeNode as raw memory barrier. 566 // Currently it is needed only on this path since other 567 // paths have stub or runtime calls as raw memory barriers. 568 MemBarNode* mb = MemBarNode::make(C, Op_Initialize, 569 Compile::AliasIdxRaw, 570 top()); 571 transform_later(mb); 572 mb->set_req(TypeFunc::Control,local_ctrl); 573 mb->set_req(TypeFunc::Memory, local_mem->memory_at(Compile::AliasIdxRaw)); 574 local_ctrl = transform_later(new ProjNode(mb, TypeFunc::Control)); 575 local_mem->set_memory_at(Compile::AliasIdxRaw, transform_later(new ProjNode(mb, TypeFunc::Memory))); 576 577 InitializeNode* init = mb->as_Initialize(); 578 init->set_complete(&_igvn); // (there is no corresponding AllocateNode) 579 } 580 } 581 582 // Present the results of the fast call. 583 result_region->init_req(zero_path, local_ctrl); 584 result_i_o ->init_req(zero_path, local_io); 585 result_memory->init_req(zero_path, local_mem->memory_at(alias_idx)); 586 } 587 588 if (!(*ctrl)->is_top() && dest_needs_zeroing) { 589 // We have to initialize the *uncopied* part of the array to zero. 590 // The copy destination is the slice dest[off..off+len]. The other slices 591 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length]. 592 Node* dest_size = alloc->in(AllocateNode::AllocSize); 593 Node* dest_tail = transform_later( new AddINode(dest_offset, copy_length)); 594 595 // If there is a head section that needs zeroing, do it now. 596 if (_igvn.find_int_con(dest_offset, -1) != 0) { 597 generate_clear_array(*ctrl, mem, 598 adr_type, dest, 599 init_value, raw_init_value, 600 basic_elem_type, 601 intcon(0), dest_offset, 602 nullptr); 603 } 604 605 // Next, perform a dynamic check on the tail length. 606 // It is often zero, and we can win big if we prove this. 607 // There are two wins: Avoid generating the ClearArray 608 // with its attendant messy index arithmetic, and upgrade 609 // the copy to a more hardware-friendly word size of 64 bits. 610 Node* tail_ctl = nullptr; 611 if (!(*ctrl)->is_top() && !dest_tail->eqv_uncast(dest_length)) { 612 Node* cmp_lt = transform_later( new CmpINode(dest_tail, dest_length) ); 613 Node* bol_lt = transform_later( new BoolNode(cmp_lt, BoolTest::lt) ); 614 tail_ctl = generate_slow_guard(ctrl, bol_lt, nullptr); 615 assert(tail_ctl != nullptr || !(*ctrl)->is_top(), "must be an outcome"); 616 } 617 618 // At this point, let's assume there is no tail. 619 if (!(*ctrl)->is_top() && alloc != nullptr && basic_elem_type != T_OBJECT) { 620 // There is no tail. Try an upgrade to a 64-bit copy. 621 bool didit = false; 622 { 623 Node* local_ctrl = *ctrl, *local_io = *io; 624 MergeMemNode* local_mem = MergeMemNode::make(mem); 625 transform_later(local_mem); 626 627 didit = generate_block_arraycopy(&local_ctrl, &local_mem, local_io, 628 adr_type, basic_elem_type, alloc, 629 src, src_offset, dest, dest_offset, 630 dest_size, acopy_to_uninitialized); 631 if (didit) { 632 // Present the results of the block-copying fast call. 633 result_region->init_req(bcopy_path, local_ctrl); 634 result_i_o ->init_req(bcopy_path, local_io); 635 result_memory->init_req(bcopy_path, local_mem->memory_at(alias_idx)); 636 } 637 } 638 if (didit) { 639 *ctrl = top(); // no regular fast path 640 } 641 } 642 643 // Clear the tail, if any. 644 if (tail_ctl != nullptr) { 645 Node* notail_ctl = (*ctrl)->is_top() ? nullptr : *ctrl; 646 *ctrl = tail_ctl; 647 if (notail_ctl == nullptr) { 648 generate_clear_array(*ctrl, mem, 649 adr_type, dest, 650 init_value, raw_init_value, 651 basic_elem_type, 652 dest_tail, nullptr, 653 dest_size); 654 } else { 655 // Make a local merge. 656 Node* done_ctl = transform_later(new RegionNode(3)); 657 Node* done_mem = transform_later(new PhiNode(done_ctl, Type::MEMORY, adr_type)); 658 done_ctl->init_req(1, notail_ctl); 659 done_mem->init_req(1, mem->memory_at(alias_idx)); 660 generate_clear_array(*ctrl, mem, 661 adr_type, dest, 662 init_value, raw_init_value, 663 basic_elem_type, 664 dest_tail, nullptr, 665 dest_size); 666 done_ctl->init_req(2, *ctrl); 667 done_mem->init_req(2, mem->memory_at(alias_idx)); 668 *ctrl = done_ctl; 669 mem->set_memory_at(alias_idx, done_mem); 670 } 671 } 672 } 673 674 BasicType copy_type = basic_elem_type; 675 assert(basic_elem_type != T_ARRAY, "caller must fix this"); 676 if (!(*ctrl)->is_top() && copy_type == T_OBJECT) { 677 // If src and dest have compatible element types, we can copy bits. 678 // Types S[] and D[] are compatible if D is a supertype of S. 679 // 680 // If they are not, we will use checked_oop_disjoint_arraycopy, 681 // which performs a fast optimistic per-oop check, and backs off 682 // further to JVM_ArrayCopy on the first per-oop check that fails. 683 // (Actually, we don't move raw bits only; the GC requires card marks.) 684 685 // We don't need a subtype check for validated copies and Object[].clone() 686 bool skip_subtype_check = ac->is_arraycopy_validated() || ac->is_copyof_validated() || 687 ac->is_copyofrange_validated() || ac->is_clone_oop_array(); 688 if (!skip_subtype_check) { 689 // Get the klass* for both src and dest 690 Node* src_klass = ac->in(ArrayCopyNode::SrcKlass); 691 Node* dest_klass = ac->in(ArrayCopyNode::DestKlass); 692 693 assert(src_klass != nullptr && dest_klass != nullptr, "should have klasses"); 694 695 // Generate the subtype check. 696 // This might fold up statically, or then again it might not. 697 // 698 // Non-static example: Copying List<String>.elements to a new String[]. 699 // The backing store for a List<String> is always an Object[], 700 // but its elements are always type String, if the generic types 701 // are correct at the source level. 702 // 703 // Test S[] against D[], not S against D, because (probably) 704 // the secondary supertype cache is less busy for S[] than S. 705 // This usually only matters when D is an interface. 706 Node* not_subtype_ctrl = Phase::gen_subtype_check(src_klass, dest_klass, ctrl, mem, _igvn, nullptr, -1); 707 // Plug failing path into checked_oop_disjoint_arraycopy 708 if (not_subtype_ctrl != top()) { 709 Node* local_ctrl = not_subtype_ctrl; 710 MergeMemNode* local_mem = MergeMemNode::make(mem); 711 transform_later(local_mem); 712 713 // (At this point we can assume disjoint_bases, since types differ.) 714 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset()); 715 Node* p1 = basic_plus_adr(dest_klass, ek_offset); 716 Node* n1 = LoadKlassNode::make(_igvn, C->immutable_memory(), p1, TypeRawPtr::BOTTOM); 717 Node* dest_elem_klass = transform_later(n1); 718 Node* cv = generate_checkcast_arraycopy(&local_ctrl, &local_mem, 719 adr_type, 720 dest_elem_klass, 721 src, src_offset, dest, dest_offset, 722 ConvI2X(copy_length), acopy_to_uninitialized); 723 if (cv == nullptr) cv = intcon(-1); // failure (no stub available) 724 checked_control = local_ctrl; 725 checked_i_o = *io; 726 checked_mem = local_mem->memory_at(alias_idx); 727 checked_value = cv; 728 } 729 } 730 // At this point we know we do not need type checks on oop stores. 731 732 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 733 if (!bs->array_copy_requires_gc_barriers(alloc != nullptr, copy_type, false, false, BarrierSetC2::Expansion)) { 734 // If we do not need gc barriers, copy using the jint or jlong stub. 735 copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT); 736 assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type), 737 "sizes agree"); 738 } 739 } 740 741 if (!(*ctrl)->is_top()) { 742 // Generate the fast path, if possible. 743 Node* local_ctrl = *ctrl; 744 MergeMemNode* local_mem = MergeMemNode::make(mem); 745 transform_later(local_mem); 746 generate_unchecked_arraycopy(&local_ctrl, &local_mem, 747 adr_type, copy_type, disjoint_bases, 748 src, src_offset, dest, dest_offset, 749 ConvI2X(copy_length), acopy_to_uninitialized); 750 751 // Present the results of the fast call. 752 result_region->init_req(fast_path, local_ctrl); 753 result_i_o ->init_req(fast_path, *io); 754 result_memory->init_req(fast_path, local_mem->memory_at(alias_idx)); 755 } 756 757 // Here are all the slow paths up to this point, in one bundle: 758 assert(slow_region != nullptr, "allocated on entry"); 759 slow_control = slow_region; 760 DEBUG_ONLY(slow_region = (RegionNode*)badAddress); 761 762 *ctrl = checked_control; 763 if (!(*ctrl)->is_top()) { 764 // Clean up after the checked call. 765 // The returned value is either 0 or -1^K, 766 // where K = number of partially transferred array elements. 767 Node* cmp = new CmpINode(checked_value, intcon(0)); 768 transform_later(cmp); 769 Node* bol = new BoolNode(cmp, BoolTest::eq); 770 transform_later(bol); 771 IfNode* iff = new IfNode(*ctrl, bol, PROB_MAX, COUNT_UNKNOWN); 772 transform_later(iff); 773 774 // If it is 0, we are done, so transfer to the end. 775 Node* checks_done = new IfTrueNode(iff); 776 transform_later(checks_done); 777 result_region->init_req(checked_path, checks_done); 778 result_i_o ->init_req(checked_path, checked_i_o); 779 result_memory->init_req(checked_path, checked_mem); 780 781 // If it is not zero, merge into the slow call. 782 *ctrl = new IfFalseNode(iff); 783 transform_later(*ctrl); 784 RegionNode* slow_reg2 = new RegionNode(3); 785 PhiNode* slow_i_o2 = new PhiNode(slow_reg2, Type::ABIO); 786 PhiNode* slow_mem2 = new PhiNode(slow_reg2, Type::MEMORY, adr_type); 787 transform_later(slow_reg2); 788 transform_later(slow_i_o2); 789 transform_later(slow_mem2); 790 slow_reg2 ->init_req(1, slow_control); 791 slow_i_o2 ->init_req(1, slow_i_o); 792 slow_mem2 ->init_req(1, slow_mem); 793 slow_reg2 ->init_req(2, *ctrl); 794 slow_i_o2 ->init_req(2, checked_i_o); 795 slow_mem2 ->init_req(2, checked_mem); 796 797 slow_control = slow_reg2; 798 slow_i_o = slow_i_o2; 799 slow_mem = slow_mem2; 800 801 if (alloc != nullptr) { 802 // We'll restart from the very beginning, after zeroing the whole thing. 803 // This can cause double writes, but that's OK since dest is brand new. 804 // So we ignore the low 31 bits of the value returned from the stub. 805 } else { 806 // We must continue the copy exactly where it failed, or else 807 // another thread might see the wrong number of writes to dest. 808 Node* checked_offset = new XorINode(checked_value, intcon(-1)); 809 Node* slow_offset = new PhiNode(slow_reg2, TypeInt::INT); 810 transform_later(checked_offset); 811 transform_later(slow_offset); 812 slow_offset->init_req(1, intcon(0)); 813 slow_offset->init_req(2, checked_offset); 814 815 // Adjust the arguments by the conditionally incoming offset. 816 Node* src_off_plus = new AddINode(src_offset, slow_offset); 817 transform_later(src_off_plus); 818 Node* dest_off_plus = new AddINode(dest_offset, slow_offset); 819 transform_later(dest_off_plus); 820 Node* length_minus = new SubINode(copy_length, slow_offset); 821 transform_later(length_minus); 822 823 // Tweak the node variables to adjust the code produced below: 824 src_offset = src_off_plus; 825 dest_offset = dest_off_plus; 826 copy_length = length_minus; 827 } 828 } 829 *ctrl = slow_control; 830 if (!(*ctrl)->is_top()) { 831 Node* local_ctrl = *ctrl, *local_io = slow_i_o; 832 MergeMemNode* local_mem = MergeMemNode::make(mem); 833 transform_later(local_mem); 834 835 // Generate the slow path, if needed. 836 local_mem->set_memory_at(alias_idx, slow_mem); 837 838 if (dest_needs_zeroing) { 839 generate_clear_array(local_ctrl, local_mem, 840 adr_type, dest, 841 init_value, raw_init_value, 842 basic_elem_type, 843 intcon(0), nullptr, 844 alloc->in(AllocateNode::AllocSize)); 845 } 846 847 local_mem = generate_slow_arraycopy(ac, 848 &local_ctrl, local_mem, &local_io, 849 adr_type, 850 src, src_offset, dest, dest_offset, 851 copy_length, /*dest_uninitialized*/false); 852 853 result_region->init_req(slow_call_path, local_ctrl); 854 result_i_o ->init_req(slow_call_path, local_io); 855 result_memory->init_req(slow_call_path, local_mem->memory_at(alias_idx)); 856 } else { 857 ShouldNotReachHere(); // no call to generate_slow_arraycopy: 858 // projections were not extracted 859 } 860 861 // Remove unused edges. 862 for (uint i = 1; i < result_region->req(); i++) { 863 if (result_region->in(i) == nullptr) { 864 result_region->init_req(i, top()); 865 } 866 } 867 868 // Finished; return the combined state. 869 *ctrl = result_region; 870 *io = result_i_o; 871 mem->set_memory_at(alias_idx, result_memory); 872 873 // mem no longer guaranteed to stay a MergeMemNode 874 Node* out_mem = mem; 875 DEBUG_ONLY(mem = nullptr); 876 877 // The memory edges above are precise in order to model effects around 878 // array copies accurately to allow value numbering of field loads around 879 // arraycopy. Such field loads, both before and after, are common in Java 880 // collections and similar classes involving header/array data structures. 881 // 882 // But with low number of register or when some registers are used or killed 883 // by arraycopy calls it causes registers spilling on stack. See 6544710. 884 // The next memory barrier is added to avoid it. If the arraycopy can be 885 // optimized away (which it can, sometimes) then we can manually remove 886 // the membar also. 887 // 888 // Do not let reads from the cloned object float above the arraycopy. 889 if (alloc != nullptr && !alloc->initialization()->does_not_escape()) { 890 // Do not let stores that initialize this object be reordered with 891 // a subsequent store that would make this object accessible by 892 // other threads. 893 assert(ac->_dest_type == TypeOopPtr::BOTTOM, "non escaping destination shouldn't have narrow slice"); 894 insert_mem_bar(ctrl, &out_mem, Op_MemBarStoreStore, Compile::AliasIdxBot); 895 } else { 896 int mem_bar_alias_idx = Compile::AliasIdxBot; 897 if (ac->_dest_type != TypeOopPtr::BOTTOM) { 898 // The graph was transformed under the assumption the ArrayCopy node only had an effect on a narrow slice. We can't 899 // insert a wide membar now that it's being expanded: a load that uses the input memory state of the ArrayCopy 900 // could then become anti dependent on the membar when it was not anti dependent on the ArrayCopy leading to a 901 // broken graph. 902 mem_bar_alias_idx = C->get_alias_index(ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr()); 903 } 904 insert_mem_bar(ctrl, &out_mem, Op_MemBarCPUOrder, mem_bar_alias_idx); 905 } 906 907 assert((*ctrl)->is_Proj(), "MemBar control projection"); 908 assert((*ctrl)->in(0)->isa_MemBar(), "MemBar node"); 909 (*ctrl)->in(0)->isa_MemBar()->set_trailing_expanded_array_copy(); 910 911 _igvn.replace_node(_callprojs->fallthrough_memproj, out_mem); 912 if (_callprojs->fallthrough_ioproj != nullptr) { 913 _igvn.replace_node(_callprojs->fallthrough_ioproj, *io); 914 } 915 _igvn.replace_node(_callprojs->fallthrough_catchproj, *ctrl); 916 917 #ifdef ASSERT 918 const TypeOopPtr* dest_t = _igvn.type(dest)->is_oopptr(); 919 if (dest_t->is_known_instance()) { 920 ArrayCopyNode* ac = nullptr; 921 assert(ArrayCopyNode::may_modify(dest_t, (*ctrl)->in(0)->as_MemBar(), &_igvn, ac), "dependency on arraycopy lost"); 922 assert(ac == nullptr, "no arraycopy anymore"); 923 } 924 #endif 925 926 return out_mem; 927 } 928 929 // Helper for initialization of arrays, creating a ClearArray. 930 // It writes zero bits in [start..end), within the body of an array object. 931 // The memory effects are all chained onto the 'adr_type' alias category. 932 // 933 // Since the object is otherwise uninitialized, we are free 934 // to put a little "slop" around the edges of the cleared area, 935 // as long as it does not go back into the array's header, 936 // or beyond the array end within the heap. 937 // 938 // The lower edge can be rounded down to the nearest jint and the 939 // upper edge can be rounded up to the nearest MinObjAlignmentInBytes. 940 // 941 // Arguments: 942 // adr_type memory slice where writes are generated 943 // dest oop of the destination array 944 // basic_elem_type element type of the destination 945 // slice_idx array index of first element to store 946 // slice_len number of elements to store (or null) 947 // dest_size total size in bytes of the array object 948 // 949 // Exactly one of slice_len or dest_size must be non-null. 950 // If dest_size is non-null, zeroing extends to the end of the object. 951 // If slice_len is non-null, the slice_idx value must be a constant. 952 void PhaseMacroExpand::generate_clear_array(Node* ctrl, MergeMemNode* merge_mem, 953 const TypePtr* adr_type, 954 Node* dest, 955 Node* val, 956 Node* raw_val, 957 BasicType basic_elem_type, 958 Node* slice_idx, 959 Node* slice_len, 960 Node* dest_size) { 961 // one or the other but not both of slice_len and dest_size: 962 assert((slice_len != nullptr? 1: 0) + (dest_size != nullptr? 1: 0) == 1, ""); 963 if (slice_len == nullptr) slice_len = top(); 964 if (dest_size == nullptr) dest_size = top(); 965 966 uint alias_idx = C->get_alias_index(adr_type); 967 968 // operate on this memory slice: 969 Node* mem = merge_mem->memory_at(alias_idx); // memory slice to operate on 970 971 // scaling and rounding of indexes: 972 int scale = exact_log2(type2aelembytes(basic_elem_type)); 973 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type); 974 int clear_low = (-1 << scale) & (BytesPerInt - 1); 975 int bump_bit = (-1 << scale) & BytesPerInt; 976 977 // determine constant starts and ends 978 const intptr_t BIG_NEG = -128; 979 assert(BIG_NEG + 2*abase < 0, "neg enough"); 980 intptr_t slice_idx_con = (intptr_t) _igvn.find_int_con(slice_idx, BIG_NEG); 981 intptr_t slice_len_con = (intptr_t) _igvn.find_int_con(slice_len, BIG_NEG); 982 if (slice_len_con == 0) { 983 return; // nothing to do here 984 } 985 intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low; 986 intptr_t end_con = _igvn.find_intptr_t_con(dest_size, -1); 987 if (slice_idx_con >= 0 && slice_len_con >= 0) { 988 assert(end_con < 0, "not two cons"); 989 end_con = align_up(abase + ((slice_idx_con + slice_len_con) << scale), 990 BytesPerLong); 991 } 992 993 if (start_con >= 0 && end_con >= 0) { 994 // Constant start and end. Simple. 995 mem = ClearArrayNode::clear_memory(ctrl, mem, dest, val, raw_val, 996 start_con, end_con, &_igvn); 997 } else if (start_con >= 0 && dest_size != top()) { 998 // Constant start, pre-rounded end after the tail of the array. 999 Node* end = dest_size; 1000 mem = ClearArrayNode::clear_memory(ctrl, mem, dest, val, raw_val, 1001 start_con, end, &_igvn); 1002 } else if (start_con >= 0 && slice_len != top()) { 1003 // Constant start, non-constant end. End needs rounding up. 1004 // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8) 1005 intptr_t end_base = abase + (slice_idx_con << scale); 1006 int end_round = (-1 << scale) & (BytesPerLong - 1); 1007 Node* end = ConvI2X(slice_len); 1008 if (scale != 0) 1009 end = transform_later(new LShiftXNode(end, intcon(scale) )); 1010 end_base += end_round; 1011 end = transform_later(new AddXNode(end, MakeConX(end_base)) ); 1012 end = transform_later(new AndXNode(end, MakeConX(~end_round)) ); 1013 mem = ClearArrayNode::clear_memory(ctrl, mem, dest, val, raw_val, 1014 start_con, end, &_igvn); 1015 } else if (start_con < 0 && dest_size != top()) { 1016 // Non-constant start, pre-rounded end after the tail of the array. 1017 // This is almost certainly a "round-to-end" operation. 1018 Node* start = slice_idx; 1019 start = ConvI2X(start); 1020 if (scale != 0) 1021 start = transform_later(new LShiftXNode( start, intcon(scale) )); 1022 start = transform_later(new AddXNode(start, MakeConX(abase)) ); 1023 if ((bump_bit | clear_low) != 0) { 1024 int to_clear = (bump_bit | clear_low); 1025 // Align up mod 8, then store a jint zero unconditionally 1026 // just before the mod-8 boundary. 1027 if (((abase + bump_bit) & ~to_clear) - bump_bit 1028 < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) { 1029 bump_bit = 0; 1030 assert((abase & to_clear) == 0, "array base must be long-aligned"); 1031 } else { 1032 // Bump 'start' up to (or past) the next jint boundary: 1033 start = transform_later( new AddXNode(start, MakeConX(bump_bit)) ); 1034 assert((abase & clear_low) == 0, "array base must be int-aligned"); 1035 } 1036 // Round bumped 'start' down to jlong boundary in body of array. 1037 start = transform_later(new AndXNode(start, MakeConX(~to_clear)) ); 1038 if (bump_bit != 0) { 1039 // Store a zero to the immediately preceding jint: 1040 Node* x1 = transform_later(new AddXNode(start, MakeConX(-bump_bit)) ); 1041 Node* p1 = basic_plus_adr(dest, x1); 1042 if (val == nullptr) { 1043 assert(raw_val == nullptr, "val may not be null"); 1044 mem = StoreNode::make(_igvn, ctrl, mem, p1, adr_type, intcon(0), T_INT, MemNode::unordered); 1045 } else { 1046 assert(_igvn.type(val)->isa_narrowoop(), "should be narrow oop"); 1047 mem = new StoreNNode(ctrl, mem, p1, adr_type, val, MemNode::unordered); 1048 } 1049 mem = transform_later(mem); 1050 } 1051 } 1052 Node* end = dest_size; // pre-rounded 1053 mem = ClearArrayNode::clear_memory(ctrl, mem, dest, raw_val, 1054 start, end, &_igvn); 1055 } else { 1056 // Non-constant start, unrounded non-constant end. 1057 // (Nobody zeroes a random midsection of an array using this routine.) 1058 ShouldNotReachHere(); // fix caller 1059 } 1060 1061 // Done. 1062 merge_mem->set_memory_at(alias_idx, mem); 1063 } 1064 1065 bool PhaseMacroExpand::generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io, 1066 const TypePtr* adr_type, 1067 BasicType basic_elem_type, 1068 AllocateNode* alloc, 1069 Node* src, Node* src_offset, 1070 Node* dest, Node* dest_offset, 1071 Node* dest_size, bool dest_uninitialized) { 1072 // See if there is an advantage from block transfer. 1073 int scale = exact_log2(type2aelembytes(basic_elem_type)); 1074 if (scale >= LogBytesPerLong) 1075 return false; // it is already a block transfer 1076 1077 // Look at the alignment of the starting offsets. 1078 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type); 1079 1080 intptr_t src_off_con = (intptr_t) _igvn.find_int_con(src_offset, -1); 1081 intptr_t dest_off_con = (intptr_t) _igvn.find_int_con(dest_offset, -1); 1082 if (src_off_con < 0 || dest_off_con < 0) { 1083 // At present, we can only understand constants. 1084 return false; 1085 } 1086 1087 intptr_t src_off = abase + (src_off_con << scale); 1088 intptr_t dest_off = abase + (dest_off_con << scale); 1089 1090 if (((src_off | dest_off) & (BytesPerLong-1)) != 0) { 1091 // Non-aligned; too bad. 1092 // One more chance: Pick off an initial 32-bit word. 1093 // This is a common case, since abase can be odd mod 8. 1094 if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt && 1095 ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) { 1096 Node* sptr = basic_plus_adr(src, src_off); 1097 Node* dptr = basic_plus_adr(dest, dest_off); 1098 const TypePtr* s_adr_type = _igvn.type(sptr)->is_ptr(); 1099 assert(s_adr_type->isa_aryptr(), "impossible slice"); 1100 uint s_alias_idx = C->get_alias_index(s_adr_type); 1101 uint d_alias_idx = C->get_alias_index(adr_type); 1102 bool is_mismatched = (basic_elem_type != T_INT); 1103 Node* sval = transform_later( 1104 LoadNode::make(_igvn, *ctrl, (*mem)->memory_at(s_alias_idx), sptr, s_adr_type, 1105 TypeInt::INT, T_INT, MemNode::unordered, LoadNode::DependsOnlyOnTest, 1106 false /*require_atomic_access*/, false /*unaligned*/, is_mismatched)); 1107 Node* st = transform_later( 1108 StoreNode::make(_igvn, *ctrl, (*mem)->memory_at(d_alias_idx), dptr, adr_type, 1109 sval, T_INT, MemNode::unordered)); 1110 if (is_mismatched) { 1111 st->as_Store()->set_mismatched_access(); 1112 } 1113 (*mem)->set_memory_at(d_alias_idx, st); 1114 src_off += BytesPerInt; 1115 dest_off += BytesPerInt; 1116 } else { 1117 return false; 1118 } 1119 } 1120 assert(src_off % BytesPerLong == 0, ""); 1121 assert(dest_off % BytesPerLong == 0, ""); 1122 1123 // Do this copy by giant steps. 1124 Node* sptr = basic_plus_adr(src, src_off); 1125 Node* dptr = basic_plus_adr(dest, dest_off); 1126 Node* countx = dest_size; 1127 countx = transform_later(new SubXNode(countx, MakeConX(dest_off))); 1128 countx = transform_later(new URShiftXNode(countx, intcon(LogBytesPerLong))); 1129 1130 bool disjoint_bases = true; // since alloc isn't null 1131 generate_unchecked_arraycopy(ctrl, mem, 1132 adr_type, T_LONG, disjoint_bases, 1133 sptr, nullptr, dptr, nullptr, countx, dest_uninitialized); 1134 1135 return true; 1136 } 1137 1138 // Helper function; generates code for the slow case. 1139 // We make a call to a runtime method which emulates the native method, 1140 // but without the native wrapper overhead. 1141 MergeMemNode* PhaseMacroExpand::generate_slow_arraycopy(ArrayCopyNode *ac, 1142 Node** ctrl, Node* mem, Node** io, 1143 const TypePtr* adr_type, 1144 Node* src, Node* src_offset, 1145 Node* dest, Node* dest_offset, 1146 Node* copy_length, bool dest_uninitialized) { 1147 assert(!dest_uninitialized, "Invariant"); 1148 1149 const TypeFunc* call_type = OptoRuntime::slow_arraycopy_Type(); 1150 CallNode* call = new CallStaticJavaNode(call_type, OptoRuntime::slow_arraycopy_Java(), 1151 "slow_arraycopy", TypePtr::BOTTOM); 1152 1153 call->init_req(TypeFunc::Control, *ctrl); 1154 call->init_req(TypeFunc::I_O , *io); 1155 call->init_req(TypeFunc::Memory , mem); 1156 call->init_req(TypeFunc::ReturnAdr, top()); 1157 call->init_req(TypeFunc::FramePtr, top()); 1158 call->init_req(TypeFunc::Parms+0, src); 1159 call->init_req(TypeFunc::Parms+1, src_offset); 1160 call->init_req(TypeFunc::Parms+2, dest); 1161 call->init_req(TypeFunc::Parms+3, dest_offset); 1162 call->init_req(TypeFunc::Parms+4, copy_length); 1163 call->copy_call_debug_info(&_igvn, ac); 1164 1165 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON. 1166 _igvn.replace_node(ac, call); 1167 transform_later(call); 1168 1169 _callprojs = call->extract_projections(false /*separate_io_proj*/, false /*do_asserts*/); 1170 *ctrl = _callprojs->fallthrough_catchproj->clone(); 1171 transform_later(*ctrl); 1172 1173 Node* m = _callprojs->fallthrough_memproj->clone(); 1174 transform_later(m); 1175 1176 uint alias_idx = C->get_alias_index(adr_type); 1177 MergeMemNode* out_mem; 1178 if (alias_idx != Compile::AliasIdxBot) { 1179 out_mem = MergeMemNode::make(mem); 1180 out_mem->set_memory_at(alias_idx, m); 1181 } else { 1182 out_mem = MergeMemNode::make(m); 1183 } 1184 transform_later(out_mem); 1185 1186 // When src is negative and arraycopy is before an infinite loop,_callprojs.fallthrough_ioproj 1187 // could be nullptr. Skip clone and update nullptr fallthrough_ioproj. 1188 if (_callprojs->fallthrough_ioproj != nullptr) { 1189 *io = _callprojs->fallthrough_ioproj->clone(); 1190 transform_later(*io); 1191 } else { 1192 *io = nullptr; 1193 } 1194 1195 return out_mem; 1196 } 1197 1198 // Helper function; generates code for cases requiring runtime checks. 1199 Node* PhaseMacroExpand::generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem, 1200 const TypePtr* adr_type, 1201 Node* dest_elem_klass, 1202 Node* src, Node* src_offset, 1203 Node* dest, Node* dest_offset, 1204 Node* copy_length, bool dest_uninitialized) { 1205 if ((*ctrl)->is_top()) return nullptr; 1206 1207 address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized); 1208 if (copyfunc_addr == nullptr) { // Stub was not generated, go slow path. 1209 return nullptr; 1210 } 1211 1212 // Pick out the parameters required to perform a store-check 1213 // for the target array. This is an optimistic check. It will 1214 // look in each non-null element's class, at the desired klass's 1215 // super_check_offset, for the desired klass. 1216 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 1217 Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset); 1218 Node* n3 = new LoadINode(nullptr, *mem /*memory(p3)*/, p3, _igvn.type(p3)->is_ptr(), TypeInt::INT, MemNode::unordered); 1219 Node* check_offset = ConvI2X(transform_later(n3)); 1220 Node* check_value = dest_elem_klass; 1221 1222 Node* src_start = array_element_address(src, src_offset, T_OBJECT); 1223 Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT); 1224 1225 const TypeFunc* call_type = OptoRuntime::checkcast_arraycopy_Type(); 1226 Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "checkcast_arraycopy", adr_type, 1227 src_start, dest_start, copy_length XTOP, check_offset XTOP, check_value); 1228 1229 finish_arraycopy_call(call, ctrl, mem, adr_type); 1230 1231 Node* proj = new ProjNode(call, TypeFunc::Parms); 1232 transform_later(proj); 1233 1234 return proj; 1235 } 1236 1237 // Helper function; generates code for cases requiring runtime checks. 1238 Node* PhaseMacroExpand::generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem, 1239 const TypePtr* adr_type, 1240 Node* src, Node* src_offset, 1241 Node* dest, Node* dest_offset, 1242 Node* copy_length, bool dest_uninitialized) { 1243 if ((*ctrl)->is_top()) return nullptr; 1244 assert(!dest_uninitialized, "Invariant"); 1245 1246 address copyfunc_addr = StubRoutines::generic_arraycopy(); 1247 if (copyfunc_addr == nullptr) { // Stub was not generated, go slow path. 1248 return nullptr; 1249 } 1250 1251 const TypeFunc* call_type = OptoRuntime::generic_arraycopy_Type(); 1252 Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, "generic_arraycopy", adr_type, 1253 src, src_offset, dest, dest_offset, copy_length); 1254 1255 finish_arraycopy_call(call, ctrl, mem, adr_type); 1256 1257 Node* proj = new ProjNode(call, TypeFunc::Parms); 1258 transform_later(proj); 1259 1260 return proj; 1261 } 1262 1263 // Helper function; generates the fast out-of-line call to an arraycopy stub. 1264 void PhaseMacroExpand::generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem, 1265 const TypePtr* adr_type, 1266 BasicType basic_elem_type, 1267 bool disjoint_bases, 1268 Node* src, Node* src_offset, 1269 Node* dest, Node* dest_offset, 1270 Node* copy_length, bool dest_uninitialized) { 1271 if ((*ctrl)->is_top()) { 1272 return; 1273 } 1274 1275 Node* src_start = src; 1276 Node* dest_start = dest; 1277 if (src_offset != nullptr || dest_offset != nullptr) { 1278 src_start = array_element_address(src, src_offset, basic_elem_type); 1279 dest_start = array_element_address(dest, dest_offset, basic_elem_type); 1280 } 1281 1282 // Figure out which arraycopy runtime method to call. 1283 const char* copyfunc_name = "arraycopy"; 1284 address copyfunc_addr = 1285 basictype2arraycopy(basic_elem_type, src_offset, dest_offset, 1286 disjoint_bases, copyfunc_name, dest_uninitialized); 1287 1288 Node* result_memory = nullptr; 1289 RegionNode* exit_block = nullptr; 1290 if (ArrayOperationPartialInlineSize > 0 && is_subword_type(basic_elem_type) && 1291 Matcher::vector_width_in_bytes(basic_elem_type) >= 16) { 1292 generate_partial_inlining_block(ctrl, mem, adr_type, &exit_block, &result_memory, 1293 copy_length, src_start, dest_start, basic_elem_type); 1294 } 1295 1296 const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type(); 1297 Node* call = make_leaf_call(*ctrl, *mem, call_type, copyfunc_addr, copyfunc_name, adr_type, 1298 src_start, dest_start, copy_length XTOP); 1299 1300 finish_arraycopy_call(call, ctrl, mem, adr_type); 1301 1302 // Connecting remaining edges for exit_block coming from stub_block. 1303 if (exit_block) { 1304 exit_block->init_req(2, *ctrl); 1305 1306 // Memory edge corresponding to stub_region. 1307 result_memory->init_req(2, *mem); 1308 1309 uint alias_idx = C->get_alias_index(adr_type); 1310 if (alias_idx != Compile::AliasIdxBot) { 1311 *mem = MergeMemNode::make(*mem); 1312 (*mem)->set_memory_at(alias_idx, result_memory); 1313 } else { 1314 *mem = MergeMemNode::make(result_memory); 1315 } 1316 transform_later(*mem); 1317 *ctrl = exit_block; 1318 } 1319 } 1320 1321 const TypePtr* PhaseMacroExpand::adjust_for_flat_array(const TypeAryPtr* top_dest, Node*& src_offset, 1322 Node*& dest_offset, Node*& length, BasicType& dest_elem, 1323 Node*& dest_length) { 1324 #ifdef ASSERT 1325 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 1326 bool needs_barriers = top_dest->elem()->inline_klass()->contains_oops() && 1327 bs->array_copy_requires_gc_barriers(dest_length != nullptr, T_OBJECT, false, false, BarrierSetC2::Optimization); 1328 assert(!needs_barriers || StressReflectiveCode, "Flat arracopy would require GC barriers"); 1329 #endif 1330 int elem_size = top_dest->flat_elem_size(); 1331 if (elem_size >= 8) { 1332 if (elem_size > 8) { 1333 // treat as array of long but scale length, src offset and dest offset 1334 assert((elem_size % 8) == 0, "not a power of 2?"); 1335 int factor = elem_size / 8; 1336 length = transform_later(new MulINode(length, intcon(factor))); 1337 src_offset = transform_later(new MulINode(src_offset, intcon(factor))); 1338 dest_offset = transform_later(new MulINode(dest_offset, intcon(factor))); 1339 if (dest_length != nullptr) { 1340 dest_length = transform_later(new MulINode(dest_length, intcon(factor))); 1341 } 1342 elem_size = 8; 1343 } 1344 dest_elem = T_LONG; 1345 } else if (elem_size == 4) { 1346 dest_elem = T_INT; 1347 } else if (elem_size == 2) { 1348 dest_elem = T_CHAR; 1349 } else if (elem_size == 1) { 1350 dest_elem = T_BYTE; 1351 } else { 1352 ShouldNotReachHere(); 1353 } 1354 return TypeRawPtr::BOTTOM; 1355 } 1356 1357 #undef XTOP 1358 1359 void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) { 1360 Node* ctrl = ac->in(TypeFunc::Control); 1361 Node* io = ac->in(TypeFunc::I_O); 1362 Node* src = ac->in(ArrayCopyNode::Src); 1363 Node* src_offset = ac->in(ArrayCopyNode::SrcPos); 1364 Node* dest = ac->in(ArrayCopyNode::Dest); 1365 Node* dest_offset = ac->in(ArrayCopyNode::DestPos); 1366 Node* length = ac->in(ArrayCopyNode::Length); 1367 MergeMemNode* merge_mem = nullptr; 1368 1369 if (ac->is_clonebasic()) { 1370 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 1371 bs->clone_at_expansion(this, ac); 1372 return; 1373 } else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_clone_oop_array()) { 1374 const Type* src_type = _igvn.type(src); 1375 const Type* dest_type = _igvn.type(dest); 1376 const TypeAryPtr* top_src = src_type->isa_aryptr(); 1377 // Note: The destination could have type Object (i.e. non-array) when directly invoking the protected method 1378 // Object::clone() with reflection on a declared Object that is an array at runtime. top_dest is then null. 1379 const TypeAryPtr* top_dest = dest_type->isa_aryptr(); 1380 BasicType dest_elem = T_OBJECT; 1381 if (top_dest != nullptr && top_dest->elem() != Type::BOTTOM) { 1382 dest_elem = top_dest->elem()->array_element_basic_type(); 1383 } 1384 if (is_reference_type(dest_elem, true)) dest_elem = T_OBJECT; 1385 1386 if (top_src != nullptr && top_src->is_flat()) { 1387 // If src is flat, dest is guaranteed to be flat as well 1388 top_dest = top_src; 1389 } 1390 1391 AllocateArrayNode* alloc = nullptr; 1392 Node* dest_length = nullptr; 1393 if (ac->is_alloc_tightly_coupled()) { 1394 alloc = AllocateArrayNode::Ideal_array_allocation(dest); 1395 assert(alloc != nullptr, "expect alloc"); 1396 dest_length = alloc->in(AllocateNode::ALength); 1397 } 1398 1399 Node* mem = ac->in(TypeFunc::Memory); 1400 const TypePtr* adr_type = nullptr; 1401 if (top_dest != nullptr && top_dest->is_flat()) { 1402 assert(dest_length != nullptr || StressReflectiveCode, "must be tightly coupled"); 1403 // Copy to a flat array modifies multiple memory slices. Conservatively insert a barrier 1404 // on all slices to prevent writes into the source from floating below the arraycopy. 1405 int mem_bar_alias_idx = Compile::AliasIdxBot; 1406 if (ac->_dest_type != TypeOopPtr::BOTTOM) { 1407 mem_bar_alias_idx = C->get_alias_index(ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr()); 1408 } 1409 insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder, mem_bar_alias_idx); 1410 adr_type = adjust_for_flat_array(top_dest, src_offset, dest_offset, length, dest_elem, dest_length); 1411 } else { 1412 adr_type = dest_type->is_oopptr()->add_offset(Type::OffsetBot); 1413 if (ac->_dest_type != TypeOopPtr::BOTTOM) { 1414 adr_type = ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr(); 1415 } 1416 if (ac->_src_type != ac->_dest_type) { 1417 adr_type = TypeRawPtr::BOTTOM; 1418 } 1419 } 1420 merge_mem = MergeMemNode::make(mem); 1421 transform_later(merge_mem); 1422 1423 generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io, 1424 adr_type, dest_elem, 1425 src, src_offset, dest, dest_offset, length, 1426 dest_length, 1427 true, ac->has_negative_length_guard()); 1428 1429 return; 1430 } 1431 1432 AllocateArrayNode* alloc = nullptr; 1433 if (ac->is_alloc_tightly_coupled()) { 1434 alloc = AllocateArrayNode::Ideal_array_allocation(dest); 1435 assert(alloc != nullptr, "expect alloc"); 1436 } 1437 1438 assert(ac->is_arraycopy() || ac->is_arraycopy_validated(), "should be an arraycopy"); 1439 1440 // Compile time checks. If any of these checks cannot be verified at compile time, 1441 // we do not make a fast path for this call. Instead, we let the call remain as it 1442 // is. The checks we choose to mandate at compile time are: 1443 // 1444 // (1) src and dest are arrays. 1445 const Type* src_type = src->Value(&_igvn); 1446 const Type* dest_type = dest->Value(&_igvn); 1447 const TypeAryPtr* top_src = src_type->isa_aryptr(); 1448 const TypeAryPtr* top_dest = dest_type->isa_aryptr(); 1449 1450 BasicType src_elem = T_CONFLICT; 1451 BasicType dest_elem = T_CONFLICT; 1452 1453 if (top_src != nullptr && top_src->elem() != Type::BOTTOM) { 1454 src_elem = top_src->elem()->array_element_basic_type(); 1455 } 1456 if (top_dest != nullptr && top_dest->elem() != Type::BOTTOM) { 1457 dest_elem = top_dest->elem()->array_element_basic_type(); 1458 } 1459 if (is_reference_type(src_elem, true)) src_elem = T_OBJECT; 1460 if (is_reference_type(dest_elem, true)) dest_elem = T_OBJECT; 1461 1462 if (ac->is_arraycopy_validated() && dest_elem != T_CONFLICT && src_elem == T_CONFLICT) { 1463 src_elem = dest_elem; 1464 } 1465 1466 if (src_elem == T_CONFLICT || dest_elem == T_CONFLICT) { 1467 // Conservatively insert a memory barrier on all memory slices. 1468 // Do not let writes into the source float below the arraycopy. 1469 { 1470 Node* mem = ac->in(TypeFunc::Memory); 1471 insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder, Compile::AliasIdxBot); 1472 1473 merge_mem = MergeMemNode::make(mem); 1474 transform_later(merge_mem); 1475 } 1476 1477 // Call StubRoutines::generic_arraycopy stub. 1478 generate_arraycopy(ac, nullptr, &ctrl, merge_mem, &io, 1479 TypeRawPtr::BOTTOM, T_CONFLICT, 1480 src, src_offset, dest, dest_offset, length, 1481 nullptr, 1482 // If a negative length guard was generated for the ArrayCopyNode, 1483 // the length of the array can never be negative. 1484 false, ac->has_negative_length_guard()); 1485 return; 1486 } 1487 1488 assert(!ac->is_arraycopy_validated() || (src_elem == dest_elem && dest_elem != T_VOID), "validated but different basic types"); 1489 1490 // (2) src and dest arrays must have elements of the same BasicType 1491 // Figure out the size and type of the elements we will be copying. 1492 // 1493 // We have no stub to copy flat inline type arrays with oop 1494 // fields if we need to emit write barriers. 1495 // 1496 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 1497 if (src_elem != dest_elem || top_src->is_flat() != top_dest->is_flat() || dest_elem == T_VOID || 1498 (top_src->is_flat() && top_dest->elem()->inline_klass()->contains_oops() && 1499 bs->array_copy_requires_gc_barriers(alloc != nullptr, T_OBJECT, false, false, BarrierSetC2::Optimization))) { 1500 // The component types are not the same or are not recognized. Punt. 1501 // (But, avoid the native method wrapper to JVM_ArrayCopy.) 1502 { 1503 Node* mem = ac->in(TypeFunc::Memory); 1504 merge_mem = generate_slow_arraycopy(ac, &ctrl, mem, &io, TypePtr::BOTTOM, src, src_offset, dest, dest_offset, length, false); 1505 } 1506 1507 _igvn.replace_node(_callprojs->fallthrough_memproj, merge_mem); 1508 if (_callprojs->fallthrough_ioproj != nullptr) { 1509 _igvn.replace_node(_callprojs->fallthrough_ioproj, io); 1510 } 1511 _igvn.replace_node(_callprojs->fallthrough_catchproj, ctrl); 1512 return; 1513 } 1514 1515 //--------------------------------------------------------------------------- 1516 // We will make a fast path for this call to arraycopy. 1517 1518 // We have the following tests left to perform: 1519 // 1520 // (3) src and dest must not be null. 1521 // (4) src_offset must not be negative. 1522 // (5) dest_offset must not be negative. 1523 // (6) length must not be negative. 1524 // (7) src_offset + length must not exceed length of src. 1525 // (8) dest_offset + length must not exceed length of dest. 1526 // (9) each element of an oop array must be assignable 1527 1528 Node* mem = ac->in(TypeFunc::Memory); 1529 if (top_dest->is_flat()) { 1530 // Copy to a flat array modifies multiple memory slices. Conservatively insert a barrier 1531 // on all slices to prevent writes into the source from floating below the arraycopy. 1532 int mem_bar_alias_idx = Compile::AliasIdxBot; 1533 if (ac->_dest_type != TypeOopPtr::BOTTOM) { 1534 mem_bar_alias_idx = C->get_alias_index(ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr()); 1535 } 1536 insert_mem_bar(&ctrl, &mem, Op_MemBarCPUOrder, mem_bar_alias_idx); 1537 } 1538 merge_mem = MergeMemNode::make(mem); 1539 transform_later(merge_mem); 1540 1541 RegionNode* slow_region = new RegionNode(1); 1542 transform_later(slow_region); 1543 1544 if (!ac->is_arraycopy_validated()) { 1545 // (3) operands must not be null 1546 // We currently perform our null checks with the null_check routine. 1547 // This means that the null exceptions will be reported in the caller 1548 // rather than (correctly) reported inside of the native arraycopy call. 1549 // This should be corrected, given time. We do our null check with the 1550 // stack pointer restored. 1551 // null checks done library_call.cpp 1552 1553 // (4) src_offset must not be negative. 1554 generate_negative_guard(&ctrl, src_offset, slow_region); 1555 1556 // (5) dest_offset must not be negative. 1557 generate_negative_guard(&ctrl, dest_offset, slow_region); 1558 1559 // (6) length must not be negative (moved to generate_arraycopy()). 1560 // generate_negative_guard(length, slow_region); 1561 1562 // (7) src_offset + length must not exceed length of src. 1563 Node* alen = ac->in(ArrayCopyNode::SrcLen); 1564 assert(alen != nullptr, "need src len"); 1565 generate_limit_guard(&ctrl, 1566 src_offset, length, 1567 alen, 1568 slow_region); 1569 1570 // (8) dest_offset + length must not exceed length of dest. 1571 alen = ac->in(ArrayCopyNode::DestLen); 1572 assert(alen != nullptr, "need dest len"); 1573 generate_limit_guard(&ctrl, 1574 dest_offset, length, 1575 alen, 1576 slow_region); 1577 1578 // (9) each element of an oop array must be assignable 1579 // The generate_arraycopy subroutine checks this. 1580 1581 // TODO 8350865 Fix below logic. Also handle atomicity. 1582 // We need to be careful here because 'adjust_for_flat_array' will adjust offsets/length etc. which then does not work anymore for the slow call to SharedRuntime::slow_arraycopy_C. 1583 if (!(top_src->is_flat() && top_dest->is_flat())) { 1584 generate_flat_array_guard(&ctrl, src, merge_mem, slow_region); 1585 generate_flat_array_guard(&ctrl, dest, merge_mem, slow_region); 1586 } 1587 1588 // Handle inline type arrays 1589 if (!top_src->is_flat()) { 1590 if (UseArrayFlattening && !top_src->is_not_flat()) { 1591 // Src might be flat and dest might not be flat. Go to the slow path if src is flat. 1592 generate_flat_array_guard(&ctrl, src, merge_mem, slow_region); 1593 } 1594 if (EnableValhalla) { 1595 // No validation. The subtype check emitted at macro expansion time will not go to the slow 1596 // path but call checkcast_arraycopy which can not handle flat/null-free inline type arrays. 1597 generate_null_free_array_guard(&ctrl, dest, merge_mem, slow_region); 1598 } 1599 } else { 1600 assert(top_dest->is_flat(), "dest array must be flat"); 1601 } 1602 } 1603 1604 // This is where the memory effects are placed: 1605 const TypePtr* adr_type = nullptr; 1606 Node* dest_length = (alloc != nullptr) ? alloc->in(AllocateNode::ALength) : nullptr; 1607 1608 if (top_src->is_flat() && top_dest->is_flat()) { 1609 adr_type = adjust_for_flat_array(top_dest, src_offset, dest_offset, length, dest_elem, dest_length); 1610 } else if (ac->_dest_type != TypeOopPtr::BOTTOM) { 1611 adr_type = ac->_dest_type->add_offset(Type::OffsetBot)->is_ptr(); 1612 } else { 1613 adr_type = TypeAryPtr::get_array_body_type(dest_elem); 1614 } 1615 1616 generate_arraycopy(ac, alloc, &ctrl, merge_mem, &io, 1617 adr_type, dest_elem, 1618 src, src_offset, dest, dest_offset, length, 1619 dest_length, 1620 // If a negative length guard was generated for the ArrayCopyNode, 1621 // the length of the array can never be negative. 1622 false, ac->has_negative_length_guard(), 1623 slow_region); 1624 }