1 /* 2 * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "precompiled.hpp" 25 #include "asm/macroAssembler.hpp" 26 #include "classfile/javaClasses.hpp" 27 #include "gc/z/c2/zBarrierSetC2.hpp" 28 #include "gc/z/zBarrierSet.hpp" 29 #include "gc/z/zBarrierSetAssembler.hpp" 30 #include "gc/z/zBarrierSetRuntime.hpp" 31 #include "opto/arraycopynode.hpp" 32 #include "opto/addnode.hpp" 33 #include "opto/block.hpp" 34 #include "opto/compile.hpp" 35 #include "opto/graphKit.hpp" 36 #include "opto/machnode.hpp" 37 #include "opto/macro.hpp" 38 #include "opto/memnode.hpp" 39 #include "opto/node.hpp" 40 #include "opto/output.hpp" 41 #include "opto/regalloc.hpp" 42 #include "opto/rootnode.hpp" 43 #include "opto/runtime.hpp" 44 #include "opto/type.hpp" 45 #include "utilities/growableArray.hpp" 46 #include "utilities/macros.hpp" 47 48 template<typename K, typename V, size_t _table_size> 49 class ZArenaHashtable : public ResourceObj { 50 class ZArenaHashtableEntry : public ResourceObj { 51 public: 52 ZArenaHashtableEntry* _next; 53 K _key; 54 V _value; 55 }; 56 57 static const size_t _table_mask = _table_size - 1; 58 59 Arena* _arena; 60 ZArenaHashtableEntry* _table[_table_size]; 61 62 public: 63 class Iterator { 64 ZArenaHashtable* _table; 65 ZArenaHashtableEntry* _current_entry; 66 size_t _current_index; 67 68 public: 69 Iterator(ZArenaHashtable* table) 70 : _table(table), 71 _current_entry(table->_table[0]), 72 _current_index(0) { 73 if (_current_entry == nullptr) { 74 next(); 75 } 76 } 77 78 bool has_next() { return _current_entry != nullptr; } 79 K key() { return _current_entry->_key; } 80 V value() { return _current_entry->_value; } 81 82 void next() { 83 if (_current_entry != nullptr) { 84 _current_entry = _current_entry->_next; 85 } 86 while (_current_entry == nullptr && ++_current_index < _table_size) { 87 _current_entry = _table->_table[_current_index]; 88 } 89 } 90 }; 91 92 ZArenaHashtable(Arena* arena) 93 : _arena(arena), 94 _table() { 95 Copy::zero_to_bytes(&_table, sizeof(_table)); 96 } 97 98 void add(K key, V value) { 99 ZArenaHashtableEntry* entry = new (_arena) ZArenaHashtableEntry(); 100 entry->_key = key; 101 entry->_value = value; 102 entry->_next = _table[key & _table_mask]; 103 _table[key & _table_mask] = entry; 104 } 105 106 V* get(K key) const { 107 for (ZArenaHashtableEntry* e = _table[key & _table_mask]; e != nullptr; e = e->_next) { 108 if (e->_key == key) { 109 return &(e->_value); 110 } 111 } 112 return nullptr; 113 } 114 115 Iterator iterator() { 116 return Iterator(this); 117 } 118 }; 119 120 typedef ZArenaHashtable<intptr_t, bool, 4> ZOffsetTable; 121 122 class ZBarrierSetC2State : public ArenaObj { 123 private: 124 GrowableArray<ZBarrierStubC2*>* _stubs; 125 Node_Array _live; 126 int _trampoline_stubs_count; 127 int _stubs_start_offset; 128 129 public: 130 ZBarrierSetC2State(Arena* arena) 131 : _stubs(new (arena) GrowableArray<ZBarrierStubC2*>(arena, 8, 0, nullptr)), 132 _live(arena), 133 _trampoline_stubs_count(0), 134 _stubs_start_offset(0) {} 135 136 GrowableArray<ZBarrierStubC2*>* stubs() { 137 return _stubs; 138 } 139 140 RegMask* live(const Node* node) { 141 if (!node->is_Mach()) { 142 // Don't need liveness for non-MachNodes 143 return nullptr; 144 } 145 146 const MachNode* const mach = node->as_Mach(); 147 if (mach->barrier_data() == ZBarrierElided) { 148 // Don't need liveness data for nodes without barriers 149 return nullptr; 150 } 151 152 RegMask* live = (RegMask*)_live[node->_idx]; 153 if (live == nullptr) { 154 live = new (Compile::current()->comp_arena()->AmallocWords(sizeof(RegMask))) RegMask(); 155 _live.map(node->_idx, (Node*)live); 156 } 157 158 return live; 159 } 160 161 void inc_trampoline_stubs_count() { 162 assert(_trampoline_stubs_count != INT_MAX, "Overflow"); 163 ++_trampoline_stubs_count; 164 } 165 166 int trampoline_stubs_count() { 167 return _trampoline_stubs_count; 168 } 169 170 void set_stubs_start_offset(int offset) { 171 _stubs_start_offset = offset; 172 } 173 174 int stubs_start_offset() { 175 return _stubs_start_offset; 176 } 177 }; 178 179 static ZBarrierSetC2State* barrier_set_state() { 180 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state()); 181 } 182 183 void ZBarrierStubC2::register_stub(ZBarrierStubC2* stub) { 184 if (!Compile::current()->output()->in_scratch_emit_size()) { 185 barrier_set_state()->stubs()->append(stub); 186 } 187 } 188 189 void ZBarrierStubC2::inc_trampoline_stubs_count() { 190 if (!Compile::current()->output()->in_scratch_emit_size()) { 191 barrier_set_state()->inc_trampoline_stubs_count(); 192 } 193 } 194 195 int ZBarrierStubC2::trampoline_stubs_count() { 196 return barrier_set_state()->trampoline_stubs_count(); 197 } 198 199 int ZBarrierStubC2::stubs_start_offset() { 200 return barrier_set_state()->stubs_start_offset(); 201 } 202 203 ZBarrierStubC2::ZBarrierStubC2(const MachNode* node) 204 : _node(node), 205 _entry(), 206 _continuation() {} 207 208 Register ZBarrierStubC2::result() const { 209 return noreg; 210 } 211 212 RegMask& ZBarrierStubC2::live() const { 213 return *barrier_set_state()->live(_node); 214 } 215 216 Label* ZBarrierStubC2::entry() { 217 // The _entry will never be bound when in_scratch_emit_size() is true. 218 // However, we still need to return a label that is not bound now, but 219 // will eventually be bound. Any eventually bound label will do, as it 220 // will only act as a placeholder, so we return the _continuation label. 221 return Compile::current()->output()->in_scratch_emit_size() ? &_continuation : &_entry; 222 } 223 224 Label* ZBarrierStubC2::continuation() { 225 return &_continuation; 226 } 227 228 ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref) { 229 ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref); 230 register_stub(stub); 231 232 return stub; 233 } 234 235 ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref) 236 : ZBarrierStubC2(node), 237 _ref_addr(ref_addr), 238 _ref(ref) { 239 assert_different_registers(ref, ref_addr.base()); 240 assert_different_registers(ref, ref_addr.index()); 241 } 242 243 Address ZLoadBarrierStubC2::ref_addr() const { 244 return _ref_addr; 245 } 246 247 Register ZLoadBarrierStubC2::ref() const { 248 return _ref; 249 } 250 251 Register ZLoadBarrierStubC2::result() const { 252 return ref(); 253 } 254 255 address ZLoadBarrierStubC2::slow_path() const { 256 const uint8_t barrier_data = _node->barrier_data(); 257 DecoratorSet decorators = DECORATORS_NONE; 258 if (barrier_data & ZBarrierStrong) { 259 decorators |= ON_STRONG_OOP_REF; 260 } 261 if (barrier_data & ZBarrierWeak) { 262 decorators |= ON_WEAK_OOP_REF; 263 } 264 if (barrier_data & ZBarrierPhantom) { 265 decorators |= ON_PHANTOM_OOP_REF; 266 } 267 if (barrier_data & ZBarrierNoKeepalive) { 268 decorators |= AS_NO_KEEPALIVE; 269 } 270 return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators); 271 } 272 273 void ZLoadBarrierStubC2::emit_code(MacroAssembler& masm) { 274 ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, static_cast<ZLoadBarrierStubC2*>(this)); 275 } 276 277 ZStoreBarrierStubC2* ZStoreBarrierStubC2::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic) { 278 ZStoreBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic); 279 register_stub(stub); 280 281 return stub; 282 } 283 284 ZStoreBarrierStubC2::ZStoreBarrierStubC2(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic) 285 : ZBarrierStubC2(node), 286 _ref_addr(ref_addr), 287 _new_zaddress(new_zaddress), 288 _new_zpointer(new_zpointer), 289 _is_native(is_native), 290 _is_atomic(is_atomic) {} 291 292 Address ZStoreBarrierStubC2::ref_addr() const { 293 return _ref_addr; 294 } 295 296 Register ZStoreBarrierStubC2::new_zaddress() const { 297 return _new_zaddress; 298 } 299 300 Register ZStoreBarrierStubC2::new_zpointer() const { 301 return _new_zpointer; 302 } 303 304 bool ZStoreBarrierStubC2::is_native() const { 305 return _is_native; 306 } 307 308 bool ZStoreBarrierStubC2::is_atomic() const { 309 return _is_atomic; 310 } 311 312 Register ZStoreBarrierStubC2::result() const { 313 return noreg; 314 } 315 316 void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) { 317 ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast<ZStoreBarrierStubC2*>(this)); 318 } 319 320 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { 321 return new (comp_arena) ZBarrierSetC2State(comp_arena); 322 } 323 324 void ZBarrierSetC2::late_barrier_analysis() const { 325 compute_liveness_at_stubs(); 326 analyze_dominating_barriers(); 327 } 328 329 void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const { 330 MacroAssembler masm(&cb); 331 GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs(); 332 barrier_set_state()->set_stubs_start_offset(masm.offset()); 333 334 for (int i = 0; i < stubs->length(); i++) { 335 // Make sure there is enough space in the code buffer 336 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) { 337 ciEnv::current()->record_failure("CodeCache is full"); 338 return; 339 } 340 341 stubs->at(i)->emit_code(masm); 342 } 343 344 masm.flush(); 345 } 346 347 int ZBarrierSetC2::estimate_stub_size() const { 348 Compile* const C = Compile::current(); 349 BufferBlob* const blob = C->output()->scratch_buffer_blob(); 350 GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs(); 351 int size = 0; 352 353 for (int i = 0; i < stubs->length(); i++) { 354 CodeBuffer cb(blob->content_begin(), (address)C->output()->scratch_locs_memory() - blob->content_begin()); 355 MacroAssembler masm(&cb); 356 stubs->at(i)->emit_code(masm); 357 size += cb.insts_size(); 358 } 359 360 return size; 361 } 362 363 static void set_barrier_data(C2Access& access) { 364 if (!ZBarrierSet::barrier_needed(access.decorators(), access.type())) { 365 return; 366 } 367 368 if (access.decorators() & C2_TIGHTLY_COUPLED_ALLOC) { 369 access.set_barrier_data(ZBarrierElided); 370 return; 371 } 372 373 uint8_t barrier_data = 0; 374 375 if (access.decorators() & ON_PHANTOM_OOP_REF) { 376 barrier_data |= ZBarrierPhantom; 377 } else if (access.decorators() & ON_WEAK_OOP_REF) { 378 barrier_data |= ZBarrierWeak; 379 } else { 380 barrier_data |= ZBarrierStrong; 381 } 382 383 if (access.decorators() & IN_NATIVE) { 384 barrier_data |= ZBarrierNative; 385 } 386 387 if (access.decorators() & AS_NO_KEEPALIVE) { 388 barrier_data |= ZBarrierNoKeepalive; 389 } 390 391 access.set_barrier_data(barrier_data); 392 } 393 394 Node* ZBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const { 395 set_barrier_data(access); 396 return BarrierSetC2::store_at_resolved(access, val); 397 } 398 399 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { 400 set_barrier_data(access); 401 return BarrierSetC2::load_at_resolved(access, val_type); 402 } 403 404 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 405 Node* new_val, const Type* val_type) const { 406 set_barrier_data(access); 407 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type); 408 } 409 410 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val, 411 Node* new_val, const Type* value_type) const { 412 set_barrier_data(access); 413 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); 414 } 415 416 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const { 417 set_barrier_data(access); 418 return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type); 419 } 420 421 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, 422 bool is_clone, bool is_clone_instance, 423 ArrayCopyPhase phase) const { 424 if (phase == ArrayCopyPhase::Parsing) { 425 return false; 426 } 427 if (phase == ArrayCopyPhase::Optimization) { 428 return is_clone_instance; 429 } 430 // else ArrayCopyPhase::Expansion 431 return type == T_OBJECT || type == T_ARRAY; 432 } 433 434 // This TypeFunc assumes a 64bit system 435 static const TypeFunc* clone_type() { 436 // Create input type (domain) 437 const Type** const domain_fields = TypeTuple::fields(4); 438 domain_fields[TypeFunc::Parms + 0] = TypeInstPtr::NOTNULL; // src 439 domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL; // dst 440 domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG; // size lower 441 domain_fields[TypeFunc::Parms + 3] = Type::HALF; // size upper 442 const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields); 443 444 // Create result type (range) 445 const Type** const range_fields = TypeTuple::fields(0); 446 const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields); 447 448 return TypeFunc::make(domain, range); 449 } 450 451 #define XTOP LP64_ONLY(COMMA phase->top()) 452 453 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const { 454 Node* const src = ac->in(ArrayCopyNode::Src); 455 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr(); 456 457 if (ac->is_clone_array() && ary_ptr != nullptr) { 458 BasicType bt = ary_ptr->elem()->array_element_basic_type(); 459 if (is_reference_type(bt)) { 460 // Clone object array 461 bt = T_OBJECT; 462 } else { 463 // Clone primitive array 464 bt = T_LONG; 465 } 466 467 Node* const ctrl = ac->in(TypeFunc::Control); 468 Node* const mem = ac->in(TypeFunc::Memory); 469 Node* const src = ac->in(ArrayCopyNode::Src); 470 Node* src_offset = ac->in(ArrayCopyNode::SrcPos); 471 Node* const dest = ac->in(ArrayCopyNode::Dest); 472 Node* dest_offset = ac->in(ArrayCopyNode::DestPos); 473 Node* length = ac->in(ArrayCopyNode::Length); 474 475 if (bt == T_OBJECT) { 476 // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset 477 // which 8-byte aligns them to allow for word size copies. Make sure the offsets point 478 // to the first element in the array when cloning object arrays. Otherwise, load 479 // barriers are applied to parts of the header. Also adjust the length accordingly. 480 assert(src_offset == dest_offset, "should be equal"); 481 const jlong offset = src_offset->get_long(); 482 if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) { 483 assert(!UseCompressedClassPointers, "should only happen without compressed class pointers"); 484 assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset"); 485 length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs 486 src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT)); 487 dest_offset = src_offset; 488 } 489 } 490 Node* const payload_src = phase->basic_plus_adr(src, src_offset); 491 Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset); 492 493 const char* copyfunc_name = "arraycopy"; 494 const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true); 495 496 const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM; 497 const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type(); 498 499 Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP); 500 phase->transform_later(call); 501 502 phase->igvn().replace_node(ac, call); 503 return; 504 } 505 506 // Clone instance 507 Node* const ctrl = ac->in(TypeFunc::Control); 508 Node* const mem = ac->in(TypeFunc::Memory); 509 Node* const dst = ac->in(ArrayCopyNode::Dest); 510 Node* const size = ac->in(ArrayCopyNode::Length); 511 512 assert(size->bottom_type()->is_long(), "Should be long"); 513 514 // The native clone we are calling here expects the instance size in words 515 // Add header/offset size to payload size to get instance size. 516 Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong); 517 Node* const full_size = phase->transform_later(new AddLNode(size, base_offset)); 518 519 Node* const call = phase->make_leaf_call(ctrl, 520 mem, 521 clone_type(), 522 ZBarrierSetRuntime::clone_addr(), 523 "ZBarrierSetRuntime::clone", 524 TypeRawPtr::BOTTOM, 525 src, 526 dst, 527 full_size, 528 phase->top()); 529 phase->transform_later(call); 530 phase->igvn().replace_node(ac, call); 531 } 532 533 #undef XTOP 534 535 // == Dominating barrier elision == 536 537 static bool block_has_safepoint(const Block* block, uint from, uint to) { 538 for (uint i = from; i < to; i++) { 539 if (block->get_node(i)->is_MachSafePoint()) { 540 // Safepoint found 541 return true; 542 } 543 } 544 545 // Safepoint not found 546 return false; 547 } 548 549 static bool block_has_safepoint(const Block* block) { 550 return block_has_safepoint(block, 0, block->number_of_nodes()); 551 } 552 553 static uint block_index(const Block* block, const Node* node) { 554 for (uint j = 0; j < block->number_of_nodes(); ++j) { 555 if (block->get_node(j) == node) { 556 return j; 557 } 558 } 559 ShouldNotReachHere(); 560 return 0; 561 } 562 563 // Look through various node aliases 564 static const Node* look_through_node(const Node* node) { 565 while (node != nullptr) { 566 const Node* new_node = node; 567 if (node->is_Mach()) { 568 const MachNode* const node_mach = node->as_Mach(); 569 if (node_mach->ideal_Opcode() == Op_CheckCastPP) { 570 new_node = node->in(1); 571 } 572 if (node_mach->is_SpillCopy()) { 573 new_node = node->in(1); 574 } 575 } 576 if (new_node == node || new_node == nullptr) { 577 break; 578 } else { 579 node = new_node; 580 } 581 } 582 583 return node; 584 } 585 586 // Whether the given offset is undefined. 587 static bool is_undefined(intptr_t offset) { 588 return offset == Type::OffsetTop; 589 } 590 591 // Whether the given offset is unknown. 592 static bool is_unknown(intptr_t offset) { 593 return offset == Type::OffsetBot; 594 } 595 596 // Whether the given offset is concrete (defined and compile-time known). 597 static bool is_concrete(intptr_t offset) { 598 return !is_undefined(offset) && !is_unknown(offset); 599 } 600 601 // Compute base + offset components of the memory address accessed by mach. 602 // Return a node representing the base address, or null if the base cannot be 603 // found or the offset is undefined or a concrete negative value. If a non-null 604 // base is returned, the offset is a concrete, nonnegative value or unknown. 605 static const Node* get_base_and_offset(const MachNode* mach, intptr_t& offset) { 606 const TypePtr* adr_type = nullptr; 607 offset = 0; 608 const Node* base = mach->get_base_and_disp(offset, adr_type); 609 610 if (base == nullptr || base == NodeSentinel) { 611 return nullptr; 612 } 613 614 if (offset == 0 && base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_AddP) { 615 // The memory address is computed by 'base' and fed to 'mach' via an 616 // indirect memory operand (indicated by offset == 0). The ultimate base and 617 // offset can be fetched directly from the inputs and Ideal type of 'base'. 618 offset = base->bottom_type()->isa_oopptr()->offset(); 619 // Even if 'base' is not an Ideal AddP node anymore, Matcher::ReduceInst() 620 // guarantees that the base address is still available at the same slot. 621 base = base->in(AddPNode::Base); 622 assert(base != nullptr, ""); 623 } 624 625 if (is_undefined(offset) || (is_concrete(offset) && offset < 0)) { 626 return nullptr; 627 } 628 629 return look_through_node(base); 630 } 631 632 // Whether a phi node corresponds to an array allocation. 633 // This test is incomplete: in some edge cases, it might return false even 634 // though the node does correspond to an array allocation. 635 static bool is_array_allocation(const Node* phi) { 636 precond(phi->is_Phi()); 637 // Check whether phi has a successor cast (CheckCastPP) to Java array pointer, 638 // possibly below spill copies and other cast nodes. Limit the exploration to 639 // a single path from the phi node consisting of these node types. 640 const Node* current = phi; 641 while (true) { 642 const Node* next = nullptr; 643 for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) { 644 if (!current->fast_out(i)->isa_Mach()) { 645 continue; 646 } 647 const MachNode* succ = current->fast_out(i)->as_Mach(); 648 if (succ->ideal_Opcode() == Op_CheckCastPP) { 649 if (succ->get_ptr_type()->isa_aryptr()) { 650 // Cast to Java array pointer: phi corresponds to an array allocation. 651 return true; 652 } 653 // Other cast: record as candidate for further exploration. 654 next = succ; 655 } else if (succ->is_SpillCopy() && next == nullptr) { 656 // Spill copy, and no better candidate found: record as candidate. 657 next = succ; 658 } 659 } 660 if (next == nullptr) { 661 // No evidence found that phi corresponds to an array allocation, and no 662 // candidates available to continue exploring. 663 return false; 664 } 665 // Continue exploring from the best candidate found. 666 current = next; 667 } 668 ShouldNotReachHere(); 669 } 670 671 // Match the phi node that connects a TLAB allocation fast path with its slowpath 672 static bool is_allocation(const Node* node) { 673 if (node->req() != 3) { 674 return false; 675 } 676 const Node* const fast_node = node->in(2); 677 if (!fast_node->is_Mach()) { 678 return false; 679 } 680 const MachNode* const fast_mach = fast_node->as_Mach(); 681 if (fast_mach->ideal_Opcode() != Op_LoadP) { 682 return false; 683 } 684 const TypePtr* const adr_type = nullptr; 685 intptr_t offset; 686 const Node* const base = get_base_and_offset(fast_mach, offset); 687 if (base == nullptr || !base->is_Mach() || !is_concrete(offset)) { 688 return false; 689 } 690 const MachNode* const base_mach = base->as_Mach(); 691 if (base_mach->ideal_Opcode() != Op_ThreadLocal) { 692 return false; 693 } 694 return offset == in_bytes(Thread::tlab_top_offset()); 695 } 696 697 static void elide_mach_barrier(MachNode* mach) { 698 mach->set_barrier_data(ZBarrierElided); 699 } 700 701 void ZBarrierSetC2::analyze_dominating_barriers_impl(Node_List& accesses, Node_List& access_dominators) const { 702 Compile* const C = Compile::current(); 703 PhaseCFG* const cfg = C->cfg(); 704 705 for (uint i = 0; i < accesses.size(); i++) { 706 MachNode* const access = accesses.at(i)->as_Mach(); 707 intptr_t access_offset; 708 const Node* const access_obj = get_base_and_offset(access, access_offset); 709 Block* const access_block = cfg->get_block_for_node(access); 710 const uint access_index = block_index(access_block, access); 711 712 if (access_obj == nullptr) { 713 // No information available 714 continue; 715 } 716 717 for (uint j = 0; j < access_dominators.size(); j++) { 718 const Node* const mem = access_dominators.at(j); 719 if (mem->is_Phi()) { 720 // Allocation node 721 if (mem != access_obj) { 722 continue; 723 } 724 if (is_unknown(access_offset) && !is_array_allocation(mem)) { 725 // The accessed address has an unknown offset, but the allocated 726 // object cannot be determined to be an array. Avoid eliding in this 727 // case, to be on the safe side. 728 continue; 729 } 730 assert((is_concrete(access_offset) && access_offset >= 0) || (is_unknown(access_offset) && is_array_allocation(mem)), 731 "candidate allocation-dominated access offsets must be either concrete and nonnegative, or unknown (for array allocations only)"); 732 } else { 733 // Access node 734 const MachNode* const mem_mach = mem->as_Mach(); 735 intptr_t mem_offset; 736 const Node* const mem_obj = get_base_and_offset(mem_mach, mem_offset); 737 738 if (mem_obj == nullptr || 739 !is_concrete(access_offset) || 740 !is_concrete(mem_offset)) { 741 // No information available 742 continue; 743 } 744 745 if (mem_obj != access_obj || mem_offset != access_offset) { 746 // Not the same addresses, not a candidate 747 continue; 748 } 749 assert(is_concrete(access_offset) && access_offset >= 0, 750 "candidate non-allocation-dominated access offsets must be concrete and nonnegative"); 751 } 752 753 Block* mem_block = cfg->get_block_for_node(mem); 754 const uint mem_index = block_index(mem_block, mem); 755 756 if (access_block == mem_block) { 757 // Earlier accesses in the same block 758 if (mem_index < access_index && !block_has_safepoint(mem_block, mem_index + 1, access_index)) { 759 elide_mach_barrier(access); 760 } 761 } else if (mem_block->dominates(access_block)) { 762 // Dominating block? Look around for safepoints 763 ResourceMark rm; 764 Block_List stack; 765 VectorSet visited; 766 stack.push(access_block); 767 bool safepoint_found = block_has_safepoint(access_block); 768 while (!safepoint_found && stack.size() > 0) { 769 const Block* const block = stack.pop(); 770 if (visited.test_set(block->_pre_order)) { 771 continue; 772 } 773 if (block_has_safepoint(block)) { 774 safepoint_found = true; 775 break; 776 } 777 if (block == mem_block) { 778 continue; 779 } 780 781 // Push predecessor blocks 782 for (uint p = 1; p < block->num_preds(); ++p) { 783 Block* const pred = cfg->get_block_for_node(block->pred(p)); 784 stack.push(pred); 785 } 786 } 787 788 if (!safepoint_found) { 789 elide_mach_barrier(access); 790 } 791 } 792 } 793 } 794 } 795 796 void ZBarrierSetC2::analyze_dominating_barriers() const { 797 ResourceMark rm; 798 Compile* const C = Compile::current(); 799 PhaseCFG* const cfg = C->cfg(); 800 801 Node_List loads; 802 Node_List load_dominators; 803 804 Node_List stores; 805 Node_List store_dominators; 806 807 Node_List atomics; 808 Node_List atomic_dominators; 809 810 // Step 1 - Find accesses and allocations, and track them in lists 811 for (uint i = 0; i < cfg->number_of_blocks(); ++i) { 812 const Block* const block = cfg->get_block(i); 813 for (uint j = 0; j < block->number_of_nodes(); ++j) { 814 Node* const node = block->get_node(j); 815 if (node->is_Phi()) { 816 if (is_allocation(node)) { 817 load_dominators.push(node); 818 store_dominators.push(node); 819 // An allocation can't be considered to "dominate" an atomic operation. 820 // For example a CAS requires the memory location to be store-good. 821 // When you have a dominating store or atomic instruction, that is 822 // indeed ensured to be the case. However, as for allocations, the 823 // initialized memory location could be raw null, which isn't store-good. 824 } 825 continue; 826 } else if (!node->is_Mach()) { 827 continue; 828 } 829 830 MachNode* const mach = node->as_Mach(); 831 switch (mach->ideal_Opcode()) { 832 case Op_LoadP: 833 if ((mach->barrier_data() & ZBarrierStrong) != 0 && 834 (mach->barrier_data() & ZBarrierNoKeepalive) == 0) { 835 loads.push(mach); 836 load_dominators.push(mach); 837 } 838 break; 839 case Op_StoreP: 840 if (mach->barrier_data() != 0) { 841 stores.push(mach); 842 load_dominators.push(mach); 843 store_dominators.push(mach); 844 atomic_dominators.push(mach); 845 } 846 break; 847 case Op_CompareAndExchangeP: 848 case Op_CompareAndSwapP: 849 case Op_GetAndSetP: 850 if (mach->barrier_data() != 0) { 851 atomics.push(mach); 852 load_dominators.push(mach); 853 store_dominators.push(mach); 854 atomic_dominators.push(mach); 855 } 856 break; 857 858 default: 859 break; 860 } 861 } 862 } 863 864 // Step 2 - Find dominating accesses or allocations for each access 865 analyze_dominating_barriers_impl(loads, load_dominators); 866 analyze_dominating_barriers_impl(stores, store_dominators); 867 analyze_dominating_barriers_impl(atomics, atomic_dominators); 868 } 869 870 // == Reduced spilling optimization == 871 872 void ZBarrierSetC2::compute_liveness_at_stubs() const { 873 ResourceMark rm; 874 Compile* const C = Compile::current(); 875 Arena* const A = Thread::current()->resource_area(); 876 PhaseCFG* const cfg = C->cfg(); 877 PhaseRegAlloc* const regalloc = C->regalloc(); 878 RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask)); 879 ZBarrierSetAssembler* const bs = ZBarrierSet::assembler(); 880 Block_List worklist; 881 882 for (uint i = 0; i < cfg->number_of_blocks(); ++i) { 883 new ((void*)(live + i)) RegMask(); 884 worklist.push(cfg->get_block(i)); 885 } 886 887 while (worklist.size() > 0) { 888 const Block* const block = worklist.pop(); 889 RegMask& old_live = live[block->_pre_order]; 890 RegMask new_live; 891 892 // Initialize to union of successors 893 for (uint i = 0; i < block->_num_succs; i++) { 894 const uint succ_id = block->_succs[i]->_pre_order; 895 new_live.OR(live[succ_id]); 896 } 897 898 // Walk block backwards, computing liveness 899 for (int i = block->number_of_nodes() - 1; i >= 0; --i) { 900 const Node* const node = block->get_node(i); 901 902 // Remove def bits 903 const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node)); 904 const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node)); 905 if (first != OptoReg::Bad) { 906 new_live.Remove(first); 907 } 908 if (second != OptoReg::Bad) { 909 new_live.Remove(second); 910 } 911 912 // Add use bits 913 for (uint j = 1; j < node->req(); ++j) { 914 const Node* const use = node->in(j); 915 const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use)); 916 const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use)); 917 if (first != OptoReg::Bad) { 918 new_live.Insert(first); 919 } 920 if (second != OptoReg::Bad) { 921 new_live.Insert(second); 922 } 923 } 924 925 // If this node tracks liveness, update it 926 RegMask* const regs = barrier_set_state()->live(node); 927 if (regs != nullptr) { 928 regs->OR(new_live); 929 } 930 } 931 932 // Now at block top, see if we have any changes 933 new_live.SUBTRACT(old_live); 934 if (new_live.is_NotEmpty()) { 935 // Liveness has refined, update and propagate to prior blocks 936 old_live.OR(new_live); 937 for (uint i = 1; i < block->num_preds(); ++i) { 938 Block* const pred = cfg->get_block_for_node(block->pred(i)); 939 worklist.push(pred); 940 } 941 } 942 } 943 } 944 945 void ZBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { 946 eliminate_gc_barrier_data(node); 947 } 948 949 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const { 950 if (node->is_LoadStore()) { 951 LoadStoreNode* loadstore = node->as_LoadStore(); 952 loadstore->set_barrier_data(ZBarrierElided); 953 } else if (node->is_Mem()) { 954 MemNode* mem = node->as_Mem(); 955 mem->set_barrier_data(ZBarrierElided); 956 } 957 } 958 959 #ifndef PRODUCT 960 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const { 961 if ((mach->barrier_data() & ZBarrierStrong) != 0) { 962 st->print("strong "); 963 } 964 if ((mach->barrier_data() & ZBarrierWeak) != 0) { 965 st->print("weak "); 966 } 967 if ((mach->barrier_data() & ZBarrierPhantom) != 0) { 968 st->print("phantom "); 969 } 970 if ((mach->barrier_data() & ZBarrierNoKeepalive) != 0) { 971 st->print("nokeepalive "); 972 } 973 if ((mach->barrier_data() & ZBarrierNative) != 0) { 974 st->print("native "); 975 } 976 if ((mach->barrier_data() & ZBarrierElided) != 0) { 977 st->print("elided "); 978 } 979 } 980 #endif // !PRODUCT