1 /*
2 * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "asm/macroAssembler.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "gc/z/c2/zBarrierSetC2.hpp"
28 #include "gc/z/zBarrierSet.hpp"
29 #include "gc/z/zBarrierSetAssembler.hpp"
30 #include "gc/z/zBarrierSetRuntime.hpp"
31 #include "opto/arraycopynode.hpp"
32 #include "opto/addnode.hpp"
33 #include "opto/block.hpp"
34 #include "opto/compile.hpp"
35 #include "opto/graphKit.hpp"
36 #include "opto/machnode.hpp"
37 #include "opto/macro.hpp"
38 #include "opto/memnode.hpp"
39 #include "opto/node.hpp"
40 #include "opto/output.hpp"
41 #include "opto/regalloc.hpp"
42 #include "opto/rootnode.hpp"
43 #include "opto/runtime.hpp"
44 #include "opto/type.hpp"
45 #include "utilities/debug.hpp"
46 #include "utilities/growableArray.hpp"
47 #include "utilities/macros.hpp"
48
49 template<typename K, typename V, size_t TableSize>
50 class ZArenaHashtable : public ResourceObj {
51 class ZArenaHashtableEntry : public ResourceObj {
52 public:
53 ZArenaHashtableEntry* _next;
54 K _key;
55 V _value;
56 };
57
58 static const size_t TableMask = TableSize - 1;
59
60 Arena* _arena;
61 ZArenaHashtableEntry* _table[TableSize];
62
63 public:
64 class Iterator {
65 ZArenaHashtable* _table;
66 ZArenaHashtableEntry* _current_entry;
67 size_t _current_index;
68
69 public:
70 Iterator(ZArenaHashtable* table)
71 : _table(table),
72 _current_entry(table->_table[0]),
73 _current_index(0) {
74 if (_current_entry == nullptr) {
75 next();
76 }
77 }
78
79 bool has_next() { return _current_entry != nullptr; }
80 K key() { return _current_entry->_key; }
81 V value() { return _current_entry->_value; }
82
83 void next() {
84 if (_current_entry != nullptr) {
85 _current_entry = _current_entry->_next;
86 }
87 while (_current_entry == nullptr && ++_current_index < TableSize) {
88 _current_entry = _table->_table[_current_index];
89 }
90 }
91 };
92
93 ZArenaHashtable(Arena* arena)
94 : _arena(arena),
95 _table() {
96 Copy::zero_to_bytes(&_table, sizeof(_table));
97 }
98
99 void add(K key, V value) {
100 ZArenaHashtableEntry* entry = new (_arena) ZArenaHashtableEntry();
101 entry->_key = key;
102 entry->_value = value;
103 entry->_next = _table[key & TableMask];
104 _table[key & TableMask] = entry;
105 }
106
107 V* get(K key) const {
108 for (ZArenaHashtableEntry* e = _table[key & TableMask]; e != nullptr; e = e->_next) {
109 if (e->_key == key) {
110 return &(e->_value);
111 }
112 }
113 return nullptr;
114 }
115
116 Iterator iterator() {
117 return Iterator(this);
118 }
119 };
120
121 typedef ZArenaHashtable<intptr_t, bool, 4> ZOffsetTable;
122
123 class ZBarrierSetC2State : public BarrierSetC2State {
124 private:
125 GrowableArray<ZBarrierStubC2*>* _stubs;
126 int _trampoline_stubs_count;
127 int _stubs_start_offset;
128
129 public:
130 ZBarrierSetC2State(Arena* arena)
131 : BarrierSetC2State(arena),
132 _stubs(new (arena) GrowableArray<ZBarrierStubC2*>(arena, 8, 0, nullptr)),
133 _trampoline_stubs_count(0),
134 _stubs_start_offset(0) {}
135
136 GrowableArray<ZBarrierStubC2*>* stubs() {
137 return _stubs;
138 }
139
140 bool needs_liveness_data(const MachNode* mach) const {
141 // Don't need liveness data for nodes without barriers
142 return mach->barrier_data() != ZBarrierElided;
143 }
144
145 bool needs_livein_data() const {
146 return true;
147 }
148
149 void inc_trampoline_stubs_count() {
150 assert(_trampoline_stubs_count != INT_MAX, "Overflow");
151 ++_trampoline_stubs_count;
152 }
153
154 int trampoline_stubs_count() {
155 return _trampoline_stubs_count;
156 }
157
158 void set_stubs_start_offset(int offset) {
159 _stubs_start_offset = offset;
160 }
161
162 int stubs_start_offset() {
163 return _stubs_start_offset;
164 }
165 };
166
167 static ZBarrierSetC2State* barrier_set_state() {
168 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
169 }
170
171 void ZBarrierStubC2::register_stub(ZBarrierStubC2* stub) {
172 if (!Compile::current()->output()->in_scratch_emit_size()) {
173 barrier_set_state()->stubs()->append(stub);
174 }
175 }
176
177 void ZBarrierStubC2::inc_trampoline_stubs_count() {
178 if (!Compile::current()->output()->in_scratch_emit_size()) {
179 barrier_set_state()->inc_trampoline_stubs_count();
180 }
181 }
182
183 int ZBarrierStubC2::trampoline_stubs_count() {
184 return barrier_set_state()->trampoline_stubs_count();
185 }
186
187 int ZBarrierStubC2::stubs_start_offset() {
188 return barrier_set_state()->stubs_start_offset();
189 }
190
191 ZBarrierStubC2::ZBarrierStubC2(const MachNode* node) : BarrierStubC2(node) {}
192
193 ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref) {
194 AARCH64_ONLY(fatal("Should use ZLoadBarrierStubC2Aarch64::create"));
195 ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref);
196 register_stub(stub);
197
198 return stub;
199 }
200
201 ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref)
202 : ZBarrierStubC2(node),
203 _ref_addr(ref_addr),
204 _ref(ref) {
205 assert_different_registers(ref, ref_addr.base());
206 assert_different_registers(ref, ref_addr.index());
207 // The runtime call updates the value of ref, so we should not spill and
208 // reload its outdated value.
209 dont_preserve(ref);
210 }
211
212 Address ZLoadBarrierStubC2::ref_addr() const {
213 return _ref_addr;
214 }
215
216 Register ZLoadBarrierStubC2::ref() const {
217 return _ref;
218 }
219
220 address ZLoadBarrierStubC2::slow_path() const {
221 const uint8_t barrier_data = _node->barrier_data();
222 DecoratorSet decorators = DECORATORS_NONE;
223 if (barrier_data & ZBarrierStrong) {
224 decorators |= ON_STRONG_OOP_REF;
225 }
226 if (barrier_data & ZBarrierWeak) {
227 decorators |= ON_WEAK_OOP_REF;
228 }
229 if (barrier_data & ZBarrierPhantom) {
230 decorators |= ON_PHANTOM_OOP_REF;
231 }
232 if (barrier_data & ZBarrierNoKeepalive) {
233 decorators |= AS_NO_KEEPALIVE;
234 }
235 return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
236 }
237
238 void ZLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
239 ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, static_cast<ZLoadBarrierStubC2*>(this));
240 }
241
242 ZStoreBarrierStubC2* ZStoreBarrierStubC2::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic, bool is_nokeepalive) {
243 AARCH64_ONLY(fatal("Should use ZStoreBarrierStubC2Aarch64::create"));
244 ZStoreBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic, is_nokeepalive);
245 register_stub(stub);
246
247 return stub;
248 }
249
250 ZStoreBarrierStubC2::ZStoreBarrierStubC2(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer,
251 bool is_native, bool is_atomic, bool is_nokeepalive)
252 : ZBarrierStubC2(node),
253 _ref_addr(ref_addr),
254 _new_zaddress(new_zaddress),
255 _new_zpointer(new_zpointer),
256 _is_native(is_native),
257 _is_atomic(is_atomic),
258 _is_nokeepalive(is_nokeepalive) {}
259
260 Address ZStoreBarrierStubC2::ref_addr() const {
261 return _ref_addr;
262 }
263
264 Register ZStoreBarrierStubC2::new_zaddress() const {
265 return _new_zaddress;
266 }
267
268 Register ZStoreBarrierStubC2::new_zpointer() const {
269 return _new_zpointer;
270 }
271
272 bool ZStoreBarrierStubC2::is_native() const {
273 return _is_native;
274 }
275
276 bool ZStoreBarrierStubC2::is_atomic() const {
277 return _is_atomic;
278 }
279
280 bool ZStoreBarrierStubC2::is_nokeepalive() const {
281 return _is_nokeepalive;
282 }
283
284 void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
285 ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast<ZStoreBarrierStubC2*>(this));
286 }
287
288 uint ZBarrierSetC2::estimated_barrier_size(const Node* node) const {
289 uint8_t barrier_data = MemNode::barrier_data(node);
290 assert(barrier_data != 0, "should be a barrier node");
291 uint uncolor_or_color_size = node->is_Load() ? 1 : 2;
292 if ((barrier_data & ZBarrierElided) != 0) {
293 return uncolor_or_color_size;
294 }
295 // A compare and branch corresponds to approximately four fast-path Ideal
296 // nodes (Cmp, Bool, If, If projection). The slow path (If projection and
297 // runtime call) is excluded since the corresponding code is laid out
298 // separately and does not directly affect performance.
299 return uncolor_or_color_size + 4;
300 }
301
302 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
303 return new (comp_arena) ZBarrierSetC2State(comp_arena);
304 }
305
306 void ZBarrierSetC2::late_barrier_analysis() const {
307 compute_liveness_at_stubs();
308 analyze_dominating_barriers();
309 }
310
311 void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
312 MacroAssembler masm(&cb);
313 GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
314 barrier_set_state()->set_stubs_start_offset(masm.offset());
315
316 for (int i = 0; i < stubs->length(); i++) {
317 // Make sure there is enough space in the code buffer
318 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
319 ciEnv::current()->record_failure("CodeCache is full");
320 return;
321 }
322
323 stubs->at(i)->emit_code(masm);
324 }
325
326 masm.flush();
327 }
328
329 int ZBarrierSetC2::estimate_stub_size() const {
330 Compile* const C = Compile::current();
331 BufferBlob* const blob = C->output()->scratch_buffer_blob();
332 GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
333 int size = 0;
334
335 for (int i = 0; i < stubs->length(); i++) {
336 CodeBuffer cb(blob->content_begin(), checked_cast<CodeBuffer::csize_t>((address)C->output()->scratch_locs_memory() - blob->content_begin()));
337 MacroAssembler masm(&cb);
338 stubs->at(i)->emit_code(masm);
339 size += cb.insts_size();
340 }
341
342 return size;
343 }
344
345 static void set_barrier_data(C2Access& access) {
346 if (!ZBarrierSet::barrier_needed(access.decorators(), access.type())) {
347 return;
348 }
349
350 if (access.decorators() & C2_TIGHTLY_COUPLED_ALLOC) {
351 access.set_barrier_data(ZBarrierElided);
352 return;
353 }
354
355 uint8_t barrier_data = 0;
356
357 if (access.decorators() & ON_PHANTOM_OOP_REF) {
358 barrier_data |= ZBarrierPhantom;
359 } else if (access.decorators() & ON_WEAK_OOP_REF) {
360 barrier_data |= ZBarrierWeak;
361 } else {
362 barrier_data |= ZBarrierStrong;
363 }
364
365 if (access.decorators() & IN_NATIVE) {
366 barrier_data |= ZBarrierNative;
367 }
368
369 if (access.decorators() & AS_NO_KEEPALIVE) {
370 barrier_data |= ZBarrierNoKeepalive;
371 }
372
373 access.set_barrier_data(barrier_data);
374 }
375
376 Node* ZBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
377 set_barrier_data(access);
378 return BarrierSetC2::store_at_resolved(access, val);
379 }
380
381 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
382 set_barrier_data(access);
383 return BarrierSetC2::load_at_resolved(access, val_type);
384 }
385
386 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
387 Node* new_val, const Type* val_type) const {
388 set_barrier_data(access);
389 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
390 }
391
392 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
393 Node* new_val, const Type* value_type) const {
394 set_barrier_data(access);
395 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
396 }
397
398 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
399 set_barrier_data(access);
400 return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
401 }
402
403 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
404 bool is_clone, bool is_clone_instance,
405 ArrayCopyPhase phase) const {
406 if (phase == ArrayCopyPhase::Parsing) {
407 return false;
408 }
409 if (phase == ArrayCopyPhase::Optimization) {
410 return is_clone_instance;
411 }
412 // else ArrayCopyPhase::Expansion
413 return type == T_OBJECT || type == T_ARRAY;
414 }
415
416 #define XTOP LP64_ONLY(COMMA phase->top())
417
418 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
419 Node* const src = ac->in(ArrayCopyNode::Src);
420 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
421
422 if (ac->is_clone_array() && ary_ptr != nullptr) {
423 BasicType bt = ary_ptr->elem()->array_element_basic_type();
424 if (is_reference_type(bt) && !ary_ptr->is_flat()) {
425 // Clone object array
426 bt = T_OBJECT;
427 } else {
428 // Clone primitive array
429 bt = T_LONG;
430 }
431
432 Node* const ctrl = ac->in(TypeFunc::Control);
433 Node* const mem = ac->in(TypeFunc::Memory);
434 Node* const src = ac->in(ArrayCopyNode::Src);
435 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
436 Node* const dest = ac->in(ArrayCopyNode::Dest);
437 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
438 Node* length = ac->in(ArrayCopyNode::Length);
439
440 if (bt == T_OBJECT) {
441 // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
442 // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
443 // to the first element in the array when cloning object arrays. Otherwise, load
444 // barriers are applied to parts of the header. Also adjust the length accordingly.
445 assert(src_offset == dest_offset, "should be equal");
446 const jlong offset = src_offset->get_long();
447 if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
448 assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
449 assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
450 length = phase->transform_later(new SubXNode(length, phase->longcon(1))); // Size is in longs
451 src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
452 dest_offset = src_offset;
453 }
454 }
455 Node* const payload_src = phase->basic_plus_adr(src, src_offset);
456 Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
457
458 const char* copyfunc_name = "arraycopy";
459 const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
460
461 const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
462 const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
463
464 Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
465 phase->transform_later(call);
466
467 phase->igvn().replace_node(ac, call);
468 return;
469 }
470
471 // Clone instance or array where 'src' is only known to be an object (ary_ptr
472 // is null). This can happen in bytecode generated dynamically to implement
473 // reflective array clones.
474 clone_in_runtime(phase, ac, ZBarrierSetRuntime::clone_addr(), "ZBarrierSetRuntime::clone");
475 }
476
477 #undef XTOP
478
479 // == Dominating barrier elision ==
480
481 static bool block_has_safepoint(const Block* block, uint from, uint to) {
482 for (uint i = from; i < to; i++) {
483 if (block->get_node(i)->is_MachSafePoint()) {
484 // Safepoint found
485 return true;
486 }
487 }
488
489 // Safepoint not found
490 return false;
491 }
492
493 static bool block_has_safepoint(const Block* block) {
494 return block_has_safepoint(block, 0, block->number_of_nodes());
495 }
496
497 static uint block_index(const Block* block, const Node* node) {
498 for (uint j = 0; j < block->number_of_nodes(); ++j) {
499 if (block->get_node(j) == node) {
500 return j;
501 }
502 }
503 ShouldNotReachHere();
504 return 0;
505 }
506
507 // Look through various node aliases
508 static const Node* look_through_node(const Node* node) {
509 while (node != nullptr) {
510 const Node* new_node = node;
511 if (node->is_Mach()) {
512 const MachNode* const node_mach = node->as_Mach();
513 if (node_mach->ideal_Opcode() == Op_CheckCastPP) {
514 new_node = node->in(1);
515 }
516 if (node_mach->is_SpillCopy()) {
517 new_node = node->in(1);
518 }
519 }
520 if (new_node == node || new_node == nullptr) {
521 break;
522 } else {
523 node = new_node;
524 }
525 }
526
527 return node;
528 }
529
530 // Whether the given offset is undefined.
531 static bool is_undefined(intptr_t offset) {
532 return offset == Type::OffsetTop;
533 }
534
535 // Whether the given offset is unknown.
536 static bool is_unknown(intptr_t offset) {
537 return offset == Type::OffsetBot;
538 }
539
540 // Whether the given offset is concrete (defined and compile-time known).
541 static bool is_concrete(intptr_t offset) {
542 return !is_undefined(offset) && !is_unknown(offset);
543 }
544
545 // Compute base + offset components of the memory address accessed by mach.
546 // Return a node representing the base address, or null if the base cannot be
547 // found or the offset is undefined or a concrete negative value. If a non-null
548 // base is returned, the offset is a concrete, nonnegative value or unknown.
549 static const Node* get_base_and_offset(const MachNode* mach, intptr_t& offset) {
550 const TypePtr* adr_type = nullptr;
551 offset = 0;
552 const Node* base = mach->get_base_and_disp(offset, adr_type);
553
554 if (base == nullptr || base == NodeSentinel) {
555 return nullptr;
556 }
557
558 if (offset == 0 && base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_AddP) {
559 // The memory address is computed by 'base' and fed to 'mach' via an
560 // indirect memory operand (indicated by offset == 0). The ultimate base and
561 // offset can be fetched directly from the inputs and Ideal type of 'base'.
562 offset = base->bottom_type()->isa_oopptr()->offset();
563 // Even if 'base' is not an Ideal AddP node anymore, Matcher::ReduceInst()
564 // guarantees that the base address is still available at the same slot.
565 base = base->in(AddPNode::Base);
566 assert(base != nullptr, "");
567 }
568
569 if (is_undefined(offset) || (is_concrete(offset) && offset < 0)) {
570 return nullptr;
571 }
572
573 return look_through_node(base);
574 }
575
576 // Whether a phi node corresponds to an array allocation.
577 // This test is incomplete: in some edge cases, it might return false even
578 // though the node does correspond to an array allocation.
579 static bool is_array_allocation(const Node* phi) {
580 precond(phi->is_Phi());
581 // Check whether phi has a successor cast (CheckCastPP) to Java array pointer,
582 // possibly below spill copies and other cast nodes. Limit the exploration to
583 // a single path from the phi node consisting of these node types.
584 const Node* current = phi;
585 while (true) {
586 const Node* next = nullptr;
587 for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) {
588 if (!current->fast_out(i)->isa_Mach()) {
589 continue;
590 }
591 const MachNode* succ = current->fast_out(i)->as_Mach();
592 if (succ->ideal_Opcode() == Op_CheckCastPP) {
593 if (succ->get_ptr_type()->isa_aryptr()) {
594 // Cast to Java array pointer: phi corresponds to an array allocation.
595 return true;
596 }
597 // Other cast: record as candidate for further exploration.
598 next = succ;
599 } else if (succ->is_SpillCopy() && next == nullptr) {
600 // Spill copy, and no better candidate found: record as candidate.
601 next = succ;
602 }
603 }
604 if (next == nullptr) {
605 // No evidence found that phi corresponds to an array allocation, and no
606 // candidates available to continue exploring.
607 return false;
608 }
609 // Continue exploring from the best candidate found.
610 current = next;
611 }
612 ShouldNotReachHere();
613 }
614
615 // Match the phi node that connects a TLAB allocation fast path with its slowpath
616 static bool is_allocation(const Node* node) {
617 if (node->req() != 3) {
618 return false;
619 }
620 const Node* const fast_node = node->in(2);
621 if (!fast_node->is_Mach()) {
622 return false;
623 }
624 const MachNode* const fast_mach = fast_node->as_Mach();
625 if (fast_mach->ideal_Opcode() != Op_LoadP) {
626 return false;
627 }
628 const TypePtr* const adr_type = nullptr;
629 intptr_t offset;
630 const Node* const base = get_base_and_offset(fast_mach, offset);
631 if (base == nullptr || !base->is_Mach() || !is_concrete(offset)) {
632 return false;
633 }
634 const MachNode* const base_mach = base->as_Mach();
635 if (base_mach->ideal_Opcode() != Op_ThreadLocal) {
636 return false;
637 }
638 return offset == in_bytes(Thread::tlab_top_offset());
639 }
640
641 static void elide_mach_barrier(MachNode* mach) {
642 mach->set_barrier_data(ZBarrierElided);
643 }
644
645 void ZBarrierSetC2::analyze_dominating_barriers_impl(Node_List& accesses, Node_List& access_dominators) const {
646 Compile* const C = Compile::current();
647 PhaseCFG* const cfg = C->cfg();
648
649 for (uint i = 0; i < accesses.size(); i++) {
650 MachNode* const access = accesses.at(i)->as_Mach();
651 intptr_t access_offset;
652 const Node* const access_obj = get_base_and_offset(access, access_offset);
653 Block* const access_block = cfg->get_block_for_node(access);
654 const uint access_index = block_index(access_block, access);
655
656 if (access_obj == nullptr) {
657 // No information available
658 continue;
659 }
660
661 for (uint j = 0; j < access_dominators.size(); j++) {
662 const Node* const mem = access_dominators.at(j);
663 if (mem->is_Phi()) {
664 // Allocation node
665 if (mem != access_obj) {
666 continue;
667 }
668 if (is_unknown(access_offset) && !is_array_allocation(mem)) {
669 // The accessed address has an unknown offset, but the allocated
670 // object cannot be determined to be an array. Avoid eliding in this
671 // case, to be on the safe side.
672 continue;
673 }
674 assert((is_concrete(access_offset) && access_offset >= 0) || (is_unknown(access_offset) && is_array_allocation(mem)),
675 "candidate allocation-dominated access offsets must be either concrete and nonnegative, or unknown (for array allocations only)");
676 } else {
677 // Access node
678 const MachNode* const mem_mach = mem->as_Mach();
679 intptr_t mem_offset;
680 const Node* const mem_obj = get_base_and_offset(mem_mach, mem_offset);
681
682 if (mem_obj == nullptr ||
683 !is_concrete(access_offset) ||
684 !is_concrete(mem_offset)) {
685 // No information available
686 continue;
687 }
688
689 if (mem_obj != access_obj || mem_offset != access_offset) {
690 // Not the same addresses, not a candidate
691 continue;
692 }
693 assert(is_concrete(access_offset) && access_offset >= 0,
694 "candidate non-allocation-dominated access offsets must be concrete and nonnegative");
695 }
696
697 Block* mem_block = cfg->get_block_for_node(mem);
698 const uint mem_index = block_index(mem_block, mem);
699
700 if (access_block == mem_block) {
701 // Earlier accesses in the same block
702 if (mem_index < access_index && !block_has_safepoint(mem_block, mem_index + 1, access_index)) {
703 elide_mach_barrier(access);
704 }
705 } else if (mem_block->dominates(access_block)) {
706 // Dominating block? Look around for safepoints
707 ResourceMark rm;
708 Block_List stack;
709 VectorSet visited;
710 stack.push(access_block);
711 bool safepoint_found = block_has_safepoint(access_block);
712 while (!safepoint_found && stack.size() > 0) {
713 const Block* const block = stack.pop();
714 if (visited.test_set(block->_pre_order)) {
715 continue;
716 }
717 if (block_has_safepoint(block)) {
718 safepoint_found = true;
719 break;
720 }
721 if (block == mem_block) {
722 continue;
723 }
724
725 // Push predecessor blocks
726 for (uint p = 1; p < block->num_preds(); ++p) {
727 Block* const pred = cfg->get_block_for_node(block->pred(p));
728 stack.push(pred);
729 }
730 }
731
732 if (!safepoint_found) {
733 elide_mach_barrier(access);
734 }
735 }
736 }
737 }
738 }
739
740 void ZBarrierSetC2::analyze_dominating_barriers() const {
741 ResourceMark rm;
742 Compile* const C = Compile::current();
743 PhaseCFG* const cfg = C->cfg();
744
745 Node_List loads;
746 Node_List load_dominators;
747
748 Node_List stores;
749 Node_List store_dominators;
750
751 Node_List atomics;
752 Node_List atomic_dominators;
753
754 // Step 1 - Find accesses and allocations, and track them in lists
755 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
756 const Block* const block = cfg->get_block(i);
757 for (uint j = 0; j < block->number_of_nodes(); ++j) {
758 Node* const node = block->get_node(j);
759 if (node->is_Phi()) {
760 if (is_allocation(node)) {
761 load_dominators.push(node);
762 store_dominators.push(node);
763 // An allocation can't be considered to "dominate" an atomic operation.
764 // For example a CAS requires the memory location to be store-good.
765 // When you have a dominating store or atomic instruction, that is
766 // indeed ensured to be the case. However, as for allocations, the
767 // initialized memory location could be raw null, which isn't store-good.
768 }
769 continue;
770 } else if (!node->is_Mach()) {
771 continue;
772 }
773
774 MachNode* const mach = node->as_Mach();
775 switch (mach->ideal_Opcode()) {
776 case Op_LoadP:
777 if ((mach->barrier_data() & ZBarrierStrong) != 0 &&
778 (mach->barrier_data() & ZBarrierNoKeepalive) == 0) {
779 loads.push(mach);
780 load_dominators.push(mach);
781 }
782 break;
783 case Op_StoreP:
784 if (mach->barrier_data() != 0) {
785 stores.push(mach);
786 load_dominators.push(mach);
787 store_dominators.push(mach);
788 atomic_dominators.push(mach);
789 }
790 break;
791 case Op_CompareAndExchangeP:
792 case Op_CompareAndSwapP:
793 case Op_GetAndSetP:
794 if (mach->barrier_data() != 0) {
795 atomics.push(mach);
796 load_dominators.push(mach);
797 store_dominators.push(mach);
798 atomic_dominators.push(mach);
799 }
800 break;
801
802 default:
803 break;
804 }
805 }
806 }
807
808 // Step 2 - Find dominating accesses or allocations for each access
809 analyze_dominating_barriers_impl(loads, load_dominators);
810 analyze_dominating_barriers_impl(stores, store_dominators);
811 analyze_dominating_barriers_impl(atomics, atomic_dominators);
812 }
813
814
815 void ZBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
816 eliminate_gc_barrier_data(node);
817 }
818
819 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
820 if (node->is_LoadStore()) {
821 LoadStoreNode* loadstore = node->as_LoadStore();
822 loadstore->set_barrier_data(ZBarrierElided);
823 } else if (node->is_Mem()) {
824 MemNode* mem = node->as_Mem();
825 mem->set_barrier_data(ZBarrierElided);
826 }
827 }
828
829 #ifndef PRODUCT
830 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
831 if ((mach->barrier_data() & ZBarrierStrong) != 0) {
832 st->print("strong ");
833 }
834 if ((mach->barrier_data() & ZBarrierWeak) != 0) {
835 st->print("weak ");
836 }
837 if ((mach->barrier_data() & ZBarrierPhantom) != 0) {
838 st->print("phantom ");
839 }
840 if ((mach->barrier_data() & ZBarrierNoKeepalive) != 0) {
841 st->print("nokeepalive ");
842 }
843 if ((mach->barrier_data() & ZBarrierNative) != 0) {
844 st->print("native ");
845 }
846 if ((mach->barrier_data() & ZBarrierElided) != 0) {
847 st->print("elided ");
848 }
849 }
850 #endif // !PRODUCT