1 /*
2 * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "asm/macroAssembler.hpp"
26 #include "classfile/javaClasses.hpp"
27 #include "gc/z/c2/zBarrierSetC2.hpp"
28 #include "gc/z/zBarrierSet.hpp"
29 #include "gc/z/zBarrierSetAssembler.hpp"
30 #include "gc/z/zBarrierSetRuntime.hpp"
31 #include "opto/arraycopynode.hpp"
32 #include "opto/addnode.hpp"
33 #include "opto/block.hpp"
34 #include "opto/compile.hpp"
35 #include "opto/graphKit.hpp"
36 #include "opto/machnode.hpp"
37 #include "opto/macro.hpp"
38 #include "opto/memnode.hpp"
39 #include "opto/node.hpp"
40 #include "opto/output.hpp"
41 #include "opto/regalloc.hpp"
42 #include "opto/rootnode.hpp"
43 #include "opto/runtime.hpp"
44 #include "opto/type.hpp"
45 #include "utilities/growableArray.hpp"
46 #include "utilities/macros.hpp"
47
48 template<typename K, typename V, size_t _table_size>
49 class ZArenaHashtable : public ResourceObj {
50 class ZArenaHashtableEntry : public ResourceObj {
51 public:
52 ZArenaHashtableEntry* _next;
53 K _key;
54 V _value;
55 };
56
57 static const size_t _table_mask = _table_size - 1;
58
59 Arena* _arena;
60 ZArenaHashtableEntry* _table[_table_size];
61
62 public:
63 class Iterator {
64 ZArenaHashtable* _table;
65 ZArenaHashtableEntry* _current_entry;
66 size_t _current_index;
67
68 public:
69 Iterator(ZArenaHashtable* table)
70 : _table(table),
71 _current_entry(table->_table[0]),
72 _current_index(0) {
73 if (_current_entry == nullptr) {
74 next();
75 }
76 }
77
78 bool has_next() { return _current_entry != nullptr; }
79 K key() { return _current_entry->_key; }
80 V value() { return _current_entry->_value; }
81
82 void next() {
83 if (_current_entry != nullptr) {
84 _current_entry = _current_entry->_next;
85 }
86 while (_current_entry == nullptr && ++_current_index < _table_size) {
87 _current_entry = _table->_table[_current_index];
88 }
89 }
90 };
91
92 ZArenaHashtable(Arena* arena)
93 : _arena(arena),
94 _table() {
95 Copy::zero_to_bytes(&_table, sizeof(_table));
96 }
97
98 void add(K key, V value) {
99 ZArenaHashtableEntry* entry = new (_arena) ZArenaHashtableEntry();
100 entry->_key = key;
101 entry->_value = value;
102 entry->_next = _table[key & _table_mask];
103 _table[key & _table_mask] = entry;
104 }
105
106 V* get(K key) const {
107 for (ZArenaHashtableEntry* e = _table[key & _table_mask]; e != nullptr; e = e->_next) {
108 if (e->_key == key) {
109 return &(e->_value);
110 }
111 }
112 return nullptr;
113 }
114
115 Iterator iterator() {
116 return Iterator(this);
117 }
118 };
119
120 typedef ZArenaHashtable<intptr_t, bool, 4> ZOffsetTable;
121
122 class ZBarrierSetC2State : public ArenaObj {
123 private:
124 GrowableArray<ZBarrierStubC2*>* _stubs;
125 Node_Array _live;
126 int _trampoline_stubs_count;
127 int _stubs_start_offset;
128
129 public:
130 ZBarrierSetC2State(Arena* arena)
131 : _stubs(new (arena) GrowableArray<ZBarrierStubC2*>(arena, 8, 0, nullptr)),
132 _live(arena),
133 _trampoline_stubs_count(0),
134 _stubs_start_offset(0) {}
135
136 GrowableArray<ZBarrierStubC2*>* stubs() {
137 return _stubs;
138 }
139
140 RegMask* live(const Node* node) {
141 if (!node->is_Mach()) {
142 // Don't need liveness for non-MachNodes
143 return nullptr;
144 }
145
146 const MachNode* const mach = node->as_Mach();
147 if (mach->barrier_data() == ZBarrierElided) {
148 // Don't need liveness data for nodes without barriers
149 return nullptr;
150 }
151
152 RegMask* live = (RegMask*)_live[node->_idx];
153 if (live == nullptr) {
154 live = new (Compile::current()->comp_arena()->AmallocWords(sizeof(RegMask))) RegMask();
155 _live.map(node->_idx, (Node*)live);
156 }
157
158 return live;
159 }
160
161 void inc_trampoline_stubs_count() {
162 assert(_trampoline_stubs_count != INT_MAX, "Overflow");
163 ++_trampoline_stubs_count;
164 }
165
166 int trampoline_stubs_count() {
167 return _trampoline_stubs_count;
168 }
169
170 void set_stubs_start_offset(int offset) {
171 _stubs_start_offset = offset;
172 }
173
174 int stubs_start_offset() {
175 return _stubs_start_offset;
176 }
177 };
178
179 static ZBarrierSetC2State* barrier_set_state() {
180 return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
181 }
182
183 void ZBarrierStubC2::register_stub(ZBarrierStubC2* stub) {
184 if (!Compile::current()->output()->in_scratch_emit_size()) {
185 barrier_set_state()->stubs()->append(stub);
186 }
187 }
188
189 void ZBarrierStubC2::inc_trampoline_stubs_count() {
190 if (!Compile::current()->output()->in_scratch_emit_size()) {
191 barrier_set_state()->inc_trampoline_stubs_count();
192 }
193 }
194
195 int ZBarrierStubC2::trampoline_stubs_count() {
196 return barrier_set_state()->trampoline_stubs_count();
197 }
198
199 int ZBarrierStubC2::stubs_start_offset() {
200 return barrier_set_state()->stubs_start_offset();
201 }
202
203 ZBarrierStubC2::ZBarrierStubC2(const MachNode* node)
204 : _node(node),
205 _entry(),
206 _continuation() {}
207
208 Register ZBarrierStubC2::result() const {
209 return noreg;
210 }
211
212 RegMask& ZBarrierStubC2::live() const {
213 return *barrier_set_state()->live(_node);
214 }
215
216 Label* ZBarrierStubC2::entry() {
217 // The _entry will never be bound when in_scratch_emit_size() is true.
218 // However, we still need to return a label that is not bound now, but
219 // will eventually be bound. Any eventually bound label will do, as it
220 // will only act as a placeholder, so we return the _continuation label.
221 return Compile::current()->output()->in_scratch_emit_size() ? &_continuation : &_entry;
222 }
223
224 Label* ZBarrierStubC2::continuation() {
225 return &_continuation;
226 }
227
228 ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref) {
229 ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref);
230 register_stub(stub);
231
232 return stub;
233 }
234
235 ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref)
236 : ZBarrierStubC2(node),
237 _ref_addr(ref_addr),
238 _ref(ref) {
239 assert_different_registers(ref, ref_addr.base());
240 assert_different_registers(ref, ref_addr.index());
241 }
242
243 Address ZLoadBarrierStubC2::ref_addr() const {
244 return _ref_addr;
245 }
246
247 Register ZLoadBarrierStubC2::ref() const {
248 return _ref;
249 }
250
251 Register ZLoadBarrierStubC2::result() const {
252 return ref();
253 }
254
255 address ZLoadBarrierStubC2::slow_path() const {
256 const uint8_t barrier_data = _node->barrier_data();
257 DecoratorSet decorators = DECORATORS_NONE;
258 if (barrier_data & ZBarrierStrong) {
259 decorators |= ON_STRONG_OOP_REF;
260 }
261 if (barrier_data & ZBarrierWeak) {
262 decorators |= ON_WEAK_OOP_REF;
263 }
264 if (barrier_data & ZBarrierPhantom) {
265 decorators |= ON_PHANTOM_OOP_REF;
266 }
267 if (barrier_data & ZBarrierNoKeepalive) {
268 decorators |= AS_NO_KEEPALIVE;
269 }
270 return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
271 }
272
273 void ZLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
274 ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, static_cast<ZLoadBarrierStubC2*>(this));
275 }
276
277 ZStoreBarrierStubC2* ZStoreBarrierStubC2::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic) {
278 ZStoreBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic);
279 register_stub(stub);
280
281 return stub;
282 }
283
284 ZStoreBarrierStubC2::ZStoreBarrierStubC2(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic)
285 : ZBarrierStubC2(node),
286 _ref_addr(ref_addr),
287 _new_zaddress(new_zaddress),
288 _new_zpointer(new_zpointer),
289 _is_native(is_native),
290 _is_atomic(is_atomic) {}
291
292 Address ZStoreBarrierStubC2::ref_addr() const {
293 return _ref_addr;
294 }
295
296 Register ZStoreBarrierStubC2::new_zaddress() const {
297 return _new_zaddress;
298 }
299
300 Register ZStoreBarrierStubC2::new_zpointer() const {
301 return _new_zpointer;
302 }
303
304 bool ZStoreBarrierStubC2::is_native() const {
305 return _is_native;
306 }
307
308 bool ZStoreBarrierStubC2::is_atomic() const {
309 return _is_atomic;
310 }
311
312 Register ZStoreBarrierStubC2::result() const {
313 return noreg;
314 }
315
316 void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
317 ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast<ZStoreBarrierStubC2*>(this));
318 }
319
320 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
321 return new (comp_arena) ZBarrierSetC2State(comp_arena);
322 }
323
324 void ZBarrierSetC2::late_barrier_analysis() const {
325 compute_liveness_at_stubs();
326 analyze_dominating_barriers();
327 }
328
329 void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
330 MacroAssembler masm(&cb);
331 GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
332 barrier_set_state()->set_stubs_start_offset(masm.offset());
333
334 for (int i = 0; i < stubs->length(); i++) {
335 // Make sure there is enough space in the code buffer
336 if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
337 ciEnv::current()->record_failure("CodeCache is full");
338 return;
339 }
340
341 stubs->at(i)->emit_code(masm);
342 }
343
344 masm.flush();
345 }
346
347 int ZBarrierSetC2::estimate_stub_size() const {
348 Compile* const C = Compile::current();
349 BufferBlob* const blob = C->output()->scratch_buffer_blob();
350 GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
351 int size = 0;
352
353 for (int i = 0; i < stubs->length(); i++) {
354 CodeBuffer cb(blob->content_begin(), (address)C->output()->scratch_locs_memory() - blob->content_begin());
355 MacroAssembler masm(&cb);
356 stubs->at(i)->emit_code(masm);
357 size += cb.insts_size();
358 }
359
360 return size;
361 }
362
363 static void set_barrier_data(C2Access& access) {
364 if (!ZBarrierSet::barrier_needed(access.decorators(), access.type())) {
365 return;
366 }
367
368 if (access.decorators() & C2_TIGHTLY_COUPLED_ALLOC) {
369 access.set_barrier_data(ZBarrierElided);
370 return;
371 }
372
373 uint8_t barrier_data = 0;
374
375 if (access.decorators() & ON_PHANTOM_OOP_REF) {
376 barrier_data |= ZBarrierPhantom;
377 } else if (access.decorators() & ON_WEAK_OOP_REF) {
378 barrier_data |= ZBarrierWeak;
379 } else {
380 barrier_data |= ZBarrierStrong;
381 }
382
383 if (access.decorators() & IN_NATIVE) {
384 barrier_data |= ZBarrierNative;
385 }
386
387 if (access.decorators() & AS_NO_KEEPALIVE) {
388 barrier_data |= ZBarrierNoKeepalive;
389 }
390
391 access.set_barrier_data(barrier_data);
392 }
393
394 Node* ZBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
395 set_barrier_data(access);
396 return BarrierSetC2::store_at_resolved(access, val);
397 }
398
399 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
400 set_barrier_data(access);
401 return BarrierSetC2::load_at_resolved(access, val_type);
402 }
403
404 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
405 Node* new_val, const Type* val_type) const {
406 set_barrier_data(access);
407 return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
408 }
409
410 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
411 Node* new_val, const Type* value_type) const {
412 set_barrier_data(access);
413 return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
414 }
415
416 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
417 set_barrier_data(access);
418 return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
419 }
420
421 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
422 bool is_clone, bool is_clone_instance,
423 ArrayCopyPhase phase) const {
424 if (phase == ArrayCopyPhase::Parsing) {
425 return false;
426 }
427 if (phase == ArrayCopyPhase::Optimization) {
428 return is_clone_instance;
429 }
430 // else ArrayCopyPhase::Expansion
431 return type == T_OBJECT || type == T_ARRAY;
432 }
433
434 // This TypeFunc assumes a 64bit system
435 static const TypeFunc* clone_type() {
436 // Create input type (domain)
437 const Type** const domain_fields = TypeTuple::fields(4);
438 domain_fields[TypeFunc::Parms + 0] = TypeInstPtr::NOTNULL; // src
439 domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL; // dst
440 domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG; // size lower
441 domain_fields[TypeFunc::Parms + 3] = Type::HALF; // size upper
442 const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
443
444 // Create result type (range)
445 const Type** const range_fields = TypeTuple::fields(0);
446 const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
447
448 return TypeFunc::make(domain, range);
449 }
450
451 #define XTOP LP64_ONLY(COMMA phase->top())
452
453 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
454 Node* const src = ac->in(ArrayCopyNode::Src);
455 const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
456
457 if (ac->is_clone_array() && ary_ptr != nullptr) {
458 BasicType bt = ary_ptr->elem()->array_element_basic_type();
459 if (is_reference_type(bt)) {
460 // Clone object array
461 bt = T_OBJECT;
462 } else {
463 // Clone primitive array
464 bt = T_LONG;
465 }
466
467 Node* const ctrl = ac->in(TypeFunc::Control);
468 Node* const mem = ac->in(TypeFunc::Memory);
469 Node* const src = ac->in(ArrayCopyNode::Src);
470 Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
471 Node* const dest = ac->in(ArrayCopyNode::Dest);
472 Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
473 Node* length = ac->in(ArrayCopyNode::Length);
474
475 if (bt == T_OBJECT) {
476 // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
477 // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
478 // to the first element in the array when cloning object arrays. Otherwise, load
479 // barriers are applied to parts of the header. Also adjust the length accordingly.
480 assert(src_offset == dest_offset, "should be equal");
481 const jlong offset = src_offset->get_long();
482 if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
483 assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
484 assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
485 length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs
486 src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
487 dest_offset = src_offset;
488 }
489 }
490 Node* const payload_src = phase->basic_plus_adr(src, src_offset);
491 Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
492
493 const char* copyfunc_name = "arraycopy";
494 const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
495
496 const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
497 const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
498
499 Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
500 phase->transform_later(call);
501
502 phase->igvn().replace_node(ac, call);
503 return;
504 }
505
506 // Clone instance
507 Node* const ctrl = ac->in(TypeFunc::Control);
508 Node* const mem = ac->in(TypeFunc::Memory);
509 Node* const dst = ac->in(ArrayCopyNode::Dest);
510 Node* const size = ac->in(ArrayCopyNode::Length);
511
512 assert(size->bottom_type()->is_long(), "Should be long");
513
514 // The native clone we are calling here expects the instance size in words
515 // Add header/offset size to payload size to get instance size.
516 Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
517 Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
518
519 Node* const call = phase->make_leaf_call(ctrl,
520 mem,
521 clone_type(),
522 ZBarrierSetRuntime::clone_addr(),
523 "ZBarrierSetRuntime::clone",
524 TypeRawPtr::BOTTOM,
525 src,
526 dst,
527 full_size,
528 phase->top());
529 phase->transform_later(call);
530 phase->igvn().replace_node(ac, call);
531 }
532
533 #undef XTOP
534
535 // == Dominating barrier elision ==
536
537 static bool block_has_safepoint(const Block* block, uint from, uint to) {
538 for (uint i = from; i < to; i++) {
539 if (block->get_node(i)->is_MachSafePoint()) {
540 // Safepoint found
541 return true;
542 }
543 }
544
545 // Safepoint not found
546 return false;
547 }
548
549 static bool block_has_safepoint(const Block* block) {
550 return block_has_safepoint(block, 0, block->number_of_nodes());
551 }
552
553 static uint block_index(const Block* block, const Node* node) {
554 for (uint j = 0; j < block->number_of_nodes(); ++j) {
555 if (block->get_node(j) == node) {
556 return j;
557 }
558 }
559 ShouldNotReachHere();
560 return 0;
561 }
562
563 // Look through various node aliases
564 static const Node* look_through_node(const Node* node) {
565 while (node != nullptr) {
566 const Node* new_node = node;
567 if (node->is_Mach()) {
568 const MachNode* const node_mach = node->as_Mach();
569 if (node_mach->ideal_Opcode() == Op_CheckCastPP) {
570 new_node = node->in(1);
571 }
572 if (node_mach->is_SpillCopy()) {
573 new_node = node->in(1);
574 }
575 }
576 if (new_node == node || new_node == nullptr) {
577 break;
578 } else {
579 node = new_node;
580 }
581 }
582
583 return node;
584 }
585
586 // Whether the given offset is undefined.
587 static bool is_undefined(intptr_t offset) {
588 return offset == Type::OffsetTop;
589 }
590
591 // Whether the given offset is unknown.
592 static bool is_unknown(intptr_t offset) {
593 return offset == Type::OffsetBot;
594 }
595
596 // Whether the given offset is concrete (defined and compile-time known).
597 static bool is_concrete(intptr_t offset) {
598 return !is_undefined(offset) && !is_unknown(offset);
599 }
600
601 // Compute base + offset components of the memory address accessed by mach.
602 // Return a node representing the base address, or null if the base cannot be
603 // found or the offset is undefined or a concrete negative value. If a non-null
604 // base is returned, the offset is a concrete, nonnegative value or unknown.
605 static const Node* get_base_and_offset(const MachNode* mach, intptr_t& offset) {
606 const TypePtr* adr_type = nullptr;
607 offset = 0;
608 const Node* base = mach->get_base_and_disp(offset, adr_type);
609
610 if (base == nullptr || base == NodeSentinel) {
611 return nullptr;
612 }
613
614 if (offset == 0 && base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_AddP) {
615 // The memory address is computed by 'base' and fed to 'mach' via an
616 // indirect memory operand (indicated by offset == 0). The ultimate base and
617 // offset can be fetched directly from the inputs and Ideal type of 'base'.
618 offset = base->bottom_type()->isa_oopptr()->offset();
619 // Even if 'base' is not an Ideal AddP node anymore, Matcher::ReduceInst()
620 // guarantees that the base address is still available at the same slot.
621 base = base->in(AddPNode::Base);
622 assert(base != nullptr, "");
623 }
624
625 if (is_undefined(offset) || (is_concrete(offset) && offset < 0)) {
626 return nullptr;
627 }
628
629 return look_through_node(base);
630 }
631
632 // Whether a phi node corresponds to an array allocation.
633 // This test is incomplete: in some edge cases, it might return false even
634 // though the node does correspond to an array allocation.
635 static bool is_array_allocation(const Node* phi) {
636 precond(phi->is_Phi());
637 // Check whether phi has a successor cast (CheckCastPP) to Java array pointer,
638 // possibly below spill copies and other cast nodes. Limit the exploration to
639 // a single path from the phi node consisting of these node types.
640 const Node* current = phi;
641 while (true) {
642 const Node* next = nullptr;
643 for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) {
644 if (!current->fast_out(i)->isa_Mach()) {
645 continue;
646 }
647 const MachNode* succ = current->fast_out(i)->as_Mach();
648 if (succ->ideal_Opcode() == Op_CheckCastPP) {
649 if (succ->get_ptr_type()->isa_aryptr()) {
650 // Cast to Java array pointer: phi corresponds to an array allocation.
651 return true;
652 }
653 // Other cast: record as candidate for further exploration.
654 next = succ;
655 } else if (succ->is_SpillCopy() && next == nullptr) {
656 // Spill copy, and no better candidate found: record as candidate.
657 next = succ;
658 }
659 }
660 if (next == nullptr) {
661 // No evidence found that phi corresponds to an array allocation, and no
662 // candidates available to continue exploring.
663 return false;
664 }
665 // Continue exploring from the best candidate found.
666 current = next;
667 }
668 ShouldNotReachHere();
669 }
670
671 // Match the phi node that connects a TLAB allocation fast path with its slowpath
672 static bool is_allocation(const Node* node) {
673 if (node->req() != 3) {
674 return false;
675 }
676 const Node* const fast_node = node->in(2);
677 if (!fast_node->is_Mach()) {
678 return false;
679 }
680 const MachNode* const fast_mach = fast_node->as_Mach();
681 if (fast_mach->ideal_Opcode() != Op_LoadP) {
682 return false;
683 }
684 const TypePtr* const adr_type = nullptr;
685 intptr_t offset;
686 const Node* const base = get_base_and_offset(fast_mach, offset);
687 if (base == nullptr || !base->is_Mach() || !is_concrete(offset)) {
688 return false;
689 }
690 const MachNode* const base_mach = base->as_Mach();
691 if (base_mach->ideal_Opcode() != Op_ThreadLocal) {
692 return false;
693 }
694 return offset == in_bytes(Thread::tlab_top_offset());
695 }
696
697 static void elide_mach_barrier(MachNode* mach) {
698 mach->set_barrier_data(ZBarrierElided);
699 }
700
701 void ZBarrierSetC2::analyze_dominating_barriers_impl(Node_List& accesses, Node_List& access_dominators) const {
702 Compile* const C = Compile::current();
703 PhaseCFG* const cfg = C->cfg();
704
705 for (uint i = 0; i < accesses.size(); i++) {
706 MachNode* const access = accesses.at(i)->as_Mach();
707 intptr_t access_offset;
708 const Node* const access_obj = get_base_and_offset(access, access_offset);
709 Block* const access_block = cfg->get_block_for_node(access);
710 const uint access_index = block_index(access_block, access);
711
712 if (access_obj == nullptr) {
713 // No information available
714 continue;
715 }
716
717 for (uint j = 0; j < access_dominators.size(); j++) {
718 const Node* const mem = access_dominators.at(j);
719 if (mem->is_Phi()) {
720 // Allocation node
721 if (mem != access_obj) {
722 continue;
723 }
724 if (is_unknown(access_offset) && !is_array_allocation(mem)) {
725 // The accessed address has an unknown offset, but the allocated
726 // object cannot be determined to be an array. Avoid eliding in this
727 // case, to be on the safe side.
728 continue;
729 }
730 assert((is_concrete(access_offset) && access_offset >= 0) || (is_unknown(access_offset) && is_array_allocation(mem)),
731 "candidate allocation-dominated access offsets must be either concrete and nonnegative, or unknown (for array allocations only)");
732 } else {
733 // Access node
734 const MachNode* const mem_mach = mem->as_Mach();
735 intptr_t mem_offset;
736 const Node* const mem_obj = get_base_and_offset(mem_mach, mem_offset);
737
738 if (mem_obj == nullptr ||
739 !is_concrete(access_offset) ||
740 !is_concrete(mem_offset)) {
741 // No information available
742 continue;
743 }
744
745 if (mem_obj != access_obj || mem_offset != access_offset) {
746 // Not the same addresses, not a candidate
747 continue;
748 }
749 assert(is_concrete(access_offset) && access_offset >= 0,
750 "candidate non-allocation-dominated access offsets must be concrete and nonnegative");
751 }
752
753 Block* mem_block = cfg->get_block_for_node(mem);
754 const uint mem_index = block_index(mem_block, mem);
755
756 if (access_block == mem_block) {
757 // Earlier accesses in the same block
758 if (mem_index < access_index && !block_has_safepoint(mem_block, mem_index + 1, access_index)) {
759 elide_mach_barrier(access);
760 }
761 } else if (mem_block->dominates(access_block)) {
762 // Dominating block? Look around for safepoints
763 ResourceMark rm;
764 Block_List stack;
765 VectorSet visited;
766 stack.push(access_block);
767 bool safepoint_found = block_has_safepoint(access_block);
768 while (!safepoint_found && stack.size() > 0) {
769 const Block* const block = stack.pop();
770 if (visited.test_set(block->_pre_order)) {
771 continue;
772 }
773 if (block_has_safepoint(block)) {
774 safepoint_found = true;
775 break;
776 }
777 if (block == mem_block) {
778 continue;
779 }
780
781 // Push predecessor blocks
782 for (uint p = 1; p < block->num_preds(); ++p) {
783 Block* const pred = cfg->get_block_for_node(block->pred(p));
784 stack.push(pred);
785 }
786 }
787
788 if (!safepoint_found) {
789 elide_mach_barrier(access);
790 }
791 }
792 }
793 }
794 }
795
796 void ZBarrierSetC2::analyze_dominating_barriers() const {
797 ResourceMark rm;
798 Compile* const C = Compile::current();
799 PhaseCFG* const cfg = C->cfg();
800
801 Node_List loads;
802 Node_List load_dominators;
803
804 Node_List stores;
805 Node_List store_dominators;
806
807 Node_List atomics;
808 Node_List atomic_dominators;
809
810 // Step 1 - Find accesses and allocations, and track them in lists
811 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
812 const Block* const block = cfg->get_block(i);
813 for (uint j = 0; j < block->number_of_nodes(); ++j) {
814 Node* const node = block->get_node(j);
815 if (node->is_Phi()) {
816 if (is_allocation(node)) {
817 load_dominators.push(node);
818 store_dominators.push(node);
819 // An allocation can't be considered to "dominate" an atomic operation.
820 // For example a CAS requires the memory location to be store-good.
821 // When you have a dominating store or atomic instruction, that is
822 // indeed ensured to be the case. However, as for allocations, the
823 // initialized memory location could be raw null, which isn't store-good.
824 }
825 continue;
826 } else if (!node->is_Mach()) {
827 continue;
828 }
829
830 MachNode* const mach = node->as_Mach();
831 switch (mach->ideal_Opcode()) {
832 case Op_LoadP:
833 if ((mach->barrier_data() & ZBarrierStrong) != 0 &&
834 (mach->barrier_data() & ZBarrierNoKeepalive) == 0) {
835 loads.push(mach);
836 load_dominators.push(mach);
837 }
838 break;
839 case Op_StoreP:
840 if (mach->barrier_data() != 0) {
841 stores.push(mach);
842 load_dominators.push(mach);
843 store_dominators.push(mach);
844 atomic_dominators.push(mach);
845 }
846 break;
847 case Op_CompareAndExchangeP:
848 case Op_CompareAndSwapP:
849 case Op_GetAndSetP:
850 if (mach->barrier_data() != 0) {
851 atomics.push(mach);
852 load_dominators.push(mach);
853 store_dominators.push(mach);
854 atomic_dominators.push(mach);
855 }
856 break;
857
858 default:
859 break;
860 }
861 }
862 }
863
864 // Step 2 - Find dominating accesses or allocations for each access
865 analyze_dominating_barriers_impl(loads, load_dominators);
866 analyze_dominating_barriers_impl(stores, store_dominators);
867 analyze_dominating_barriers_impl(atomics, atomic_dominators);
868 }
869
870 // == Reduced spilling optimization ==
871
872 void ZBarrierSetC2::compute_liveness_at_stubs() const {
873 ResourceMark rm;
874 Compile* const C = Compile::current();
875 Arena* const A = Thread::current()->resource_area();
876 PhaseCFG* const cfg = C->cfg();
877 PhaseRegAlloc* const regalloc = C->regalloc();
878 RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
879 ZBarrierSetAssembler* const bs = ZBarrierSet::assembler();
880 Block_List worklist;
881
882 for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
883 new ((void*)(live + i)) RegMask();
884 worklist.push(cfg->get_block(i));
885 }
886
887 while (worklist.size() > 0) {
888 const Block* const block = worklist.pop();
889 RegMask& old_live = live[block->_pre_order];
890 RegMask new_live;
891
892 // Initialize to union of successors
893 for (uint i = 0; i < block->_num_succs; i++) {
894 const uint succ_id = block->_succs[i]->_pre_order;
895 new_live.OR(live[succ_id]);
896 }
897
898 // Walk block backwards, computing liveness
899 for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
900 const Node* const node = block->get_node(i);
901
902 // Remove def bits
903 const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
904 const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
905 if (first != OptoReg::Bad) {
906 new_live.Remove(first);
907 }
908 if (second != OptoReg::Bad) {
909 new_live.Remove(second);
910 }
911
912 // Add use bits
913 for (uint j = 1; j < node->req(); ++j) {
914 const Node* const use = node->in(j);
915 const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
916 const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
917 if (first != OptoReg::Bad) {
918 new_live.Insert(first);
919 }
920 if (second != OptoReg::Bad) {
921 new_live.Insert(second);
922 }
923 }
924
925 // If this node tracks liveness, update it
926 RegMask* const regs = barrier_set_state()->live(node);
927 if (regs != nullptr) {
928 regs->OR(new_live);
929 }
930 }
931
932 // Now at block top, see if we have any changes
933 new_live.SUBTRACT(old_live);
934 if (new_live.is_NotEmpty()) {
935 // Liveness has refined, update and propagate to prior blocks
936 old_live.OR(new_live);
937 for (uint i = 1; i < block->num_preds(); ++i) {
938 Block* const pred = cfg->get_block_for_node(block->pred(i));
939 worklist.push(pred);
940 }
941 }
942 }
943 }
944
945 void ZBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
946 eliminate_gc_barrier_data(node);
947 }
948
949 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
950 if (node->is_LoadStore()) {
951 LoadStoreNode* loadstore = node->as_LoadStore();
952 loadstore->set_barrier_data(ZBarrierElided);
953 } else if (node->is_Mem()) {
954 MemNode* mem = node->as_Mem();
955 mem->set_barrier_data(ZBarrierElided);
956 }
957 }
958
959 #ifndef PRODUCT
960 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
961 if ((mach->barrier_data() & ZBarrierStrong) != 0) {
962 st->print("strong ");
963 }
964 if ((mach->barrier_data() & ZBarrierWeak) != 0) {
965 st->print("weak ");
966 }
967 if ((mach->barrier_data() & ZBarrierPhantom) != 0) {
968 st->print("phantom ");
969 }
970 if ((mach->barrier_data() & ZBarrierNoKeepalive) != 0) {
971 st->print("nokeepalive ");
972 }
973 if ((mach->barrier_data() & ZBarrierNative) != 0) {
974 st->print("native ");
975 }
976 if ((mach->barrier_data() & ZBarrierElided) != 0) {
977 st->print("elided ");
978 }
979 }
980 #endif // !PRODUCT