1 /*
  2  * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #include "precompiled.hpp"
 25 #include "asm/macroAssembler.hpp"
 26 #include "classfile/javaClasses.hpp"
 27 #include "gc/z/c2/zBarrierSetC2.hpp"
 28 #include "gc/z/zBarrierSet.hpp"
 29 #include "gc/z/zBarrierSetAssembler.hpp"
 30 #include "gc/z/zBarrierSetRuntime.hpp"
 31 #include "opto/arraycopynode.hpp"
 32 #include "opto/addnode.hpp"
 33 #include "opto/block.hpp"
 34 #include "opto/compile.hpp"
 35 #include "opto/graphKit.hpp"
 36 #include "opto/machnode.hpp"
 37 #include "opto/macro.hpp"
 38 #include "opto/memnode.hpp"
 39 #include "opto/node.hpp"
 40 #include "opto/output.hpp"
 41 #include "opto/regalloc.hpp"
 42 #include "opto/rootnode.hpp"
 43 #include "opto/runtime.hpp"
 44 #include "opto/type.hpp"
 45 #include "utilities/debug.hpp"
 46 #include "utilities/growableArray.hpp"
 47 #include "utilities/macros.hpp"
 48 
 49 template<typename K, typename V, size_t _table_size>
 50 class ZArenaHashtable : public ResourceObj {
 51   class ZArenaHashtableEntry : public ResourceObj {
 52   public:
 53     ZArenaHashtableEntry* _next;
 54     K _key;
 55     V _value;
 56   };
 57 
 58   static const size_t _table_mask = _table_size - 1;
 59 
 60   Arena* _arena;
 61   ZArenaHashtableEntry* _table[_table_size];
 62 
 63 public:
 64   class Iterator {
 65     ZArenaHashtable* _table;
 66     ZArenaHashtableEntry* _current_entry;
 67     size_t _current_index;
 68 
 69   public:
 70     Iterator(ZArenaHashtable* table)
 71       : _table(table),
 72         _current_entry(table->_table[0]),
 73         _current_index(0) {
 74       if (_current_entry == nullptr) {
 75         next();
 76       }
 77     }
 78 
 79     bool has_next() { return _current_entry != nullptr; }
 80     K key()         { return _current_entry->_key; }
 81     V value()       { return _current_entry->_value; }
 82 
 83     void next() {
 84       if (_current_entry != nullptr) {
 85         _current_entry = _current_entry->_next;
 86       }
 87       while (_current_entry == nullptr && ++_current_index < _table_size) {
 88         _current_entry = _table->_table[_current_index];
 89       }
 90     }
 91   };
 92 
 93   ZArenaHashtable(Arena* arena)
 94     : _arena(arena),
 95       _table() {
 96     Copy::zero_to_bytes(&_table, sizeof(_table));
 97   }
 98 
 99   void add(K key, V value) {
100     ZArenaHashtableEntry* entry = new (_arena) ZArenaHashtableEntry();
101     entry->_key = key;
102     entry->_value = value;
103     entry->_next = _table[key & _table_mask];
104     _table[key & _table_mask] = entry;
105   }
106 
107   V* get(K key) const {
108     for (ZArenaHashtableEntry* e = _table[key & _table_mask]; e != nullptr; e = e->_next) {
109       if (e->_key == key) {
110         return &(e->_value);
111       }
112     }
113     return nullptr;
114   }
115 
116   Iterator iterator() {
117     return Iterator(this);
118   }
119 };
120 
121 typedef ZArenaHashtable<intptr_t, bool, 4> ZOffsetTable;
122 
123 class ZBarrierSetC2State : public BarrierSetC2State {
124 private:
125   GrowableArray<ZBarrierStubC2*>* _stubs;
126   int                             _trampoline_stubs_count;
127   int                             _stubs_start_offset;
128 
129 public:
130   ZBarrierSetC2State(Arena* arena)
131     : BarrierSetC2State(arena),
132       _stubs(new (arena) GrowableArray<ZBarrierStubC2*>(arena, 8,  0, nullptr)),
133       _trampoline_stubs_count(0),
134       _stubs_start_offset(0) {}
135 
136   GrowableArray<ZBarrierStubC2*>* stubs() {
137     return _stubs;
138   }
139 
140   bool needs_liveness_data(const MachNode* mach) const {
141     // Don't need liveness data for nodes without barriers
142     return mach->barrier_data() != ZBarrierElided;
143   }
144 
145   bool needs_livein_data() const {
146     return true;
147   }
148 
149   void inc_trampoline_stubs_count() {
150     assert(_trampoline_stubs_count != INT_MAX, "Overflow");
151     ++_trampoline_stubs_count;
152   }
153 
154   int trampoline_stubs_count() {
155     return _trampoline_stubs_count;
156   }
157 
158   void set_stubs_start_offset(int offset) {
159     _stubs_start_offset = offset;
160   }
161 
162   int stubs_start_offset() {
163     return _stubs_start_offset;
164   }
165 };
166 
167 static ZBarrierSetC2State* barrier_set_state() {
168   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
169 }
170 
171 void ZBarrierStubC2::register_stub(ZBarrierStubC2* stub) {
172   if (!Compile::current()->output()->in_scratch_emit_size()) {
173     barrier_set_state()->stubs()->append(stub);
174   }
175 }
176 
177 void ZBarrierStubC2::inc_trampoline_stubs_count() {
178   if (!Compile::current()->output()->in_scratch_emit_size()) {
179     barrier_set_state()->inc_trampoline_stubs_count();
180   }
181 }
182 
183 int ZBarrierStubC2::trampoline_stubs_count() {
184   return barrier_set_state()->trampoline_stubs_count();
185 }
186 
187 int ZBarrierStubC2::stubs_start_offset() {
188   return barrier_set_state()->stubs_start_offset();
189 }
190 
191 ZBarrierStubC2::ZBarrierStubC2(const MachNode* node) : BarrierStubC2(node) {}
192 
193 ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref) {
194   AARCH64_ONLY(fatal("Should use ZLoadBarrierStubC2Aarch64::create"));
195   ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref);
196   register_stub(stub);
197 
198   return stub;
199 }
200 
201 ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref)
202   : ZBarrierStubC2(node),
203     _ref_addr(ref_addr),
204     _ref(ref) {
205   assert_different_registers(ref, ref_addr.base());
206   assert_different_registers(ref, ref_addr.index());
207   // The runtime call updates the value of ref, so we should not spill and
208   // reload its outdated value.
209   dont_preserve(ref);
210 }
211 
212 Address ZLoadBarrierStubC2::ref_addr() const {
213   return _ref_addr;
214 }
215 
216 Register ZLoadBarrierStubC2::ref() const {
217   return _ref;
218 }
219 
220 address ZLoadBarrierStubC2::slow_path() const {
221   const uint8_t barrier_data = _node->barrier_data();
222   DecoratorSet decorators = DECORATORS_NONE;
223   if (barrier_data & ZBarrierStrong) {
224     decorators |= ON_STRONG_OOP_REF;
225   }
226   if (barrier_data & ZBarrierWeak) {
227     decorators |= ON_WEAK_OOP_REF;
228   }
229   if (barrier_data & ZBarrierPhantom) {
230     decorators |= ON_PHANTOM_OOP_REF;
231   }
232   if (barrier_data & ZBarrierNoKeepalive) {
233     decorators |= AS_NO_KEEPALIVE;
234   }
235   return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
236 }
237 
238 void ZLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
239   ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, static_cast<ZLoadBarrierStubC2*>(this));
240 }
241 
242 ZStoreBarrierStubC2* ZStoreBarrierStubC2::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic) {
243   AARCH64_ONLY(fatal("Should use ZStoreBarrierStubC2Aarch64::create"));
244   ZStoreBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic);
245   register_stub(stub);
246 
247   return stub;
248 }
249 
250 ZStoreBarrierStubC2::ZStoreBarrierStubC2(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic)
251   : ZBarrierStubC2(node),
252     _ref_addr(ref_addr),
253     _new_zaddress(new_zaddress),
254     _new_zpointer(new_zpointer),
255     _is_native(is_native),
256     _is_atomic(is_atomic) {}
257 
258 Address ZStoreBarrierStubC2::ref_addr() const {
259   return _ref_addr;
260 }
261 
262 Register ZStoreBarrierStubC2::new_zaddress() const {
263   return _new_zaddress;
264 }
265 
266 Register ZStoreBarrierStubC2::new_zpointer() const {
267   return _new_zpointer;
268 }
269 
270 bool ZStoreBarrierStubC2::is_native() const {
271   return _is_native;
272 }
273 
274 bool ZStoreBarrierStubC2::is_atomic() const {
275   return _is_atomic;
276 }
277 
278 void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
279   ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast<ZStoreBarrierStubC2*>(this));
280 }
281 
282 uint ZBarrierSetC2::estimated_barrier_size(const Node* node) const {
283   uint8_t barrier_data = MemNode::barrier_data(node);
284   assert(barrier_data != 0, "should be a barrier node");
285   uint uncolor_or_color_size = node->is_Load() ? 1 : 2;
286   if ((barrier_data & ZBarrierElided) != 0) {
287     return uncolor_or_color_size;
288   }
289   // A compare and branch corresponds to approximately four fast-path Ideal
290   // nodes (Cmp, Bool, If, If projection). The slow path (If projection and
291   // runtime call) is excluded since the corresponding code is laid out
292   // separately and does not directly affect performance.
293   return uncolor_or_color_size + 4;
294 }
295 
296 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
297   return new (comp_arena) ZBarrierSetC2State(comp_arena);
298 }
299 
300 void ZBarrierSetC2::late_barrier_analysis() const {
301   compute_liveness_at_stubs();
302   analyze_dominating_barriers();
303 }
304 
305 void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
306   MacroAssembler masm(&cb);
307   GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
308   barrier_set_state()->set_stubs_start_offset(masm.offset());
309 
310   for (int i = 0; i < stubs->length(); i++) {
311     // Make sure there is enough space in the code buffer
312     if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
313       ciEnv::current()->record_failure("CodeCache is full");
314       return;
315     }
316 
317     stubs->at(i)->emit_code(masm);
318   }
319 
320   masm.flush();
321 }
322 
323 int ZBarrierSetC2::estimate_stub_size() const {
324   Compile* const C = Compile::current();
325   BufferBlob* const blob = C->output()->scratch_buffer_blob();
326   GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
327   int size = 0;
328 
329   for (int i = 0; i < stubs->length(); i++) {
330     CodeBuffer cb(blob->content_begin(), (address)C->output()->scratch_locs_memory() - blob->content_begin());
331     MacroAssembler masm(&cb);
332     stubs->at(i)->emit_code(masm);
333     size += cb.insts_size();
334   }
335 
336   return size;
337 }
338 
339 static void set_barrier_data(C2Access& access) {
340   if (!ZBarrierSet::barrier_needed(access.decorators(), access.type())) {
341     return;
342   }
343 
344   if (access.decorators() & C2_TIGHTLY_COUPLED_ALLOC) {
345     access.set_barrier_data(ZBarrierElided);
346     return;
347   }
348 
349   uint8_t barrier_data = 0;
350 
351   if (access.decorators() & ON_PHANTOM_OOP_REF) {
352     barrier_data |= ZBarrierPhantom;
353   } else if (access.decorators() & ON_WEAK_OOP_REF) {
354     barrier_data |= ZBarrierWeak;
355   } else {
356     barrier_data |= ZBarrierStrong;
357   }
358 
359   if (access.decorators() & IN_NATIVE) {
360     barrier_data |= ZBarrierNative;
361   }
362 
363   if (access.decorators() & AS_NO_KEEPALIVE) {
364     barrier_data |= ZBarrierNoKeepalive;
365   }
366 
367   access.set_barrier_data(barrier_data);
368 }
369 
370 Node* ZBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
371   set_barrier_data(access);
372   return BarrierSetC2::store_at_resolved(access, val);
373 }
374 
375 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
376   set_barrier_data(access);
377   return BarrierSetC2::load_at_resolved(access, val_type);
378 }
379 
380 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
381                                                     Node* new_val, const Type* val_type) const {
382   set_barrier_data(access);
383   return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
384 }
385 
386 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
387                                                      Node* new_val, const Type* value_type) const {
388   set_barrier_data(access);
389   return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
390 }
391 
392 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
393   set_barrier_data(access);
394   return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
395 }
396 
397 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
398                                                     bool is_clone, bool is_clone_instance,
399                                                     ArrayCopyPhase phase) const {
400   if (phase == ArrayCopyPhase::Parsing) {
401     return false;
402   }
403   if (phase == ArrayCopyPhase::Optimization) {
404     return is_clone_instance;
405   }
406   // else ArrayCopyPhase::Expansion
407   return type == T_OBJECT || type == T_ARRAY;
408 }
409 
410 // This TypeFunc assumes a 64bit system
411 static const TypeFunc* clone_type() {
412   // Create input type (domain)
413   const Type** const domain_fields = TypeTuple::fields(4);
414   domain_fields[TypeFunc::Parms + 0] = TypeInstPtr::NOTNULL;  // src
415   domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL;  // dst
416   domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG;        // size lower
417   domain_fields[TypeFunc::Parms + 3] = Type::HALF;            // size upper
418   const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
419 
420   // Create result type (range)
421   const Type** const range_fields = TypeTuple::fields(0);
422   const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
423 
424   return TypeFunc::make(domain, range);
425 }
426 
427 #define XTOP LP64_ONLY(COMMA phase->top())
428 
429 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
430   Node* const src = ac->in(ArrayCopyNode::Src);
431   const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
432 
433   if (ac->is_clone_array() && ary_ptr != nullptr) {
434     BasicType bt = ary_ptr->elem()->array_element_basic_type();
435     if (is_reference_type(bt)) {
436       // Clone object array
437       bt = T_OBJECT;
438     } else {
439       // Clone primitive array
440       bt = T_LONG;
441     }
442 
443     Node* const ctrl = ac->in(TypeFunc::Control);
444     Node* const mem = ac->in(TypeFunc::Memory);
445     Node* const src = ac->in(ArrayCopyNode::Src);
446     Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
447     Node* const dest = ac->in(ArrayCopyNode::Dest);
448     Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
449     Node* length = ac->in(ArrayCopyNode::Length);
450 
451     if (bt == T_OBJECT) {
452       // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
453       // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
454       // to the first element in the array when cloning object arrays. Otherwise, load
455       // barriers are applied to parts of the header. Also adjust the length accordingly.
456       assert(src_offset == dest_offset, "should be equal");
457       const jlong offset = src_offset->get_long();
458       if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
459         assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
460         assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
461         length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs
462         src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
463         dest_offset = src_offset;
464       }
465     }
466     Node* const payload_src = phase->basic_plus_adr(src, src_offset);
467     Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
468 
469     const char*   copyfunc_name = "arraycopy";
470     const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
471 
472     const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
473     const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
474 
475     Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
476     phase->transform_later(call);
477 
478     phase->igvn().replace_node(ac, call);
479     return;
480   }
481 
482   // Clone instance
483   Node* const ctrl       = ac->in(TypeFunc::Control);
484   Node* const mem        = ac->in(TypeFunc::Memory);
485   Node* const dst        = ac->in(ArrayCopyNode::Dest);
486   Node* const size       = ac->in(ArrayCopyNode::Length);
487 
488   assert(size->bottom_type()->is_long(), "Should be long");
489 
490   // The native clone we are calling here expects the instance size in words
491   // Add header/offset size to payload size to get instance size.
492   Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
493   Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
494 
495   Node* const call = phase->make_leaf_call(ctrl,
496                                            mem,
497                                            clone_type(),
498                                            ZBarrierSetRuntime::clone_addr(),
499                                            "ZBarrierSetRuntime::clone",
500                                            TypeRawPtr::BOTTOM,
501                                            src,
502                                            dst,
503                                            full_size,
504                                            phase->top());
505   phase->transform_later(call);
506   phase->igvn().replace_node(ac, call);
507 }
508 
509 #undef XTOP
510 
511 // == Dominating barrier elision ==
512 
513 static bool block_has_safepoint(const Block* block, uint from, uint to) {
514   for (uint i = from; i < to; i++) {
515     if (block->get_node(i)->is_MachSafePoint()) {
516       // Safepoint found
517       return true;
518     }
519   }
520 
521   // Safepoint not found
522   return false;
523 }
524 
525 static bool block_has_safepoint(const Block* block) {
526   return block_has_safepoint(block, 0, block->number_of_nodes());
527 }
528 
529 static uint block_index(const Block* block, const Node* node) {
530   for (uint j = 0; j < block->number_of_nodes(); ++j) {
531     if (block->get_node(j) == node) {
532       return j;
533     }
534   }
535   ShouldNotReachHere();
536   return 0;
537 }
538 
539 // Look through various node aliases
540 static const Node* look_through_node(const Node* node) {
541   while (node != nullptr) {
542     const Node* new_node = node;
543     if (node->is_Mach()) {
544       const MachNode* const node_mach = node->as_Mach();
545       if (node_mach->ideal_Opcode() == Op_CheckCastPP) {
546         new_node = node->in(1);
547       }
548       if (node_mach->is_SpillCopy()) {
549         new_node = node->in(1);
550       }
551     }
552     if (new_node == node || new_node == nullptr) {
553       break;
554     } else {
555       node = new_node;
556     }
557   }
558 
559   return node;
560 }
561 
562 // Whether the given offset is undefined.
563 static bool is_undefined(intptr_t offset) {
564   return offset == Type::OffsetTop;
565 }
566 
567 // Whether the given offset is unknown.
568 static bool is_unknown(intptr_t offset) {
569   return offset == Type::OffsetBot;
570 }
571 
572 // Whether the given offset is concrete (defined and compile-time known).
573 static bool is_concrete(intptr_t offset) {
574   return !is_undefined(offset) && !is_unknown(offset);
575 }
576 
577 // Compute base + offset components of the memory address accessed by mach.
578 // Return a node representing the base address, or null if the base cannot be
579 // found or the offset is undefined or a concrete negative value. If a non-null
580 // base is returned, the offset is a concrete, nonnegative value or unknown.
581 static const Node* get_base_and_offset(const MachNode* mach, intptr_t& offset) {
582   const TypePtr* adr_type = nullptr;
583   offset = 0;
584   const Node* base = mach->get_base_and_disp(offset, adr_type);
585 
586   if (base == nullptr || base == NodeSentinel) {
587     return nullptr;
588   }
589 
590   if (offset == 0 && base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_AddP) {
591     // The memory address is computed by 'base' and fed to 'mach' via an
592     // indirect memory operand (indicated by offset == 0). The ultimate base and
593     // offset can be fetched directly from the inputs and Ideal type of 'base'.
594     offset = base->bottom_type()->isa_oopptr()->offset();
595     // Even if 'base' is not an Ideal AddP node anymore, Matcher::ReduceInst()
596     // guarantees that the base address is still available at the same slot.
597     base = base->in(AddPNode::Base);
598     assert(base != nullptr, "");
599   }
600 
601   if (is_undefined(offset) || (is_concrete(offset) && offset < 0)) {
602     return nullptr;
603   }
604 
605   return look_through_node(base);
606 }
607 
608 // Whether a phi node corresponds to an array allocation.
609 // This test is incomplete: in some edge cases, it might return false even
610 // though the node does correspond to an array allocation.
611 static bool is_array_allocation(const Node* phi) {
612   precond(phi->is_Phi());
613   // Check whether phi has a successor cast (CheckCastPP) to Java array pointer,
614   // possibly below spill copies and other cast nodes. Limit the exploration to
615   // a single path from the phi node consisting of these node types.
616   const Node* current = phi;
617   while (true) {
618     const Node* next = nullptr;
619     for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) {
620       if (!current->fast_out(i)->isa_Mach()) {
621         continue;
622       }
623       const MachNode* succ = current->fast_out(i)->as_Mach();
624       if (succ->ideal_Opcode() == Op_CheckCastPP) {
625         if (succ->get_ptr_type()->isa_aryptr()) {
626           // Cast to Java array pointer: phi corresponds to an array allocation.
627           return true;
628         }
629         // Other cast: record as candidate for further exploration.
630         next = succ;
631       } else if (succ->is_SpillCopy() && next == nullptr) {
632         // Spill copy, and no better candidate found: record as candidate.
633         next = succ;
634       }
635     }
636     if (next == nullptr) {
637       // No evidence found that phi corresponds to an array allocation, and no
638       // candidates available to continue exploring.
639       return false;
640     }
641     // Continue exploring from the best candidate found.
642     current = next;
643   }
644   ShouldNotReachHere();
645 }
646 
647 // Match the phi node that connects a TLAB allocation fast path with its slowpath
648 static bool is_allocation(const Node* node) {
649   if (node->req() != 3) {
650     return false;
651   }
652   const Node* const fast_node = node->in(2);
653   if (!fast_node->is_Mach()) {
654     return false;
655   }
656   const MachNode* const fast_mach = fast_node->as_Mach();
657   if (fast_mach->ideal_Opcode() != Op_LoadP) {
658     return false;
659   }
660   const TypePtr* const adr_type = nullptr;
661   intptr_t offset;
662   const Node* const base = get_base_and_offset(fast_mach, offset);
663   if (base == nullptr || !base->is_Mach() || !is_concrete(offset)) {
664     return false;
665   }
666   const MachNode* const base_mach = base->as_Mach();
667   if (base_mach->ideal_Opcode() != Op_ThreadLocal) {
668     return false;
669   }
670   return offset == in_bytes(Thread::tlab_top_offset());
671 }
672 
673 static void elide_mach_barrier(MachNode* mach) {
674   mach->set_barrier_data(ZBarrierElided);
675 }
676 
677 void ZBarrierSetC2::analyze_dominating_barriers_impl(Node_List& accesses, Node_List& access_dominators) const {
678   Compile* const C = Compile::current();
679   PhaseCFG* const cfg = C->cfg();
680 
681   for (uint i = 0; i < accesses.size(); i++) {
682     MachNode* const access = accesses.at(i)->as_Mach();
683     intptr_t access_offset;
684     const Node* const access_obj = get_base_and_offset(access, access_offset);
685     Block* const access_block = cfg->get_block_for_node(access);
686     const uint access_index = block_index(access_block, access);
687 
688     if (access_obj == nullptr) {
689       // No information available
690       continue;
691     }
692 
693     for (uint j = 0; j < access_dominators.size(); j++) {
694      const  Node* const mem = access_dominators.at(j);
695       if (mem->is_Phi()) {
696         // Allocation node
697         if (mem != access_obj) {
698           continue;
699         }
700         if (is_unknown(access_offset) && !is_array_allocation(mem)) {
701           // The accessed address has an unknown offset, but the allocated
702           // object cannot be determined to be an array. Avoid eliding in this
703           // case, to be on the safe side.
704           continue;
705         }
706         assert((is_concrete(access_offset) && access_offset >= 0) || (is_unknown(access_offset) && is_array_allocation(mem)),
707                "candidate allocation-dominated access offsets must be either concrete and nonnegative, or unknown (for array allocations only)");
708       } else {
709         // Access node
710         const MachNode* const mem_mach = mem->as_Mach();
711         intptr_t mem_offset;
712         const Node* const mem_obj = get_base_and_offset(mem_mach, mem_offset);
713 
714         if (mem_obj == nullptr ||
715             !is_concrete(access_offset) ||
716             !is_concrete(mem_offset)) {
717           // No information available
718           continue;
719         }
720 
721         if (mem_obj != access_obj || mem_offset != access_offset) {
722           // Not the same addresses, not a candidate
723           continue;
724         }
725         assert(is_concrete(access_offset) && access_offset >= 0,
726                "candidate non-allocation-dominated access offsets must be concrete and nonnegative");
727       }
728 
729       Block* mem_block = cfg->get_block_for_node(mem);
730       const uint mem_index = block_index(mem_block, mem);
731 
732       if (access_block == mem_block) {
733         // Earlier accesses in the same block
734         if (mem_index < access_index && !block_has_safepoint(mem_block, mem_index + 1, access_index)) {
735           elide_mach_barrier(access);
736         }
737       } else if (mem_block->dominates(access_block)) {
738         // Dominating block? Look around for safepoints
739         ResourceMark rm;
740         Block_List stack;
741         VectorSet visited;
742         stack.push(access_block);
743         bool safepoint_found = block_has_safepoint(access_block);
744         while (!safepoint_found && stack.size() > 0) {
745           const Block* const block = stack.pop();
746           if (visited.test_set(block->_pre_order)) {
747             continue;
748           }
749           if (block_has_safepoint(block)) {
750             safepoint_found = true;
751             break;
752           }
753           if (block == mem_block) {
754             continue;
755           }
756 
757           // Push predecessor blocks
758           for (uint p = 1; p < block->num_preds(); ++p) {
759             Block* const pred = cfg->get_block_for_node(block->pred(p));
760             stack.push(pred);
761           }
762         }
763 
764         if (!safepoint_found) {
765           elide_mach_barrier(access);
766         }
767       }
768     }
769   }
770 }
771 
772 void ZBarrierSetC2::analyze_dominating_barriers() const {
773   ResourceMark rm;
774   Compile* const C = Compile::current();
775   PhaseCFG* const cfg = C->cfg();
776 
777   Node_List loads;
778   Node_List load_dominators;
779 
780   Node_List stores;
781   Node_List store_dominators;
782 
783   Node_List atomics;
784   Node_List atomic_dominators;
785 
786   // Step 1 - Find accesses and allocations, and track them in lists
787   for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
788     const Block* const block = cfg->get_block(i);
789     for (uint j = 0; j < block->number_of_nodes(); ++j) {
790       Node* const node = block->get_node(j);
791       if (node->is_Phi()) {
792         if (is_allocation(node)) {
793           load_dominators.push(node);
794           store_dominators.push(node);
795           // An allocation can't be considered to "dominate" an atomic operation.
796           // For example a CAS requires the memory location to be store-good.
797           // When you have a dominating store or atomic instruction, that is
798           // indeed ensured to be the case. However, as for allocations, the
799           // initialized memory location could be raw null, which isn't store-good.
800         }
801         continue;
802       } else if (!node->is_Mach()) {
803         continue;
804       }
805 
806       MachNode* const mach = node->as_Mach();
807       switch (mach->ideal_Opcode()) {
808       case Op_LoadP:
809         if ((mach->barrier_data() & ZBarrierStrong) != 0 &&
810             (mach->barrier_data() & ZBarrierNoKeepalive) == 0) {
811           loads.push(mach);
812           load_dominators.push(mach);
813         }
814         break;
815       case Op_StoreP:
816         if (mach->barrier_data() != 0) {
817           stores.push(mach);
818           load_dominators.push(mach);
819           store_dominators.push(mach);
820           atomic_dominators.push(mach);
821         }
822         break;
823       case Op_CompareAndExchangeP:
824       case Op_CompareAndSwapP:
825       case Op_GetAndSetP:
826         if (mach->barrier_data() != 0) {
827           atomics.push(mach);
828           load_dominators.push(mach);
829           store_dominators.push(mach);
830           atomic_dominators.push(mach);
831         }
832         break;
833 
834       default:
835         break;
836       }
837     }
838   }
839 
840   // Step 2 - Find dominating accesses or allocations for each access
841   analyze_dominating_barriers_impl(loads, load_dominators);
842   analyze_dominating_barriers_impl(stores, store_dominators);
843   analyze_dominating_barriers_impl(atomics, atomic_dominators);
844 }
845 
846 void ZBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
847   eliminate_gc_barrier_data(node);
848 }
849 
850 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
851   if (node->is_LoadStore()) {
852     LoadStoreNode* loadstore = node->as_LoadStore();
853     loadstore->set_barrier_data(ZBarrierElided);
854   } else if (node->is_Mem()) {
855     MemNode* mem = node->as_Mem();
856     mem->set_barrier_data(ZBarrierElided);
857   }
858 }
859 
860 #ifndef PRODUCT
861 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
862   if ((mach->barrier_data() & ZBarrierStrong) != 0) {
863     st->print("strong ");
864   }
865   if ((mach->barrier_data() & ZBarrierWeak) != 0) {
866     st->print("weak ");
867   }
868   if ((mach->barrier_data() & ZBarrierPhantom) != 0) {
869     st->print("phantom ");
870   }
871   if ((mach->barrier_data() & ZBarrierNoKeepalive) != 0) {
872     st->print("nokeepalive ");
873   }
874   if ((mach->barrier_data() & ZBarrierNative) != 0) {
875     st->print("native ");
876   }
877   if ((mach->barrier_data() & ZBarrierElided) != 0) {
878     st->print("elided ");
879   }
880 }
881 #endif // !PRODUCT