1 /*
  2  * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #include "precompiled.hpp"
 25 #include "asm/macroAssembler.hpp"
 26 #include "classfile/javaClasses.hpp"
 27 #include "gc/z/c2/zBarrierSetC2.hpp"
 28 #include "gc/z/zBarrierSet.hpp"
 29 #include "gc/z/zBarrierSetAssembler.hpp"
 30 #include "gc/z/zBarrierSetRuntime.hpp"
 31 #include "opto/arraycopynode.hpp"
 32 #include "opto/addnode.hpp"
 33 #include "opto/block.hpp"
 34 #include "opto/compile.hpp"
 35 #include "opto/graphKit.hpp"
 36 #include "opto/machnode.hpp"
 37 #include "opto/macro.hpp"
 38 #include "opto/memnode.hpp"
 39 #include "opto/node.hpp"
 40 #include "opto/output.hpp"
 41 #include "opto/regalloc.hpp"
 42 #include "opto/rootnode.hpp"
 43 #include "opto/runtime.hpp"
 44 #include "opto/type.hpp"
 45 #include "utilities/debug.hpp"
 46 #include "utilities/growableArray.hpp"
 47 #include "utilities/macros.hpp"
 48 
 49 template<typename K, typename V, size_t _table_size>
 50 class ZArenaHashtable : public ResourceObj {
 51   class ZArenaHashtableEntry : public ResourceObj {
 52   public:
 53     ZArenaHashtableEntry* _next;
 54     K _key;
 55     V _value;
 56   };
 57 
 58   static const size_t _table_mask = _table_size - 1;
 59 
 60   Arena* _arena;
 61   ZArenaHashtableEntry* _table[_table_size];
 62 
 63 public:
 64   class Iterator {
 65     ZArenaHashtable* _table;
 66     ZArenaHashtableEntry* _current_entry;
 67     size_t _current_index;
 68 
 69   public:
 70     Iterator(ZArenaHashtable* table)
 71       : _table(table),
 72         _current_entry(table->_table[0]),
 73         _current_index(0) {
 74       if (_current_entry == nullptr) {
 75         next();
 76       }
 77     }
 78 
 79     bool has_next() { return _current_entry != nullptr; }
 80     K key()         { return _current_entry->_key; }
 81     V value()       { return _current_entry->_value; }
 82 
 83     void next() {
 84       if (_current_entry != nullptr) {
 85         _current_entry = _current_entry->_next;
 86       }
 87       while (_current_entry == nullptr && ++_current_index < _table_size) {
 88         _current_entry = _table->_table[_current_index];
 89       }
 90     }
 91   };
 92 
 93   ZArenaHashtable(Arena* arena)
 94     : _arena(arena),
 95       _table() {
 96     Copy::zero_to_bytes(&_table, sizeof(_table));
 97   }
 98 
 99   void add(K key, V value) {
100     ZArenaHashtableEntry* entry = new (_arena) ZArenaHashtableEntry();
101     entry->_key = key;
102     entry->_value = value;
103     entry->_next = _table[key & _table_mask];
104     _table[key & _table_mask] = entry;
105   }
106 
107   V* get(K key) const {
108     for (ZArenaHashtableEntry* e = _table[key & _table_mask]; e != nullptr; e = e->_next) {
109       if (e->_key == key) {
110         return &(e->_value);
111       }
112     }
113     return nullptr;
114   }
115 
116   Iterator iterator() {
117     return Iterator(this);
118   }
119 };
120 
121 typedef ZArenaHashtable<intptr_t, bool, 4> ZOffsetTable;
122 
123 class ZBarrierSetC2State : public ArenaObj {
124 private:
125   GrowableArray<ZBarrierStubC2*>* _stubs;
126   Node_Array                      _live;
127   int                             _trampoline_stubs_count;
128   int                             _stubs_start_offset;
129 
130 public:
131   ZBarrierSetC2State(Arena* arena)
132     : _stubs(new (arena) GrowableArray<ZBarrierStubC2*>(arena, 8,  0, nullptr)),
133       _live(arena),
134       _trampoline_stubs_count(0),
135       _stubs_start_offset(0) {}
136 
137   GrowableArray<ZBarrierStubC2*>* stubs() {
138     return _stubs;
139   }
140 
141   RegMask* live(const Node* node) {
142     if (!node->is_Mach()) {
143       // Don't need liveness for non-MachNodes
144       return nullptr;
145     }
146 
147     const MachNode* const mach = node->as_Mach();
148     if (mach->barrier_data() == ZBarrierElided) {
149       // Don't need liveness data for nodes without barriers
150       return nullptr;
151     }
152 
153     RegMask* live = (RegMask*)_live[node->_idx];
154     if (live == nullptr) {
155       live = new (Compile::current()->comp_arena()->AmallocWords(sizeof(RegMask))) RegMask();
156       _live.map(node->_idx, (Node*)live);
157     }
158 
159     return live;
160   }
161 
162   void inc_trampoline_stubs_count() {
163     assert(_trampoline_stubs_count != INT_MAX, "Overflow");
164     ++_trampoline_stubs_count;
165   }
166 
167   int trampoline_stubs_count() {
168     return _trampoline_stubs_count;
169   }
170 
171   void set_stubs_start_offset(int offset) {
172     _stubs_start_offset = offset;
173   }
174 
175   int stubs_start_offset() {
176     return _stubs_start_offset;
177   }
178 };
179 
180 static ZBarrierSetC2State* barrier_set_state() {
181   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
182 }
183 
184 void ZBarrierStubC2::register_stub(ZBarrierStubC2* stub) {
185   if (!Compile::current()->output()->in_scratch_emit_size()) {
186     barrier_set_state()->stubs()->append(stub);
187   }
188 }
189 
190 void ZBarrierStubC2::inc_trampoline_stubs_count() {
191   if (!Compile::current()->output()->in_scratch_emit_size()) {
192     barrier_set_state()->inc_trampoline_stubs_count();
193   }
194 }
195 
196 int ZBarrierStubC2::trampoline_stubs_count() {
197   return barrier_set_state()->trampoline_stubs_count();
198 }
199 
200 int ZBarrierStubC2::stubs_start_offset() {
201   return barrier_set_state()->stubs_start_offset();
202 }
203 
204 ZBarrierStubC2::ZBarrierStubC2(const MachNode* node)
205   : _node(node),
206     _entry(),
207     _continuation() {}
208 
209 Register ZBarrierStubC2::result() const {
210   return noreg;
211 }
212 
213 RegMask& ZBarrierStubC2::live() const {
214   return *barrier_set_state()->live(_node);
215 }
216 
217 Label* ZBarrierStubC2::entry() {
218   // The _entry will never be bound when in_scratch_emit_size() is true.
219   // However, we still need to return a label that is not bound now, but
220   // will eventually be bound. Any eventually bound label will do, as it
221   // will only act as a placeholder, so we return the _continuation label.
222   return Compile::current()->output()->in_scratch_emit_size() ? &_continuation : &_entry;
223 }
224 
225 Label* ZBarrierStubC2::continuation() {
226   return &_continuation;
227 }
228 
229 ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref) {
230   AARCH64_ONLY(fatal("Should use ZLoadBarrierStubC2Aarch64::create"));
231   ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref);
232   register_stub(stub);
233 
234   return stub;
235 }
236 
237 ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref)
238   : ZBarrierStubC2(node),
239     _ref_addr(ref_addr),
240     _ref(ref) {
241   assert_different_registers(ref, ref_addr.base());
242   assert_different_registers(ref, ref_addr.index());
243 }
244 
245 Address ZLoadBarrierStubC2::ref_addr() const {
246   return _ref_addr;
247 }
248 
249 Register ZLoadBarrierStubC2::ref() const {
250   return _ref;
251 }
252 
253 Register ZLoadBarrierStubC2::result() const {
254   return ref();
255 }
256 
257 address ZLoadBarrierStubC2::slow_path() const {
258   const uint8_t barrier_data = _node->barrier_data();
259   DecoratorSet decorators = DECORATORS_NONE;
260   if (barrier_data & ZBarrierStrong) {
261     decorators |= ON_STRONG_OOP_REF;
262   }
263   if (barrier_data & ZBarrierWeak) {
264     decorators |= ON_WEAK_OOP_REF;
265   }
266   if (barrier_data & ZBarrierPhantom) {
267     decorators |= ON_PHANTOM_OOP_REF;
268   }
269   if (barrier_data & ZBarrierNoKeepalive) {
270     decorators |= AS_NO_KEEPALIVE;
271   }
272   return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
273 }
274 
275 void ZLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
276   ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, static_cast<ZLoadBarrierStubC2*>(this));
277 }
278 
279 ZStoreBarrierStubC2* ZStoreBarrierStubC2::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic) {
280   AARCH64_ONLY(fatal("Should use ZStoreBarrierStubC2Aarch64::create"));
281   ZStoreBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic);
282   register_stub(stub);
283 
284   return stub;
285 }
286 
287 ZStoreBarrierStubC2::ZStoreBarrierStubC2(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic)
288   : ZBarrierStubC2(node),
289     _ref_addr(ref_addr),
290     _new_zaddress(new_zaddress),
291     _new_zpointer(new_zpointer),
292     _is_native(is_native),
293     _is_atomic(is_atomic) {}
294 
295 Address ZStoreBarrierStubC2::ref_addr() const {
296   return _ref_addr;
297 }
298 
299 Register ZStoreBarrierStubC2::new_zaddress() const {
300   return _new_zaddress;
301 }
302 
303 Register ZStoreBarrierStubC2::new_zpointer() const {
304   return _new_zpointer;
305 }
306 
307 bool ZStoreBarrierStubC2::is_native() const {
308   return _is_native;
309 }
310 
311 bool ZStoreBarrierStubC2::is_atomic() const {
312   return _is_atomic;
313 }
314 
315 Register ZStoreBarrierStubC2::result() const {
316   return noreg;
317 }
318 
319 void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
320   ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast<ZStoreBarrierStubC2*>(this));
321 }
322 
323 uint ZBarrierSetC2::estimated_barrier_size(const Node* node) const {
324   uint8_t barrier_data = MemNode::barrier_data(node);
325   assert(barrier_data != 0, "should be a barrier node");
326   uint uncolor_or_color_size = node->is_Load() ? 1 : 2;
327   if ((barrier_data & ZBarrierElided) != 0) {
328     return uncolor_or_color_size;
329   }
330   // A compare and branch corresponds to approximately four fast-path Ideal
331   // nodes (Cmp, Bool, If, If projection). The slow path (If projection and
332   // runtime call) is excluded since the corresponding code is laid out
333   // separately and does not directly affect performance.
334   return uncolor_or_color_size + 4;
335 }
336 
337 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
338   return new (comp_arena) ZBarrierSetC2State(comp_arena);
339 }
340 
341 void ZBarrierSetC2::late_barrier_analysis() const {
342   compute_liveness_at_stubs();
343   analyze_dominating_barriers();
344 }
345 
346 void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
347   MacroAssembler masm(&cb);
348   GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
349   barrier_set_state()->set_stubs_start_offset(masm.offset());
350 
351   for (int i = 0; i < stubs->length(); i++) {
352     // Make sure there is enough space in the code buffer
353     if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
354       ciEnv::current()->record_failure("CodeCache is full");
355       return;
356     }
357 
358     stubs->at(i)->emit_code(masm);
359   }
360 
361   masm.flush();
362 }
363 
364 int ZBarrierSetC2::estimate_stub_size() const {
365   Compile* const C = Compile::current();
366   BufferBlob* const blob = C->output()->scratch_buffer_blob();
367   GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
368   int size = 0;
369 
370   for (int i = 0; i < stubs->length(); i++) {
371     CodeBuffer cb(blob->content_begin(), (address)C->output()->scratch_locs_memory() - blob->content_begin());
372     MacroAssembler masm(&cb);
373     stubs->at(i)->emit_code(masm);
374     size += cb.insts_size();
375   }
376 
377   return size;
378 }
379 
380 static void set_barrier_data(C2Access& access) {
381   if (!ZBarrierSet::barrier_needed(access.decorators(), access.type())) {
382     return;
383   }
384 
385   if (access.decorators() & C2_TIGHTLY_COUPLED_ALLOC) {
386     access.set_barrier_data(ZBarrierElided);
387     return;
388   }
389 
390   uint8_t barrier_data = 0;
391 
392   if (access.decorators() & ON_PHANTOM_OOP_REF) {
393     barrier_data |= ZBarrierPhantom;
394   } else if (access.decorators() & ON_WEAK_OOP_REF) {
395     barrier_data |= ZBarrierWeak;
396   } else {
397     barrier_data |= ZBarrierStrong;
398   }
399 
400   if (access.decorators() & IN_NATIVE) {
401     barrier_data |= ZBarrierNative;
402   }
403 
404   if (access.decorators() & AS_NO_KEEPALIVE) {
405     barrier_data |= ZBarrierNoKeepalive;
406   }
407 
408   access.set_barrier_data(barrier_data);
409 }
410 
411 Node* ZBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
412   set_barrier_data(access);
413   return BarrierSetC2::store_at_resolved(access, val);
414 }
415 
416 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
417   set_barrier_data(access);
418   return BarrierSetC2::load_at_resolved(access, val_type);
419 }
420 
421 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
422                                                     Node* new_val, const Type* val_type) const {
423   set_barrier_data(access);
424   return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
425 }
426 
427 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
428                                                      Node* new_val, const Type* value_type) const {
429   set_barrier_data(access);
430   return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
431 }
432 
433 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
434   set_barrier_data(access);
435   return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
436 }
437 
438 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
439                                                     bool is_clone, bool is_clone_instance,
440                                                     ArrayCopyPhase phase) const {
441   if (phase == ArrayCopyPhase::Parsing) {
442     return false;
443   }
444   if (phase == ArrayCopyPhase::Optimization) {
445     return is_clone_instance;
446   }
447   // else ArrayCopyPhase::Expansion
448   return type == T_OBJECT || type == T_ARRAY;
449 }
450 
451 // This TypeFunc assumes a 64bit system
452 static const TypeFunc* clone_type() {
453   // Create input type (domain)
454   const Type** const domain_fields = TypeTuple::fields(4);
455   domain_fields[TypeFunc::Parms + 0] = TypeInstPtr::NOTNULL;  // src
456   domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL;  // dst
457   domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG;        // size lower
458   domain_fields[TypeFunc::Parms + 3] = Type::HALF;            // size upper
459   const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
460 
461   // Create result type (range)
462   const Type** const range_fields = TypeTuple::fields(0);
463   const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
464 
465   return TypeFunc::make(domain, range);
466 }
467 
468 #define XTOP LP64_ONLY(COMMA phase->top())
469 
470 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
471   Node* const src = ac->in(ArrayCopyNode::Src);
472   const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
473 
474   if (ac->is_clone_array() && ary_ptr != nullptr) {
475     BasicType bt = ary_ptr->elem()->array_element_basic_type();
476     if (is_reference_type(bt) && !ary_ptr->is_flat()) {
477       // Clone object array
478       bt = T_OBJECT;
479     } else {
480       // Clone primitive array
481       bt = T_LONG;
482     }
483 
484     Node* const ctrl = ac->in(TypeFunc::Control);
485     Node* const mem = ac->in(TypeFunc::Memory);
486     Node* const src = ac->in(ArrayCopyNode::Src);
487     Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
488     Node* const dest = ac->in(ArrayCopyNode::Dest);
489     Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
490     Node* length = ac->in(ArrayCopyNode::Length);
491 
492     if (bt == T_OBJECT) {
493       // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
494       // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
495       // to the first element in the array when cloning object arrays. Otherwise, load
496       // barriers are applied to parts of the header. Also adjust the length accordingly.
497       assert(src_offset == dest_offset, "should be equal");
498       const jlong offset = src_offset->get_long();
499       if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
500         assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
501         assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
502         length = phase->transform_later(new SubXNode(length, phase->longcon(1))); // Size is in longs
503         src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
504         dest_offset = src_offset;
505       }
506     }
507     Node* const payload_src = phase->basic_plus_adr(src, src_offset);
508     Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
509 
510     const char*   copyfunc_name = "arraycopy";
511     const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
512 
513     const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
514     const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
515 
516     Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
517     phase->transform_later(call);
518 
519     phase->igvn().replace_node(ac, call);
520     return;
521   }
522 
523   // Clone instance
524   Node* const ctrl       = ac->in(TypeFunc::Control);
525   Node* const mem        = ac->in(TypeFunc::Memory);
526   Node* const dst        = ac->in(ArrayCopyNode::Dest);
527   Node* const size       = ac->in(ArrayCopyNode::Length);
528 
529   assert(size->bottom_type()->is_long(), "Should be long");
530 
531   // The native clone we are calling here expects the instance size in words
532   // Add header/offset size to payload size to get instance size.
533   Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
534   Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
535 
536   Node* const call = phase->make_leaf_call(ctrl,
537                                            mem,
538                                            clone_type(),
539                                            ZBarrierSetRuntime::clone_addr(),
540                                            "ZBarrierSetRuntime::clone",
541                                            TypeRawPtr::BOTTOM,
542                                            src,
543                                            dst,
544                                            full_size,
545                                            phase->top());
546   phase->transform_later(call);
547   phase->replace_node(ac, call);
548 }
549 
550 #undef XTOP
551 
552 // == Dominating barrier elision ==
553 
554 static bool block_has_safepoint(const Block* block, uint from, uint to) {
555   for (uint i = from; i < to; i++) {
556     if (block->get_node(i)->is_MachSafePoint()) {
557       // Safepoint found
558       return true;
559     }
560   }
561 
562   // Safepoint not found
563   return false;
564 }
565 
566 static bool block_has_safepoint(const Block* block) {
567   return block_has_safepoint(block, 0, block->number_of_nodes());
568 }
569 
570 static uint block_index(const Block* block, const Node* node) {
571   for (uint j = 0; j < block->number_of_nodes(); ++j) {
572     if (block->get_node(j) == node) {
573       return j;
574     }
575   }
576   ShouldNotReachHere();
577   return 0;
578 }
579 
580 // Look through various node aliases
581 static const Node* look_through_node(const Node* node) {
582   while (node != nullptr) {
583     const Node* new_node = node;
584     if (node->is_Mach()) {
585       const MachNode* const node_mach = node->as_Mach();
586       if (node_mach->ideal_Opcode() == Op_CheckCastPP) {
587         new_node = node->in(1);
588       }
589       if (node_mach->is_SpillCopy()) {
590         new_node = node->in(1);
591       }
592     }
593     if (new_node == node || new_node == nullptr) {
594       break;
595     } else {
596       node = new_node;
597     }
598   }
599 
600   return node;
601 }
602 
603 // Whether the given offset is undefined.
604 static bool is_undefined(intptr_t offset) {
605   return offset == Type::OffsetTop;
606 }
607 
608 // Whether the given offset is unknown.
609 static bool is_unknown(intptr_t offset) {
610   return offset == Type::OffsetBot;
611 }
612 
613 // Whether the given offset is concrete (defined and compile-time known).
614 static bool is_concrete(intptr_t offset) {
615   return !is_undefined(offset) && !is_unknown(offset);
616 }
617 
618 // Compute base + offset components of the memory address accessed by mach.
619 // Return a node representing the base address, or null if the base cannot be
620 // found or the offset is undefined or a concrete negative value. If a non-null
621 // base is returned, the offset is a concrete, nonnegative value or unknown.
622 static const Node* get_base_and_offset(const MachNode* mach, intptr_t& offset) {
623   const TypePtr* adr_type = nullptr;
624   offset = 0;
625   const Node* base = mach->get_base_and_disp(offset, adr_type);
626 
627   if (base == nullptr || base == NodeSentinel) {
628     return nullptr;
629   }
630 
631   if (offset == 0 && base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_AddP) {
632     // The memory address is computed by 'base' and fed to 'mach' via an
633     // indirect memory operand (indicated by offset == 0). The ultimate base and
634     // offset can be fetched directly from the inputs and Ideal type of 'base'.
635     offset = base->bottom_type()->isa_oopptr()->offset();
636     // Even if 'base' is not an Ideal AddP node anymore, Matcher::ReduceInst()
637     // guarantees that the base address is still available at the same slot.
638     base = base->in(AddPNode::Base);
639     assert(base != nullptr, "");
640   }
641 
642   if (is_undefined(offset) || (is_concrete(offset) && offset < 0)) {
643     return nullptr;
644   }
645 
646   return look_through_node(base);
647 }
648 
649 // Whether a phi node corresponds to an array allocation.
650 // This test is incomplete: in some edge cases, it might return false even
651 // though the node does correspond to an array allocation.
652 static bool is_array_allocation(const Node* phi) {
653   precond(phi->is_Phi());
654   // Check whether phi has a successor cast (CheckCastPP) to Java array pointer,
655   // possibly below spill copies and other cast nodes. Limit the exploration to
656   // a single path from the phi node consisting of these node types.
657   const Node* current = phi;
658   while (true) {
659     const Node* next = nullptr;
660     for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) {
661       if (!current->fast_out(i)->isa_Mach()) {
662         continue;
663       }
664       const MachNode* succ = current->fast_out(i)->as_Mach();
665       if (succ->ideal_Opcode() == Op_CheckCastPP) {
666         if (succ->get_ptr_type()->isa_aryptr()) {
667           // Cast to Java array pointer: phi corresponds to an array allocation.
668           return true;
669         }
670         // Other cast: record as candidate for further exploration.
671         next = succ;
672       } else if (succ->is_SpillCopy() && next == nullptr) {
673         // Spill copy, and no better candidate found: record as candidate.
674         next = succ;
675       }
676     }
677     if (next == nullptr) {
678       // No evidence found that phi corresponds to an array allocation, and no
679       // candidates available to continue exploring.
680       return false;
681     }
682     // Continue exploring from the best candidate found.
683     current = next;
684   }
685   ShouldNotReachHere();
686 }
687 
688 // Match the phi node that connects a TLAB allocation fast path with its slowpath
689 static bool is_allocation(const Node* node) {
690   if (node->req() != 3) {
691     return false;
692   }
693   const Node* const fast_node = node->in(2);
694   if (!fast_node->is_Mach()) {
695     return false;
696   }
697   const MachNode* const fast_mach = fast_node->as_Mach();
698   if (fast_mach->ideal_Opcode() != Op_LoadP) {
699     return false;
700   }
701   const TypePtr* const adr_type = nullptr;
702   intptr_t offset;
703   const Node* const base = get_base_and_offset(fast_mach, offset);
704   if (base == nullptr || !base->is_Mach() || !is_concrete(offset)) {
705     return false;
706   }
707   const MachNode* const base_mach = base->as_Mach();
708   if (base_mach->ideal_Opcode() != Op_ThreadLocal) {
709     return false;
710   }
711   return offset == in_bytes(Thread::tlab_top_offset());
712 }
713 
714 static void elide_mach_barrier(MachNode* mach) {
715   mach->set_barrier_data(ZBarrierElided);
716 }
717 
718 void ZBarrierSetC2::analyze_dominating_barriers_impl(Node_List& accesses, Node_List& access_dominators) const {
719   Compile* const C = Compile::current();
720   PhaseCFG* const cfg = C->cfg();
721 
722   for (uint i = 0; i < accesses.size(); i++) {
723     MachNode* const access = accesses.at(i)->as_Mach();
724     intptr_t access_offset;
725     const Node* const access_obj = get_base_and_offset(access, access_offset);
726     Block* const access_block = cfg->get_block_for_node(access);
727     const uint access_index = block_index(access_block, access);
728 
729     if (access_obj == nullptr) {
730       // No information available
731       continue;
732     }
733 
734     for (uint j = 0; j < access_dominators.size(); j++) {
735      const  Node* const mem = access_dominators.at(j);
736       if (mem->is_Phi()) {
737         // Allocation node
738         if (mem != access_obj) {
739           continue;
740         }
741         if (is_unknown(access_offset) && !is_array_allocation(mem)) {
742           // The accessed address has an unknown offset, but the allocated
743           // object cannot be determined to be an array. Avoid eliding in this
744           // case, to be on the safe side.
745           continue;
746         }
747         assert((is_concrete(access_offset) && access_offset >= 0) || (is_unknown(access_offset) && is_array_allocation(mem)),
748                "candidate allocation-dominated access offsets must be either concrete and nonnegative, or unknown (for array allocations only)");
749       } else {
750         // Access node
751         const MachNode* const mem_mach = mem->as_Mach();
752         intptr_t mem_offset;
753         const Node* const mem_obj = get_base_and_offset(mem_mach, mem_offset);
754 
755         if (mem_obj == nullptr ||
756             !is_concrete(access_offset) ||
757             !is_concrete(mem_offset)) {
758           // No information available
759           continue;
760         }
761 
762         if (mem_obj != access_obj || mem_offset != access_offset) {
763           // Not the same addresses, not a candidate
764           continue;
765         }
766         assert(is_concrete(access_offset) && access_offset >= 0,
767                "candidate non-allocation-dominated access offsets must be concrete and nonnegative");
768       }
769 
770       Block* mem_block = cfg->get_block_for_node(mem);
771       const uint mem_index = block_index(mem_block, mem);
772 
773       if (access_block == mem_block) {
774         // Earlier accesses in the same block
775         if (mem_index < access_index && !block_has_safepoint(mem_block, mem_index + 1, access_index)) {
776           elide_mach_barrier(access);
777         }
778       } else if (mem_block->dominates(access_block)) {
779         // Dominating block? Look around for safepoints
780         ResourceMark rm;
781         Block_List stack;
782         VectorSet visited;
783         stack.push(access_block);
784         bool safepoint_found = block_has_safepoint(access_block);
785         while (!safepoint_found && stack.size() > 0) {
786           const Block* const block = stack.pop();
787           if (visited.test_set(block->_pre_order)) {
788             continue;
789           }
790           if (block_has_safepoint(block)) {
791             safepoint_found = true;
792             break;
793           }
794           if (block == mem_block) {
795             continue;
796           }
797 
798           // Push predecessor blocks
799           for (uint p = 1; p < block->num_preds(); ++p) {
800             Block* const pred = cfg->get_block_for_node(block->pred(p));
801             stack.push(pred);
802           }
803         }
804 
805         if (!safepoint_found) {
806           elide_mach_barrier(access);
807         }
808       }
809     }
810   }
811 }
812 
813 void ZBarrierSetC2::analyze_dominating_barriers() const {
814   ResourceMark rm;
815   Compile* const C = Compile::current();
816   PhaseCFG* const cfg = C->cfg();
817 
818   Node_List loads;
819   Node_List load_dominators;
820 
821   Node_List stores;
822   Node_List store_dominators;
823 
824   Node_List atomics;
825   Node_List atomic_dominators;
826 
827   // Step 1 - Find accesses and allocations, and track them in lists
828   for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
829     const Block* const block = cfg->get_block(i);
830     for (uint j = 0; j < block->number_of_nodes(); ++j) {
831       Node* const node = block->get_node(j);
832       if (node->is_Phi()) {
833         if (is_allocation(node)) {
834           load_dominators.push(node);
835           store_dominators.push(node);
836           // An allocation can't be considered to "dominate" an atomic operation.
837           // For example a CAS requires the memory location to be store-good.
838           // When you have a dominating store or atomic instruction, that is
839           // indeed ensured to be the case. However, as for allocations, the
840           // initialized memory location could be raw null, which isn't store-good.
841         }
842         continue;
843       } else if (!node->is_Mach()) {
844         continue;
845       }
846 
847       MachNode* const mach = node->as_Mach();
848       switch (mach->ideal_Opcode()) {
849       case Op_LoadP:
850         if ((mach->barrier_data() & ZBarrierStrong) != 0 &&
851             (mach->barrier_data() & ZBarrierNoKeepalive) == 0) {
852           loads.push(mach);
853           load_dominators.push(mach);
854         }
855         break;
856       case Op_StoreP:
857         if (mach->barrier_data() != 0) {
858           stores.push(mach);
859           load_dominators.push(mach);
860           store_dominators.push(mach);
861           atomic_dominators.push(mach);
862         }
863         break;
864       case Op_CompareAndExchangeP:
865       case Op_CompareAndSwapP:
866       case Op_GetAndSetP:
867         if (mach->barrier_data() != 0) {
868           atomics.push(mach);
869           load_dominators.push(mach);
870           store_dominators.push(mach);
871           atomic_dominators.push(mach);
872         }
873         break;
874 
875       default:
876         break;
877       }
878     }
879   }
880 
881   // Step 2 - Find dominating accesses or allocations for each access
882   analyze_dominating_barriers_impl(loads, load_dominators);
883   analyze_dominating_barriers_impl(stores, store_dominators);
884   analyze_dominating_barriers_impl(atomics, atomic_dominators);
885 }
886 
887 // == Reduced spilling optimization ==
888 
889 void ZBarrierSetC2::compute_liveness_at_stubs() const {
890   ResourceMark rm;
891   Compile* const C = Compile::current();
892   Arena* const A = Thread::current()->resource_area();
893   PhaseCFG* const cfg = C->cfg();
894   PhaseRegAlloc* const regalloc = C->regalloc();
895   RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
896   ZBarrierSetAssembler* const bs = ZBarrierSet::assembler();
897   Block_List worklist;
898 
899   for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
900     new ((void*)(live + i)) RegMask();
901     worklist.push(cfg->get_block(i));
902   }
903 
904   while (worklist.size() > 0) {
905     const Block* const block = worklist.pop();
906     RegMask& old_live = live[block->_pre_order];
907     RegMask new_live;
908 
909     // Initialize to union of successors
910     for (uint i = 0; i < block->_num_succs; i++) {
911       const uint succ_id = block->_succs[i]->_pre_order;
912       new_live.OR(live[succ_id]);
913     }
914 
915     // Walk block backwards, computing liveness
916     for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
917       const Node* const node = block->get_node(i);
918 
919       // Remove def bits
920       const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
921       const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
922       if (first != OptoReg::Bad) {
923         new_live.Remove(first);
924       }
925       if (second != OptoReg::Bad) {
926         new_live.Remove(second);
927       }
928 
929       // Add use bits
930       for (uint j = 1; j < node->req(); ++j) {
931         const Node* const use = node->in(j);
932         const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
933         const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
934         if (first != OptoReg::Bad) {
935           new_live.Insert(first);
936         }
937         if (second != OptoReg::Bad) {
938           new_live.Insert(second);
939         }
940       }
941 
942       // If this node tracks liveness, update it
943       RegMask* const regs = barrier_set_state()->live(node);
944       if (regs != nullptr) {
945         regs->OR(new_live);
946       }
947     }
948 
949     // Now at block top, see if we have any changes
950     new_live.SUBTRACT(old_live);
951     if (new_live.is_NotEmpty()) {
952       // Liveness has refined, update and propagate to prior blocks
953       old_live.OR(new_live);
954       for (uint i = 1; i < block->num_preds(); ++i) {
955         Block* const pred = cfg->get_block_for_node(block->pred(i));
956         worklist.push(pred);
957       }
958     }
959   }
960 }
961 
962 void ZBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
963   eliminate_gc_barrier_data(node);
964 }
965 
966 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
967   if (node->is_LoadStore()) {
968     LoadStoreNode* loadstore = node->as_LoadStore();
969     loadstore->set_barrier_data(ZBarrierElided);
970   } else if (node->is_Mem()) {
971     MemNode* mem = node->as_Mem();
972     mem->set_barrier_data(ZBarrierElided);
973   }
974 }
975 
976 #ifndef PRODUCT
977 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
978   if ((mach->barrier_data() & ZBarrierStrong) != 0) {
979     st->print("strong ");
980   }
981   if ((mach->barrier_data() & ZBarrierWeak) != 0) {
982     st->print("weak ");
983   }
984   if ((mach->barrier_data() & ZBarrierPhantom) != 0) {
985     st->print("phantom ");
986   }
987   if ((mach->barrier_data() & ZBarrierNoKeepalive) != 0) {
988     st->print("nokeepalive ");
989   }
990   if ((mach->barrier_data() & ZBarrierNative) != 0) {
991     st->print("native ");
992   }
993   if ((mach->barrier_data() & ZBarrierElided) != 0) {
994     st->print("elided ");
995   }
996 }
997 #endif // !PRODUCT
--- EOF ---