1 /*
  2  * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #include "precompiled.hpp"
 25 #include "asm/macroAssembler.hpp"
 26 #include "classfile/javaClasses.hpp"
 27 #include "gc/z/c2/zBarrierSetC2.hpp"
 28 #include "gc/z/zBarrierSet.hpp"
 29 #include "gc/z/zBarrierSetAssembler.hpp"
 30 #include "gc/z/zBarrierSetRuntime.hpp"
 31 #include "opto/arraycopynode.hpp"
 32 #include "opto/addnode.hpp"
 33 #include "opto/block.hpp"
 34 #include "opto/compile.hpp"
 35 #include "opto/graphKit.hpp"
 36 #include "opto/machnode.hpp"
 37 #include "opto/macro.hpp"
 38 #include "opto/memnode.hpp"
 39 #include "opto/node.hpp"
 40 #include "opto/output.hpp"
 41 #include "opto/regalloc.hpp"
 42 #include "opto/rootnode.hpp"
 43 #include "opto/runtime.hpp"
 44 #include "opto/type.hpp"
 45 #include "utilities/growableArray.hpp"
 46 #include "utilities/macros.hpp"
 47 
 48 template<typename K, typename V, size_t _table_size>
 49 class ZArenaHashtable : public ResourceObj {
 50   class ZArenaHashtableEntry : public ResourceObj {
 51   public:
 52     ZArenaHashtableEntry* _next;
 53     K _key;
 54     V _value;
 55   };
 56 
 57   static const size_t _table_mask = _table_size - 1;
 58 
 59   Arena* _arena;
 60   ZArenaHashtableEntry* _table[_table_size];
 61 
 62 public:
 63   class Iterator {
 64     ZArenaHashtable* _table;
 65     ZArenaHashtableEntry* _current_entry;
 66     size_t _current_index;
 67 
 68   public:
 69     Iterator(ZArenaHashtable* table)
 70       : _table(table),
 71         _current_entry(table->_table[0]),
 72         _current_index(0) {
 73       if (_current_entry == nullptr) {
 74         next();
 75       }
 76     }
 77 
 78     bool has_next() { return _current_entry != nullptr; }
 79     K key()         { return _current_entry->_key; }
 80     V value()       { return _current_entry->_value; }
 81 
 82     void next() {
 83       if (_current_entry != nullptr) {
 84         _current_entry = _current_entry->_next;
 85       }
 86       while (_current_entry == nullptr && ++_current_index < _table_size) {
 87         _current_entry = _table->_table[_current_index];
 88       }
 89     }
 90   };
 91 
 92   ZArenaHashtable(Arena* arena)
 93     : _arena(arena),
 94       _table() {
 95     Copy::zero_to_bytes(&_table, sizeof(_table));
 96   }
 97 
 98   void add(K key, V value) {
 99     ZArenaHashtableEntry* entry = new (_arena) ZArenaHashtableEntry();
100     entry->_key = key;
101     entry->_value = value;
102     entry->_next = _table[key & _table_mask];
103     _table[key & _table_mask] = entry;
104   }
105 
106   V* get(K key) const {
107     for (ZArenaHashtableEntry* e = _table[key & _table_mask]; e != nullptr; e = e->_next) {
108       if (e->_key == key) {
109         return &(e->_value);
110       }
111     }
112     return nullptr;
113   }
114 
115   Iterator iterator() {
116     return Iterator(this);
117   }
118 };
119 
120 typedef ZArenaHashtable<intptr_t, bool, 4> ZOffsetTable;
121 
122 class ZBarrierSetC2State : public ArenaObj {
123 private:
124   GrowableArray<ZBarrierStubC2*>* _stubs;
125   Node_Array                      _live;
126   int                             _trampoline_stubs_count;
127   int                             _stubs_start_offset;
128 
129 public:
130   ZBarrierSetC2State(Arena* arena)
131     : _stubs(new (arena) GrowableArray<ZBarrierStubC2*>(arena, 8,  0, nullptr)),
132       _live(arena),
133       _trampoline_stubs_count(0),
134       _stubs_start_offset(0) {}
135 
136   GrowableArray<ZBarrierStubC2*>* stubs() {
137     return _stubs;
138   }
139 
140   RegMask* live(const Node* node) {
141     if (!node->is_Mach()) {
142       // Don't need liveness for non-MachNodes
143       return nullptr;
144     }
145 
146     const MachNode* const mach = node->as_Mach();
147     if (mach->barrier_data() == ZBarrierElided) {
148       // Don't need liveness data for nodes without barriers
149       return nullptr;
150     }
151 
152     RegMask* live = (RegMask*)_live[node->_idx];
153     if (live == nullptr) {
154       live = new (Compile::current()->comp_arena()->AmallocWords(sizeof(RegMask))) RegMask();
155       _live.map(node->_idx, (Node*)live);
156     }
157 
158     return live;
159   }
160 
161   void inc_trampoline_stubs_count() {
162     assert(_trampoline_stubs_count != INT_MAX, "Overflow");
163     ++_trampoline_stubs_count;
164   }
165 
166   int trampoline_stubs_count() {
167     return _trampoline_stubs_count;
168   }
169 
170   void set_stubs_start_offset(int offset) {
171     _stubs_start_offset = offset;
172   }
173 
174   int stubs_start_offset() {
175     return _stubs_start_offset;
176   }
177 };
178 
179 static ZBarrierSetC2State* barrier_set_state() {
180   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
181 }
182 
183 void ZBarrierStubC2::register_stub(ZBarrierStubC2* stub) {
184   if (!Compile::current()->output()->in_scratch_emit_size()) {
185     barrier_set_state()->stubs()->append(stub);
186   }
187 }
188 
189 void ZBarrierStubC2::inc_trampoline_stubs_count() {
190   if (!Compile::current()->output()->in_scratch_emit_size()) {
191     barrier_set_state()->inc_trampoline_stubs_count();
192   }
193 }
194 
195 int ZBarrierStubC2::trampoline_stubs_count() {
196   return barrier_set_state()->trampoline_stubs_count();
197 }
198 
199 int ZBarrierStubC2::stubs_start_offset() {
200   return barrier_set_state()->stubs_start_offset();
201 }
202 
203 ZBarrierStubC2::ZBarrierStubC2(const MachNode* node)
204   : _node(node),
205     _entry(),
206     _continuation() {}
207 
208 Register ZBarrierStubC2::result() const {
209   return noreg;
210 }
211 
212 RegMask& ZBarrierStubC2::live() const {
213   return *barrier_set_state()->live(_node);
214 }
215 
216 Label* ZBarrierStubC2::entry() {
217   // The _entry will never be bound when in_scratch_emit_size() is true.
218   // However, we still need to return a label that is not bound now, but
219   // will eventually be bound. Any eventually bound label will do, as it
220   // will only act as a placeholder, so we return the _continuation label.
221   return Compile::current()->output()->in_scratch_emit_size() ? &_continuation : &_entry;
222 }
223 
224 Label* ZBarrierStubC2::continuation() {
225   return &_continuation;
226 }
227 
228 ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref) {
229   ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref);
230   register_stub(stub);
231 
232   return stub;
233 }
234 
235 ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref)
236   : ZBarrierStubC2(node),
237     _ref_addr(ref_addr),
238     _ref(ref) {
239   assert_different_registers(ref, ref_addr.base());
240   assert_different_registers(ref, ref_addr.index());
241 }
242 
243 Address ZLoadBarrierStubC2::ref_addr() const {
244   return _ref_addr;
245 }
246 
247 Register ZLoadBarrierStubC2::ref() const {
248   return _ref;
249 }
250 
251 Register ZLoadBarrierStubC2::result() const {
252   return ref();
253 }
254 
255 address ZLoadBarrierStubC2::slow_path() const {
256   const uint8_t barrier_data = _node->barrier_data();
257   DecoratorSet decorators = DECORATORS_NONE;
258   if (barrier_data & ZBarrierStrong) {
259     decorators |= ON_STRONG_OOP_REF;
260   }
261   if (barrier_data & ZBarrierWeak) {
262     decorators |= ON_WEAK_OOP_REF;
263   }
264   if (barrier_data & ZBarrierPhantom) {
265     decorators |= ON_PHANTOM_OOP_REF;
266   }
267   if (barrier_data & ZBarrierNoKeepalive) {
268     decorators |= AS_NO_KEEPALIVE;
269   }
270   return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
271 }
272 
273 void ZLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
274   ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, static_cast<ZLoadBarrierStubC2*>(this));
275 }
276 
277 ZStoreBarrierStubC2* ZStoreBarrierStubC2::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic) {
278   ZStoreBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic);
279   register_stub(stub);
280 
281   return stub;
282 }
283 
284 ZStoreBarrierStubC2::ZStoreBarrierStubC2(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic)
285   : ZBarrierStubC2(node),
286     _ref_addr(ref_addr),
287     _new_zaddress(new_zaddress),
288     _new_zpointer(new_zpointer),
289     _is_native(is_native),
290     _is_atomic(is_atomic) {}
291 
292 Address ZStoreBarrierStubC2::ref_addr() const {
293   return _ref_addr;
294 }
295 
296 Register ZStoreBarrierStubC2::new_zaddress() const {
297   return _new_zaddress;
298 }
299 
300 Register ZStoreBarrierStubC2::new_zpointer() const {
301   return _new_zpointer;
302 }
303 
304 bool ZStoreBarrierStubC2::is_native() const {
305   return _is_native;
306 }
307 
308 bool ZStoreBarrierStubC2::is_atomic() const {
309   return _is_atomic;
310 }
311 
312 Register ZStoreBarrierStubC2::result() const {
313   return noreg;
314 }
315 
316 void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
317   ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast<ZStoreBarrierStubC2*>(this));
318 }
319 
320 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
321   return new (comp_arena) ZBarrierSetC2State(comp_arena);
322 }
323 
324 void ZBarrierSetC2::late_barrier_analysis() const {
325   compute_liveness_at_stubs();
326   analyze_dominating_barriers();
327 }
328 
329 void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
330   MacroAssembler masm(&cb);
331   GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
332   barrier_set_state()->set_stubs_start_offset(masm.offset());
333 
334   for (int i = 0; i < stubs->length(); i++) {
335     // Make sure there is enough space in the code buffer
336     if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
337       ciEnv::current()->record_failure("CodeCache is full");
338       return;
339     }
340 
341     stubs->at(i)->emit_code(masm);
342   }
343 
344   masm.flush();
345 }
346 
347 int ZBarrierSetC2::estimate_stub_size() const {
348   Compile* const C = Compile::current();
349   BufferBlob* const blob = C->output()->scratch_buffer_blob();
350   GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
351   int size = 0;
352 
353   for (int i = 0; i < stubs->length(); i++) {
354     CodeBuffer cb(blob->content_begin(), (address)C->output()->scratch_locs_memory() - blob->content_begin());
355     MacroAssembler masm(&cb);
356     stubs->at(i)->emit_code(masm);
357     size += cb.insts_size();
358   }
359 
360   return size;
361 }
362 
363 static void set_barrier_data(C2Access& access) {
364   if (!ZBarrierSet::barrier_needed(access.decorators(), access.type())) {
365     return;
366   }
367 
368   if (access.decorators() & C2_TIGHTLY_COUPLED_ALLOC) {
369     access.set_barrier_data(ZBarrierElided);
370     return;
371   }
372 
373   uint8_t barrier_data = 0;
374 
375   if (access.decorators() & ON_PHANTOM_OOP_REF) {
376     barrier_data |= ZBarrierPhantom;
377   } else if (access.decorators() & ON_WEAK_OOP_REF) {
378     barrier_data |= ZBarrierWeak;
379   } else {
380     barrier_data |= ZBarrierStrong;
381   }
382 
383   if (access.decorators() & IN_NATIVE) {
384     barrier_data |= ZBarrierNative;
385   }
386 
387   if (access.decorators() & AS_NO_KEEPALIVE) {
388     barrier_data |= ZBarrierNoKeepalive;
389   }
390 
391   access.set_barrier_data(barrier_data);
392 }
393 
394 Node* ZBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
395   set_barrier_data(access);
396   return BarrierSetC2::store_at_resolved(access, val);
397 }
398 
399 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
400   set_barrier_data(access);
401   return BarrierSetC2::load_at_resolved(access, val_type);
402 }
403 
404 Node* ZBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
405                                                     Node* new_val, const Type* val_type) const {
406   set_barrier_data(access);
407   return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
408 }
409 
410 Node* ZBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
411                                                      Node* new_val, const Type* value_type) const {
412   set_barrier_data(access);
413   return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
414 }
415 
416 Node* ZBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
417   set_barrier_data(access);
418   return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
419 }
420 
421 bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
422                                                     bool is_clone, bool is_clone_instance,
423                                                     ArrayCopyPhase phase) const {
424   if (phase == ArrayCopyPhase::Parsing) {
425     return false;
426   }
427   if (phase == ArrayCopyPhase::Optimization) {
428     return is_clone_instance;
429   }
430   // else ArrayCopyPhase::Expansion
431   return type == T_OBJECT || type == T_ARRAY;
432 }
433 
434 // This TypeFunc assumes a 64bit system
435 static const TypeFunc* clone_type() {
436   // Create input type (domain)
437   const Type** const domain_fields = TypeTuple::fields(4);
438   domain_fields[TypeFunc::Parms + 0] = TypeInstPtr::NOTNULL;  // src
439   domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL;  // dst
440   domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG;        // size lower
441   domain_fields[TypeFunc::Parms + 3] = Type::HALF;            // size upper
442   const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
443 
444   // Create result type (range)
445   const Type** const range_fields = TypeTuple::fields(0);
446   const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
447 
448   return TypeFunc::make(domain, range);
449 }
450 
451 #define XTOP LP64_ONLY(COMMA phase->top())
452 
453 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
454   Node* const src = ac->in(ArrayCopyNode::Src);
455   const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
456 
457   if (ac->is_clone_array() && ary_ptr != nullptr) {
458     BasicType bt = ary_ptr->elem()->array_element_basic_type();
459     if (is_reference_type(bt) && !ary_ptr->is_flat()) {
460       // Clone object array
461       bt = T_OBJECT;
462     } else {
463       // Clone primitive array
464       bt = T_LONG;
465     }
466 
467     Node* const ctrl = ac->in(TypeFunc::Control);
468     Node* const mem = ac->in(TypeFunc::Memory);
469     Node* const src = ac->in(ArrayCopyNode::Src);
470     Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
471     Node* const dest = ac->in(ArrayCopyNode::Dest);
472     Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
473     Node* length = ac->in(ArrayCopyNode::Length);
474 
475     if (bt == T_OBJECT) {
476       // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
477       // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
478       // to the first element in the array when cloning object arrays. Otherwise, load
479       // barriers are applied to parts of the header. Also adjust the length accordingly.
480       assert(src_offset == dest_offset, "should be equal");
481       const jlong offset = src_offset->get_long();
482       if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
483         assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
484         assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
485         length = phase->transform_later(new SubXNode(length, phase->longcon(1))); // Size is in longs
486         src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
487         dest_offset = src_offset;
488       }
489     }
490     Node* const payload_src = phase->basic_plus_adr(src, src_offset);
491     Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
492 
493     const char*   copyfunc_name = "arraycopy";
494     const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
495 
496     const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
497     const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
498 
499     Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
500     phase->transform_later(call);
501 
502     phase->igvn().replace_node(ac, call);
503     return;
504   }
505 
506   // Clone instance
507   Node* const ctrl       = ac->in(TypeFunc::Control);
508   Node* const mem        = ac->in(TypeFunc::Memory);
509   Node* const dst        = ac->in(ArrayCopyNode::Dest);
510   Node* const size       = ac->in(ArrayCopyNode::Length);
511 
512   assert(size->bottom_type()->is_long(), "Should be long");
513 
514   // The native clone we are calling here expects the instance size in words
515   // Add header/offset size to payload size to get instance size.
516   Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
517   Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
518 
519   Node* const call = phase->make_leaf_call(ctrl,
520                                            mem,
521                                            clone_type(),
522                                            ZBarrierSetRuntime::clone_addr(),
523                                            "ZBarrierSetRuntime::clone",
524                                            TypeRawPtr::BOTTOM,
525                                            src,
526                                            dst,
527                                            full_size,
528                                            phase->top());
529   phase->transform_later(call);
530   phase->replace_node(ac, call);
531 }
532 
533 #undef XTOP
534 
535 // == Dominating barrier elision ==
536 
537 static bool block_has_safepoint(const Block* block, uint from, uint to) {
538   for (uint i = from; i < to; i++) {
539     if (block->get_node(i)->is_MachSafePoint()) {
540       // Safepoint found
541       return true;
542     }
543   }
544 
545   // Safepoint not found
546   return false;
547 }
548 
549 static bool block_has_safepoint(const Block* block) {
550   return block_has_safepoint(block, 0, block->number_of_nodes());
551 }
552 
553 static uint block_index(const Block* block, const Node* node) {
554   for (uint j = 0; j < block->number_of_nodes(); ++j) {
555     if (block->get_node(j) == node) {
556       return j;
557     }
558   }
559   ShouldNotReachHere();
560   return 0;
561 }
562 
563 // Look through various node aliases
564 static const Node* look_through_node(const Node* node) {
565   while (node != nullptr) {
566     const Node* new_node = node;
567     if (node->is_Mach()) {
568       const MachNode* const node_mach = node->as_Mach();
569       if (node_mach->ideal_Opcode() == Op_CheckCastPP) {
570         new_node = node->in(1);
571       }
572       if (node_mach->is_SpillCopy()) {
573         new_node = node->in(1);
574       }
575     }
576     if (new_node == node || new_node == nullptr) {
577       break;
578     } else {
579       node = new_node;
580     }
581   }
582 
583   return node;
584 }
585 
586 // Whether the given offset is undefined.
587 static bool is_undefined(intptr_t offset) {
588   return offset == Type::OffsetTop;
589 }
590 
591 // Whether the given offset is unknown.
592 static bool is_unknown(intptr_t offset) {
593   return offset == Type::OffsetBot;
594 }
595 
596 // Whether the given offset is concrete (defined and compile-time known).
597 static bool is_concrete(intptr_t offset) {
598   return !is_undefined(offset) && !is_unknown(offset);
599 }
600 
601 // Compute base + offset components of the memory address accessed by mach.
602 // Return a node representing the base address, or null if the base cannot be
603 // found or the offset is undefined or a concrete negative value. If a non-null
604 // base is returned, the offset is a concrete, nonnegative value or unknown.
605 static const Node* get_base_and_offset(const MachNode* mach, intptr_t& offset) {
606   const TypePtr* adr_type = nullptr;
607   offset = 0;
608   const Node* base = mach->get_base_and_disp(offset, adr_type);
609 
610   if (base == nullptr || base == NodeSentinel) {
611     return nullptr;
612   }
613 
614   if (offset == 0 && base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_AddP) {
615     // The memory address is computed by 'base' and fed to 'mach' via an
616     // indirect memory operand (indicated by offset == 0). The ultimate base and
617     // offset can be fetched directly from the inputs and Ideal type of 'base'.
618     offset = base->bottom_type()->isa_oopptr()->offset();
619     // Even if 'base' is not an Ideal AddP node anymore, Matcher::ReduceInst()
620     // guarantees that the base address is still available at the same slot.
621     base = base->in(AddPNode::Base);
622     assert(base != nullptr, "");
623   }
624 
625   if (is_undefined(offset) || (is_concrete(offset) && offset < 0)) {
626     return nullptr;
627   }
628 
629   return look_through_node(base);
630 }
631 
632 // Whether a phi node corresponds to an array allocation.
633 // This test is incomplete: in some edge cases, it might return false even
634 // though the node does correspond to an array allocation.
635 static bool is_array_allocation(const Node* phi) {
636   precond(phi->is_Phi());
637   // Check whether phi has a successor cast (CheckCastPP) to Java array pointer,
638   // possibly below spill copies and other cast nodes. Limit the exploration to
639   // a single path from the phi node consisting of these node types.
640   const Node* current = phi;
641   while (true) {
642     const Node* next = nullptr;
643     for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) {
644       if (!current->fast_out(i)->isa_Mach()) {
645         continue;
646       }
647       const MachNode* succ = current->fast_out(i)->as_Mach();
648       if (succ->ideal_Opcode() == Op_CheckCastPP) {
649         if (succ->get_ptr_type()->isa_aryptr()) {
650           // Cast to Java array pointer: phi corresponds to an array allocation.
651           return true;
652         }
653         // Other cast: record as candidate for further exploration.
654         next = succ;
655       } else if (succ->is_SpillCopy() && next == nullptr) {
656         // Spill copy, and no better candidate found: record as candidate.
657         next = succ;
658       }
659     }
660     if (next == nullptr) {
661       // No evidence found that phi corresponds to an array allocation, and no
662       // candidates available to continue exploring.
663       return false;
664     }
665     // Continue exploring from the best candidate found.
666     current = next;
667   }
668   ShouldNotReachHere();
669 }
670 
671 // Match the phi node that connects a TLAB allocation fast path with its slowpath
672 static bool is_allocation(const Node* node) {
673   if (node->req() != 3) {
674     return false;
675   }
676   const Node* const fast_node = node->in(2);
677   if (!fast_node->is_Mach()) {
678     return false;
679   }
680   const MachNode* const fast_mach = fast_node->as_Mach();
681   if (fast_mach->ideal_Opcode() != Op_LoadP) {
682     return false;
683   }
684   const TypePtr* const adr_type = nullptr;
685   intptr_t offset;
686   const Node* const base = get_base_and_offset(fast_mach, offset);
687   if (base == nullptr || !base->is_Mach() || !is_concrete(offset)) {
688     return false;
689   }
690   const MachNode* const base_mach = base->as_Mach();
691   if (base_mach->ideal_Opcode() != Op_ThreadLocal) {
692     return false;
693   }
694   return offset == in_bytes(Thread::tlab_top_offset());
695 }
696 
697 static void elide_mach_barrier(MachNode* mach) {
698   mach->set_barrier_data(ZBarrierElided);
699 }
700 
701 void ZBarrierSetC2::analyze_dominating_barriers_impl(Node_List& accesses, Node_List& access_dominators) const {
702   Compile* const C = Compile::current();
703   PhaseCFG* const cfg = C->cfg();
704 
705   for (uint i = 0; i < accesses.size(); i++) {
706     MachNode* const access = accesses.at(i)->as_Mach();
707     intptr_t access_offset;
708     const Node* const access_obj = get_base_and_offset(access, access_offset);
709     Block* const access_block = cfg->get_block_for_node(access);
710     const uint access_index = block_index(access_block, access);
711 
712     if (access_obj == nullptr) {
713       // No information available
714       continue;
715     }
716 
717     for (uint j = 0; j < access_dominators.size(); j++) {
718      const  Node* const mem = access_dominators.at(j);
719       if (mem->is_Phi()) {
720         // Allocation node
721         if (mem != access_obj) {
722           continue;
723         }
724         if (is_unknown(access_offset) && !is_array_allocation(mem)) {
725           // The accessed address has an unknown offset, but the allocated
726           // object cannot be determined to be an array. Avoid eliding in this
727           // case, to be on the safe side.
728           continue;
729         }
730         assert((is_concrete(access_offset) && access_offset >= 0) || (is_unknown(access_offset) && is_array_allocation(mem)),
731                "candidate allocation-dominated access offsets must be either concrete and nonnegative, or unknown (for array allocations only)");
732       } else {
733         // Access node
734         const MachNode* const mem_mach = mem->as_Mach();
735         intptr_t mem_offset;
736         const Node* const mem_obj = get_base_and_offset(mem_mach, mem_offset);
737 
738         if (mem_obj == nullptr ||
739             !is_concrete(access_offset) ||
740             !is_concrete(mem_offset)) {
741           // No information available
742           continue;
743         }
744 
745         if (mem_obj != access_obj || mem_offset != access_offset) {
746           // Not the same addresses, not a candidate
747           continue;
748         }
749         assert(is_concrete(access_offset) && access_offset >= 0,
750                "candidate non-allocation-dominated access offsets must be concrete and nonnegative");
751       }
752 
753       Block* mem_block = cfg->get_block_for_node(mem);
754       const uint mem_index = block_index(mem_block, mem);
755 
756       if (access_block == mem_block) {
757         // Earlier accesses in the same block
758         if (mem_index < access_index && !block_has_safepoint(mem_block, mem_index + 1, access_index)) {
759           elide_mach_barrier(access);
760         }
761       } else if (mem_block->dominates(access_block)) {
762         // Dominating block? Look around for safepoints
763         ResourceMark rm;
764         Block_List stack;
765         VectorSet visited;
766         stack.push(access_block);
767         bool safepoint_found = block_has_safepoint(access_block);
768         while (!safepoint_found && stack.size() > 0) {
769           const Block* const block = stack.pop();
770           if (visited.test_set(block->_pre_order)) {
771             continue;
772           }
773           if (block_has_safepoint(block)) {
774             safepoint_found = true;
775             break;
776           }
777           if (block == mem_block) {
778             continue;
779           }
780 
781           // Push predecessor blocks
782           for (uint p = 1; p < block->num_preds(); ++p) {
783             Block* const pred = cfg->get_block_for_node(block->pred(p));
784             stack.push(pred);
785           }
786         }
787 
788         if (!safepoint_found) {
789           elide_mach_barrier(access);
790         }
791       }
792     }
793   }
794 }
795 
796 void ZBarrierSetC2::analyze_dominating_barriers() const {
797   ResourceMark rm;
798   Compile* const C = Compile::current();
799   PhaseCFG* const cfg = C->cfg();
800 
801   Node_List loads;
802   Node_List load_dominators;
803 
804   Node_List stores;
805   Node_List store_dominators;
806 
807   Node_List atomics;
808   Node_List atomic_dominators;
809 
810   // Step 1 - Find accesses and allocations, and track them in lists
811   for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
812     const Block* const block = cfg->get_block(i);
813     for (uint j = 0; j < block->number_of_nodes(); ++j) {
814       Node* const node = block->get_node(j);
815       if (node->is_Phi()) {
816         if (is_allocation(node)) {
817           load_dominators.push(node);
818           store_dominators.push(node);
819           // An allocation can't be considered to "dominate" an atomic operation.
820           // For example a CAS requires the memory location to be store-good.
821           // When you have a dominating store or atomic instruction, that is
822           // indeed ensured to be the case. However, as for allocations, the
823           // initialized memory location could be raw null, which isn't store-good.
824         }
825         continue;
826       } else if (!node->is_Mach()) {
827         continue;
828       }
829 
830       MachNode* const mach = node->as_Mach();
831       switch (mach->ideal_Opcode()) {
832       case Op_LoadP:
833         if ((mach->barrier_data() & ZBarrierStrong) != 0 &&
834             (mach->barrier_data() & ZBarrierNoKeepalive) == 0) {
835           loads.push(mach);
836           load_dominators.push(mach);
837         }
838         break;
839       case Op_StoreP:
840         if (mach->barrier_data() != 0) {
841           stores.push(mach);
842           load_dominators.push(mach);
843           store_dominators.push(mach);
844           atomic_dominators.push(mach);
845         }
846         break;
847       case Op_CompareAndExchangeP:
848       case Op_CompareAndSwapP:
849       case Op_GetAndSetP:
850         if (mach->barrier_data() != 0) {
851           atomics.push(mach);
852           load_dominators.push(mach);
853           store_dominators.push(mach);
854           atomic_dominators.push(mach);
855         }
856         break;
857 
858       default:
859         break;
860       }
861     }
862   }
863 
864   // Step 2 - Find dominating accesses or allocations for each access
865   analyze_dominating_barriers_impl(loads, load_dominators);
866   analyze_dominating_barriers_impl(stores, store_dominators);
867   analyze_dominating_barriers_impl(atomics, atomic_dominators);
868 }
869 
870 // == Reduced spilling optimization ==
871 
872 void ZBarrierSetC2::compute_liveness_at_stubs() const {
873   ResourceMark rm;
874   Compile* const C = Compile::current();
875   Arena* const A = Thread::current()->resource_area();
876   PhaseCFG* const cfg = C->cfg();
877   PhaseRegAlloc* const regalloc = C->regalloc();
878   RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
879   ZBarrierSetAssembler* const bs = ZBarrierSet::assembler();
880   Block_List worklist;
881 
882   for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
883     new ((void*)(live + i)) RegMask();
884     worklist.push(cfg->get_block(i));
885   }
886 
887   while (worklist.size() > 0) {
888     const Block* const block = worklist.pop();
889     RegMask& old_live = live[block->_pre_order];
890     RegMask new_live;
891 
892     // Initialize to union of successors
893     for (uint i = 0; i < block->_num_succs; i++) {
894       const uint succ_id = block->_succs[i]->_pre_order;
895       new_live.OR(live[succ_id]);
896     }
897 
898     // Walk block backwards, computing liveness
899     for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
900       const Node* const node = block->get_node(i);
901 
902       // Remove def bits
903       const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
904       const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
905       if (first != OptoReg::Bad) {
906         new_live.Remove(first);
907       }
908       if (second != OptoReg::Bad) {
909         new_live.Remove(second);
910       }
911 
912       // Add use bits
913       for (uint j = 1; j < node->req(); ++j) {
914         const Node* const use = node->in(j);
915         const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
916         const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
917         if (first != OptoReg::Bad) {
918           new_live.Insert(first);
919         }
920         if (second != OptoReg::Bad) {
921           new_live.Insert(second);
922         }
923       }
924 
925       // If this node tracks liveness, update it
926       RegMask* const regs = barrier_set_state()->live(node);
927       if (regs != nullptr) {
928         regs->OR(new_live);
929       }
930     }
931 
932     // Now at block top, see if we have any changes
933     new_live.SUBTRACT(old_live);
934     if (new_live.is_NotEmpty()) {
935       // Liveness has refined, update and propagate to prior blocks
936       old_live.OR(new_live);
937       for (uint i = 1; i < block->num_preds(); ++i) {
938         Block* const pred = cfg->get_block_for_node(block->pred(i));
939         worklist.push(pred);
940       }
941     }
942   }
943 }
944 
945 void ZBarrierSetC2::eliminate_gc_barrier(PhaseIterGVN* igvn, Node* node) const {
946   eliminate_gc_barrier_data(node);
947 }
948 
949 void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
950   if (node->is_LoadStore()) {
951     LoadStoreNode* loadstore = node->as_LoadStore();
952     loadstore->set_barrier_data(ZBarrierElided);
953   } else if (node->is_Mem()) {
954     MemNode* mem = node->as_Mem();
955     mem->set_barrier_data(ZBarrierElided);
956   }
957 }
958 
959 #ifndef PRODUCT
960 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
961   if ((mach->barrier_data() & ZBarrierStrong) != 0) {
962     st->print("strong ");
963   }
964   if ((mach->barrier_data() & ZBarrierWeak) != 0) {
965     st->print("weak ");
966   }
967   if ((mach->barrier_data() & ZBarrierPhantom) != 0) {
968     st->print("phantom ");
969   }
970   if ((mach->barrier_data() & ZBarrierNoKeepalive) != 0) {
971     st->print("nokeepalive ");
972   }
973   if ((mach->barrier_data() & ZBarrierNative) != 0) {
974     st->print("native ");
975   }
976   if ((mach->barrier_data() & ZBarrierElided) != 0) {
977     st->print("elided ");
978   }
979 }
980 #endif // !PRODUCT