1 /*
  2  * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #ifndef SHARE_GC_Z_ZBARRIER_HPP
 25 #define SHARE_GC_Z_ZBARRIER_HPP
 26 
 27 #include "gc/z/zAddress.hpp"
 28 #include "memory/allStatic.hpp"
 29 #include "memory/iterator.hpp"
 30 
 31 // == Shift based load barrier ==
 32 //
 33 // The load barriers of ZGC check if a loaded value is safe to expose or not, and
 34 // then shifts the pointer to remove metadata bits, such that it points to mapped
 35 // memory.
 36 //
 37 // A pointer is safe to expose if it does not have any load-bad bits set in its
 38 // metadata bits. In the C++ code and non-nmethod generated code, that is checked
 39 // by testing the pointer value against a load-bad mask, checking that no bad bit
 40 // is set, followed by a shift, removing the metadata bits if they were good.
 41 // However, for nmethod code, the test + shift sequence is optimized in such
 42 // a way that the shift both tests if the pointer is exposable or not, and removes
 43 // the metadata bits, with the same instruction. This is a speculative optimization
 44 // that assumes that the loaded pointer is frequently going to be load-good or null
 45 // when checked. Therefore, the nmethod load barriers just apply the shift with the
 46 // current "good" shift (which is patched with nmethod entry barriers for each GC
 47 // phase). If the result of that shift was a raw null value, then the ZF flag is set.
 48 // If the result is a good pointer, then the very last bit that was removed by the
 49 // shift, must have been a 1, which would have set the CF flag. Therefore, the "above"
 50 // branch condition code is used to take a slowpath only iff CF == 0 and ZF == 0.
 51 // CF == 0 implies it was not a good pointer, and ZF == 0 implies the resulting address
 52 // was not a null value. Then we decide that the pointer is bad. This optimization
 53 // is necessary to get satisfactory performance, but does come with a few constraints:
 54 //
 55 // 1) The load barrier can only recognize 4 different good patterns across all GC phases.
 56 //    The reason is that when a load barrier applies the currently good shift, then
 57 //    the value of said shift may differ only by 3, until we risk shifting away more
 58 //    than the low order three zeroes of an address, given a bad pointer, which would
 59 //    yield spurious false positives.
 60 //
 61 // 2) Those bit patterns must have only a single bit set. We achieve that by moving
 62 //    non-relocation work to store barriers.
 63 //
 64 // Another consequence of this speculative optimization, is that when the compiled code
 65 // takes a slow path, it needs to reload the oop, because the shifted oop is now
 66 // broken after being shifted with a different shift to what was used when the oop
 67 // was stored.
 68 
 69 typedef bool (*ZBarrierFastPath)(zpointer);
 70 typedef zpointer (*ZBarrierColor)(zaddress, zpointer);
 71 
 72 class ZGeneration;
 73 
 74 void z_assert_is_barrier_safe();
 75 
 76 class ZBarrier : public AllStatic {
 77   friend class ZContinuation;
 78   friend class ZStoreBarrierBuffer;
 79   friend class ZUncoloredRoot;
 80 
 81 private:
 82   static void assert_transition_monotonicity(zpointer ptr, zpointer heal_ptr);
 83   static void self_heal(ZBarrierFastPath fast_path, volatile zpointer* p, zpointer ptr, zpointer heal_ptr, bool allow_null);
 84 
 85   template <typename ZBarrierSlowPath>
 86   static zaddress barrier(ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path, ZBarrierColor color, volatile zpointer* p, zpointer o, bool allow_null = false);
 87 
 88   static zaddress make_load_good(zpointer ptr);
 89   static zaddress make_load_good_no_relocate(zpointer ptr);
 90   static zaddress relocate_or_remap(zaddress_unsafe addr, ZGeneration* generation);
 91   static zaddress remap(zaddress_unsafe addr, ZGeneration* generation);
 92   static void remember(volatile zpointer* p);
 93   static void mark_and_remember(volatile zpointer* p, zaddress addr);
 94 
 95   // Fast paths in increasing strength level
 96   static bool is_load_good_or_null_fast_path(zpointer ptr);
 97   static bool is_mark_good_fast_path(zpointer ptr);
 98   static bool is_store_good_fast_path(zpointer ptr);
 99   static bool is_store_good_or_null_fast_path(zpointer ptr);
100   static bool is_store_good_or_null_any_fast_path(zpointer ptr);
101 
102   static bool is_mark_young_good_fast_path(zpointer ptr);
103   static bool is_finalizable_good_fast_path(zpointer ptr);
104 
105   // Slow paths
106   static zaddress blocking_keep_alive_on_weak_slow_path(volatile zpointer* p, zaddress addr);
107   static zaddress blocking_keep_alive_on_phantom_slow_path(volatile zpointer* p, zaddress addr);
108   static zaddress blocking_load_barrier_on_weak_slow_path(volatile zpointer* p, zaddress addr);
109   static zaddress blocking_load_barrier_on_phantom_slow_path(volatile zpointer* p, zaddress addr);
110 
111   static zaddress mark_slow_path(zaddress addr);
112   static zaddress mark_young_slow_path(zaddress addr);
113   static zaddress mark_from_young_slow_path(zaddress addr);
114   static zaddress mark_from_old_slow_path(zaddress addr);
115   static zaddress mark_finalizable_slow_path(zaddress addr);
116   static zaddress mark_finalizable_from_old_slow_path(zaddress addr);
117 
118   static zaddress keep_alive_slow_path(zaddress addr);
119   static zaddress heap_store_slow_path(volatile zpointer* p, zaddress addr, zpointer prev, bool heal);
120   static zaddress native_store_slow_path(zaddress addr);
121   static zaddress no_keep_alive_heap_store_slow_path(volatile zpointer* p, zaddress addr);
122 
123   static zaddress promote_slow_path(zaddress addr);
124 
125   // Helpers for non-strong oop refs barriers
126   static zaddress blocking_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
127   static zaddress blocking_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
128   static zaddress blocking_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
129   static zaddress blocking_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
130 
131   // Verification
132   static void verify_on_weak(volatile zpointer* referent_addr) NOT_DEBUG_RETURN;
133 
134 public:
135 
136   static zpointer load_atomic(volatile zpointer* p);
137 
138   // Helpers for relocation
139   static ZGeneration* remap_generation(zpointer ptr);
140   static void remap_young_relocated(volatile zpointer* p, zpointer o);
141 
142   // Helpers for marking
143   template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
144   static void mark(zaddress addr);
145   template <bool resurrect, bool gc_thread, bool follow>
146   static void mark_young(zaddress addr);
147   template <bool resurrect, bool gc_thread, bool follow>
148   static void mark_if_young(zaddress addr);
149 
150   // Load barrier
151   static zaddress load_barrier_on_oop_field(volatile zpointer* p);
152   static zaddress load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o);
153 
154   static void load_barrier_on_oop_array(volatile zpointer* p, size_t length);
155 
156   static zaddress keep_alive_load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o);
157 
158   // Load barriers on non-strong oop refs
159   static zaddress load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
160   static zaddress load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
161 
162   static zaddress no_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
163   static zaddress no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
164 
165   // Reference processor / weak cleaning barriers
166   static bool clean_barrier_on_weak_oop_field(volatile zpointer* p);
167   static bool clean_barrier_on_phantom_oop_field(volatile zpointer* p);
168   static bool clean_barrier_on_final_oop_field(volatile zpointer* p);
169 
170   // Mark barrier
171   static void mark_barrier_on_young_oop_field(volatile zpointer* p);
172   static void mark_barrier_on_old_oop_field(volatile zpointer* p, bool finalizable);
173   static void mark_barrier_on_oop_field(volatile zpointer* p, bool finalizable);
174   static void mark_young_good_barrier_on_oop_field(volatile zpointer* p);
175   static zaddress remset_barrier_on_oop_field(volatile zpointer* p);
176   static void promote_barrier_on_young_oop_field(volatile zpointer* p);
177 
178   // Store barrier
179   static void store_barrier_on_heap_oop_field(volatile zpointer* p, bool heal);
180   static void store_barrier_on_native_oop_field(volatile zpointer* p, bool heal);
181 
182   static void no_keep_alive_store_barrier_on_heap_oop_field(volatile zpointer* p);
183 };
184 
185 #endif // SHARE_GC_Z_ZBARRIER_HPP