1 /*
  2  * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef SHARE_OOPS_ACCESS_INLINE_HPP
 26 #define SHARE_OOPS_ACCESS_INLINE_HPP
 27 
 28 #include "oops/access.hpp"
 29 
 30 #include "gc/shared/barrierSet.inline.hpp"
 31 #include "gc/shared/barrierSetConfig.inline.hpp"
 32 #include "oops/accessBackend.inline.hpp"
 33 
 34 // This file outlines the last 2 steps of the template pipeline of accesses going through
 35 // the Access API.
 36 // * Step 5.a: Barrier resolution. This step is invoked the first time a runtime-dispatch
 37 //             happens for an access. The appropriate BarrierSet::AccessBarrier accessor
 38 //             is resolved, then the function pointer is updated to that accessor for
 39 //             future invocations.
 40 // * Step 5.b: Post-runtime dispatch. This step now casts previously unknown types such
 41 //             as the address type of an oop on the heap (is it oop* or narrowOop*) to
 42 //             the appropriate type. It also splits sufficiently orthogonal accesses into
 43 //             different functions, such as whether the access involves oops or primitives
 44 //             and whether the access is performed on the heap or outside. Then the
 45 //             appropriate BarrierSet::AccessBarrier is called to perform the access.
 46 
 47 namespace AccessInternal {
 48   // Step 5.b: Post-runtime dispatch.
 49   // This class is the last step before calling the BarrierSet::AccessBarrier.
 50   // Here we make sure to figure out types that were not known prior to the
 51   // runtime dispatch, such as whether an oop on the heap is oop or narrowOop.
 52   // We also split orthogonal barriers such as handling primitives vs oops
 53   // and on-heap vs off-heap into different calls to the barrier set.
 54   template <class GCBarrierType, BarrierType type, DecoratorSet decorators>
 55   struct PostRuntimeDispatch: public AllStatic { };
 56 
 57   template <class GCBarrierType, DecoratorSet decorators>
 58   struct PostRuntimeDispatch<GCBarrierType, BARRIER_STORE, decorators>: public AllStatic {
 59     template <typename T>
 60     static void access_barrier(void* addr, T value) {
 61       GCBarrierType::store_in_heap(reinterpret_cast<T*>(addr), value);
 62     }
 63 
 64     static void oop_access_barrier(void* addr, oop value) {
 65       typedef typename HeapOopType<decorators>::type OopType;
 66       if (HasDecorator<decorators, IN_HEAP>::value) {
 67         GCBarrierType::oop_store_in_heap(reinterpret_cast<OopType*>(addr), value);
 68       } else {
 69         GCBarrierType::oop_store_not_in_heap(reinterpret_cast<OopType*>(addr), value);
 70       }
 71     }
 72   };
 73 
 74   template <class GCBarrierType, DecoratorSet decorators>
 75   struct PostRuntimeDispatch<GCBarrierType, BARRIER_LOAD, decorators>: public AllStatic {
 76     template <typename T>
 77     static T access_barrier(void* addr) {
 78       return GCBarrierType::load_in_heap(reinterpret_cast<T*>(addr));
 79     }
 80 
 81     static oop oop_access_barrier(void* addr) {
 82       typedef typename HeapOopType<decorators>::type OopType;
 83       if (HasDecorator<decorators, IN_HEAP>::value) {
 84         return GCBarrierType::oop_load_in_heap(reinterpret_cast<OopType*>(addr));
 85       } else {
 86         return GCBarrierType::oop_load_not_in_heap(reinterpret_cast<OopType*>(addr));
 87       }
 88     }
 89   };
 90 
 91   template <class GCBarrierType, DecoratorSet decorators>
 92   struct PostRuntimeDispatch<GCBarrierType, BARRIER_ATOMIC_XCHG, decorators>: public AllStatic {
 93     template <typename T>
 94     static T access_barrier(void* addr, T new_value) {
 95       return GCBarrierType::atomic_xchg_in_heap(reinterpret_cast<T*>(addr), new_value);
 96     }
 97 
 98     static oop oop_access_barrier(void* addr, oop new_value) {
 99       typedef typename HeapOopType<decorators>::type OopType;
100       if (HasDecorator<decorators, IN_HEAP>::value) {
101         return GCBarrierType::oop_atomic_xchg_in_heap(reinterpret_cast<OopType*>(addr), new_value);
102       } else {
103         return GCBarrierType::oop_atomic_xchg_not_in_heap(reinterpret_cast<OopType*>(addr), new_value);
104       }
105     }
106   };
107 
108   template <class GCBarrierType, DecoratorSet decorators>
109   struct PostRuntimeDispatch<GCBarrierType, BARRIER_ATOMIC_CMPXCHG, decorators>: public AllStatic {
110     template <typename T>
111     static T access_barrier(void* addr, T compare_value, T new_value) {
112       return GCBarrierType::atomic_cmpxchg_in_heap(reinterpret_cast<T*>(addr), compare_value, new_value);
113     }
114 
115     static oop oop_access_barrier(void* addr, oop compare_value, oop new_value) {
116       typedef typename HeapOopType<decorators>::type OopType;
117       if (HasDecorator<decorators, IN_HEAP>::value) {
118         return GCBarrierType::oop_atomic_cmpxchg_in_heap(reinterpret_cast<OopType*>(addr), compare_value, new_value);
119       } else {
120         return GCBarrierType::oop_atomic_cmpxchg_not_in_heap(reinterpret_cast<OopType*>(addr), compare_value, new_value);
121       }
122     }
123   };
124 
125   template <class GCBarrierType, DecoratorSet decorators>
126   struct PostRuntimeDispatch<GCBarrierType, BARRIER_ARRAYCOPY, decorators>: public AllStatic {
127     template <typename T>
128     static void access_barrier(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
129                                arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
130                                size_t length) {
131       GCBarrierType::arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw,
132                                        dst_obj, dst_offset_in_bytes, dst_raw,
133                                        length);
134     }
135 
136     template <typename T>
137     static void oop_access_barrier(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
138                                    arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
139                                    size_t length) {
140       typedef typename HeapOopType<decorators>::type OopType;
141       GCBarrierType::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, reinterpret_cast<OopType*>(src_raw),
142                                            dst_obj, dst_offset_in_bytes, reinterpret_cast<OopType*>(dst_raw),
143                                            length);
144     }
145   };
146 
147   template <class GCBarrierType, DecoratorSet decorators>
148   struct PostRuntimeDispatch<GCBarrierType, BARRIER_STORE_AT, decorators>: public AllStatic {
149     template <typename T>
150     static void access_barrier(oop base, ptrdiff_t offset, T value) {
151       GCBarrierType::store_in_heap_at(base, offset, value);
152     }
153 
154     static void oop_access_barrier(oop base, ptrdiff_t offset, oop value) {
155       GCBarrierType::oop_store_in_heap_at(base, offset, value);
156     }
157   };
158 
159   template <class GCBarrierType, DecoratorSet decorators>
160   struct PostRuntimeDispatch<GCBarrierType, BARRIER_LOAD_AT, decorators>: public AllStatic {
161     template <typename T>
162     static T access_barrier(oop base, ptrdiff_t offset) {
163       return GCBarrierType::template load_in_heap_at<T>(base, offset);
164     }
165 
166     static oop oop_access_barrier(oop base, ptrdiff_t offset) {
167       return GCBarrierType::oop_load_in_heap_at(base, offset);
168     }
169   };
170 
171   template <class GCBarrierType, DecoratorSet decorators>
172   struct PostRuntimeDispatch<GCBarrierType, BARRIER_ATOMIC_XCHG_AT, decorators>: public AllStatic {
173     template <typename T>
174     static T access_barrier(oop base, ptrdiff_t offset, T new_value) {
175       return GCBarrierType::atomic_xchg_in_heap_at(base, offset, new_value);
176     }
177 
178     static oop oop_access_barrier(oop base, ptrdiff_t offset, oop new_value) {
179       return GCBarrierType::oop_atomic_xchg_in_heap_at(base, offset, new_value);
180     }
181   };
182 
183   template <class GCBarrierType, DecoratorSet decorators>
184   struct PostRuntimeDispatch<GCBarrierType, BARRIER_ATOMIC_CMPXCHG_AT, decorators>: public AllStatic {
185     template <typename T>
186     static T access_barrier(oop base, ptrdiff_t offset, T compare_value, T new_value) {
187       return GCBarrierType::atomic_cmpxchg_in_heap_at(base, offset, compare_value, new_value);
188     }
189 
190     static oop oop_access_barrier(oop base, ptrdiff_t offset, oop compare_value, oop new_value) {
191       return GCBarrierType::oop_atomic_cmpxchg_in_heap_at(base, offset, compare_value, new_value);
192     }
193   };
194 
195   template <class GCBarrierType, DecoratorSet decorators>
196   struct PostRuntimeDispatch<GCBarrierType, BARRIER_CLONE, decorators>: public AllStatic {
197     static void access_barrier(oop src, oop dst, size_t size) {
198       GCBarrierType::clone_in_heap(src, dst, size);
199     }
200   };
201 
202   template <class GCBarrierType, DecoratorSet decorators>
203   struct PostRuntimeDispatch<GCBarrierType, BARRIER_VALUE_COPY, decorators>: public AllStatic {
204     static void access_barrier(void* src, void* dst, InlineKlass* md) {
205       GCBarrierType::value_copy_in_heap(src, dst, md);
206     }
207   };
208 
209   // Resolving accessors with barriers from the barrier set happens in two steps.
210   // 1. Expand paths with runtime-decorators, e.g. is UseCompressedOops on or off.
211   // 2. Expand paths for each BarrierSet available in the system.
212   template <DecoratorSet decorators, typename FunctionPointerT, BarrierType barrier_type>
213   struct BarrierResolver: public AllStatic {
214     template <DecoratorSet ds>
215     static typename EnableIf<
216       HasDecorator<ds, INTERNAL_VALUE_IS_OOP>::value,
217       FunctionPointerT>::type
218     resolve_barrier_gc() {
219       BarrierSet* bs = BarrierSet::barrier_set();
220       assert(bs != NULL, "GC barriers invoked before BarrierSet is set");
221       switch (bs->kind()) {
222 #define BARRIER_SET_RESOLVE_BARRIER_CLOSURE(bs_name)                    \
223         case BarrierSet::bs_name: {                                     \
224           return PostRuntimeDispatch<typename BarrierSet::GetType<BarrierSet::bs_name>::type:: \
225             AccessBarrier<ds>, barrier_type, ds>::oop_access_barrier; \
226         }                                                               \
227         break;
228         FOR_EACH_CONCRETE_BARRIER_SET_DO(BARRIER_SET_RESOLVE_BARRIER_CLOSURE)
229 #undef BARRIER_SET_RESOLVE_BARRIER_CLOSURE
230 
231       default:
232         fatal("BarrierSet AccessBarrier resolving not implemented");
233         return NULL;
234       };
235     }
236 
237     template <DecoratorSet ds>
238     static typename EnableIf<
239       !HasDecorator<ds, INTERNAL_VALUE_IS_OOP>::value,
240       FunctionPointerT>::type
241     resolve_barrier_gc() {
242       BarrierSet* bs = BarrierSet::barrier_set();
243       assert(bs != NULL, "GC barriers invoked before BarrierSet is set");
244       switch (bs->kind()) {
245 #define BARRIER_SET_RESOLVE_BARRIER_CLOSURE(bs_name)                    \
246         case BarrierSet::bs_name: {                                       \
247           return PostRuntimeDispatch<typename BarrierSet::GetType<BarrierSet::bs_name>::type:: \
248             AccessBarrier<ds>, barrier_type, ds>::access_barrier; \
249         }                                                                 \
250         break;
251         FOR_EACH_CONCRETE_BARRIER_SET_DO(BARRIER_SET_RESOLVE_BARRIER_CLOSURE)
252 #undef BARRIER_SET_RESOLVE_BARRIER_CLOSURE
253 
254       default:
255         fatal("BarrierSet AccessBarrier resolving not implemented");
256         return NULL;
257       };
258     }
259 
260     static FunctionPointerT resolve_barrier_rt() {
261       if (UseCompressedOops) {
262         const DecoratorSet expanded_decorators = decorators | INTERNAL_RT_USE_COMPRESSED_OOPS;
263         return resolve_barrier_gc<expanded_decorators>();
264       } else {
265         return resolve_barrier_gc<decorators>();
266       }
267     }
268 
269     static FunctionPointerT resolve_barrier() {
270       return resolve_barrier_rt();
271     }
272   };
273 
274   // Step 5.a: Barrier resolution
275   // The RuntimeDispatch class is responsible for performing a runtime dispatch of the
276   // accessor. This is required when the access either depends on whether compressed oops
277   // is being used, or it depends on which GC implementation was chosen (e.g. requires GC
278   // barriers). The way it works is that a function pointer initially pointing to an
279   // accessor resolution function gets called for each access. Upon first invocation,
280   // it resolves which accessor to be used in future invocations and patches the
281   // function pointer to this new accessor.
282 
283   template <DecoratorSet decorators, typename T>
284   void RuntimeDispatch<decorators, T, BARRIER_STORE>::store_init(void* addr, T value) {
285     func_t function = BarrierResolver<decorators, func_t, BARRIER_STORE>::resolve_barrier();
286     _store_func = function;
287     function(addr, value);
288   }
289 
290   template <DecoratorSet decorators, typename T>
291   void RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::store_at_init(oop base, ptrdiff_t offset, T value) {
292     func_t function = BarrierResolver<decorators, func_t, BARRIER_STORE_AT>::resolve_barrier();
293     _store_at_func = function;
294     function(base, offset, value);
295   }
296 
297   template <DecoratorSet decorators, typename T>
298   T RuntimeDispatch<decorators, T, BARRIER_LOAD>::load_init(void* addr) {
299     func_t function = BarrierResolver<decorators, func_t, BARRIER_LOAD>::resolve_barrier();
300     _load_func = function;
301     return function(addr);
302   }
303 
304   template <DecoratorSet decorators, typename T>
305   T RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::load_at_init(oop base, ptrdiff_t offset) {
306     func_t function = BarrierResolver<decorators, func_t, BARRIER_LOAD_AT>::resolve_barrier();
307     _load_at_func = function;
308     return function(base, offset);
309   }
310 
311   template <DecoratorSet decorators, typename T>
312   T RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::atomic_cmpxchg_init(void* addr, T compare_value, T new_value) {
313     func_t function = BarrierResolver<decorators, func_t, BARRIER_ATOMIC_CMPXCHG>::resolve_barrier();
314     _atomic_cmpxchg_func = function;
315     return function(addr, compare_value, new_value);
316   }
317 
318   template <DecoratorSet decorators, typename T>
319   T RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::atomic_cmpxchg_at_init(oop base, ptrdiff_t offset, T compare_value, T new_value) {
320     func_t function = BarrierResolver<decorators, func_t, BARRIER_ATOMIC_CMPXCHG_AT>::resolve_barrier();
321     _atomic_cmpxchg_at_func = function;
322     return function(base, offset, compare_value, new_value);
323   }
324 
325   template <DecoratorSet decorators, typename T>
326   T RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::atomic_xchg_init(void* addr, T new_value) {
327     func_t function = BarrierResolver<decorators, func_t, BARRIER_ATOMIC_XCHG>::resolve_barrier();
328     _atomic_xchg_func = function;
329     return function(addr, new_value);
330   }
331 
332   template <DecoratorSet decorators, typename T>
333   T RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at_init(oop base, ptrdiff_t offset, T new_value) {
334     func_t function = BarrierResolver<decorators, func_t, BARRIER_ATOMIC_XCHG_AT>::resolve_barrier();
335     _atomic_xchg_at_func = function;
336     return function(base, offset, new_value);
337   }
338 
339   template <DecoratorSet decorators, typename T>
340   void RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
341                                                                          arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
342                                                                          size_t length) {
343     func_t function = BarrierResolver<decorators, func_t, BARRIER_ARRAYCOPY>::resolve_barrier();
344     _arraycopy_func = function;
345     function(src_obj, src_offset_in_bytes, src_raw,
346              dst_obj, dst_offset_in_bytes, dst_raw,
347              length);
348   }
349 
350   template <DecoratorSet decorators, typename T>
351   void RuntimeDispatch<decorators, T, BARRIER_CLONE>::clone_init(oop src, oop dst, size_t size) {
352     func_t function = BarrierResolver<decorators, func_t, BARRIER_CLONE>::resolve_barrier();
353     _clone_func = function;
354     function(src, dst, size);
355   }
356 
357   template <DecoratorSet decorators, typename T>
358   void RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>::value_copy_init(void* src, void* dst, InlineKlass* md) {
359     func_t function = BarrierResolver<decorators, func_t, BARRIER_VALUE_COPY>::resolve_barrier();
360     _value_copy_func = function;
361     function(src, dst, md);
362   }
363 }
364 
365 #endif // SHARE_OOPS_ACCESS_INLINE_HPP