1 /*
  2  * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
 26 #define SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
 27 
 28 #include "oops/accessBackend.hpp"
 29 
 30 #include "oops/access.hpp"
 31 #include "oops/arrayOop.hpp"
 32 #include "oops/compressedOops.inline.hpp"
 33 #include "oops/oopsHierarchy.hpp"
 34 #include "runtime/atomic.hpp"
 35 #include "runtime/orderAccess.hpp"
 36 #include "oops/inlineKlass.hpp"
 37 
 38 #include <type_traits>
 39 
 40 template <DecoratorSet decorators>
 41 template <DecoratorSet idecorators, typename T>
 42 inline typename EnableIf<
 43   AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
 44 RawAccessBarrier<decorators>::decode_internal(typename HeapOopType<idecorators>::type value) {
 45   if (HasDecorator<decorators, IS_NOT_NULL>::value) {
 46     return CompressedOops::decode_not_null(value);
 47   } else {
 48     return CompressedOops::decode(value);
 49   }
 50 }
 51 
 52 template <DecoratorSet decorators>
 53 template <DecoratorSet idecorators, typename T>
 54 inline typename EnableIf<
 55   AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
 56   typename HeapOopType<idecorators>::type>::type
 57 RawAccessBarrier<decorators>::encode_internal(T value) {
 58   if (HasDecorator<decorators, IS_NOT_NULL>::value) {
 59     return CompressedOops::encode_not_null(value);
 60   } else {
 61     return CompressedOops::encode(value);
 62   }
 63 }
 64 
 65 template <DecoratorSet decorators>
 66 template <typename T>
 67 inline void RawAccessBarrier<decorators>::oop_store(void* addr, T value) {
 68   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
 69   Encoded encoded = encode(value);
 70   store(reinterpret_cast<Encoded*>(addr), encoded);
 71 }
 72 
 73 template <DecoratorSet decorators>
 74 template <typename T>
 75 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) {
 76   oop_store(field_addr(base, offset), value);
 77 }
 78 
 79 template <DecoratorSet decorators>
 80 template <typename T>
 81 inline T RawAccessBarrier<decorators>::oop_load(void* addr) {
 82   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
 83   Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr));
 84   return decode<T>(encoded);
 85 }
 86 
 87 template <DecoratorSet decorators>
 88 template <typename T>
 89 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) {
 90   return oop_load<T>(field_addr(base, offset));
 91 }
 92 
 93 template <DecoratorSet decorators>
 94 template <typename T>
 95 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 96   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
 97   Encoded encoded_new = encode(new_value);
 98   Encoded encoded_compare = encode(compare_value);
 99   Encoded encoded_result = atomic_cmpxchg(reinterpret_cast<Encoded*>(addr),
100                                           encoded_compare,
101                                           encoded_new);
102   return decode<T>(encoded_result);
103 }
104 
105 template <DecoratorSet decorators>
106 template <typename T>
107 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
108   return oop_atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
109 }
110 
111 template <DecoratorSet decorators>
112 template <typename T>
113 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(void* addr, T new_value) {
114   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
115   Encoded encoded_new = encode(new_value);
116   Encoded encoded_result = atomic_xchg(reinterpret_cast<Encoded*>(addr), encoded_new);
117   return decode<T>(encoded_result);
118 }
119 
120 template <DecoratorSet decorators>
121 template <typename T>
122 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
123   return oop_atomic_xchg(field_addr(base, offset), new_value);
124 }
125 
126 template <DecoratorSet decorators>
127 template <typename T>
128 inline void RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
129                                                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
130                                                         size_t length) {
131   arraycopy(src_obj, src_offset_in_bytes, src_raw,
132             dst_obj, dst_offset_in_bytes, dst_raw,
133             length);
134 }
135 
136 template <DecoratorSet decorators>
137 template <DecoratorSet ds, typename T>
138 inline typename EnableIf<
139   HasDecorator<ds, MO_SEQ_CST>::value, T>::type
140 RawAccessBarrier<decorators>::load_internal(void* addr) {
141   if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
142     OrderAccess::fence();
143   }
144   return Atomic::load_acquire(reinterpret_cast<const volatile T*>(addr));
145 }
146 
147 template <DecoratorSet decorators>
148 template <DecoratorSet ds, typename T>
149 inline typename EnableIf<
150   HasDecorator<ds, MO_ACQUIRE>::value, T>::type
151 RawAccessBarrier<decorators>::load_internal(void* addr) {
152   return Atomic::load_acquire(reinterpret_cast<const volatile T*>(addr));
153 }
154 
155 template <DecoratorSet decorators>
156 template <DecoratorSet ds, typename T>
157 inline typename EnableIf<
158   HasDecorator<ds, MO_RELAXED>::value, T>::type
159 RawAccessBarrier<decorators>::load_internal(void* addr) {
160   return Atomic::load(reinterpret_cast<const volatile T*>(addr));
161 }
162 
163 template <DecoratorSet decorators>
164 template <DecoratorSet ds, typename T>
165 inline typename EnableIf<
166   HasDecorator<ds, MO_SEQ_CST>::value>::type
167 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
168   Atomic::release_store_fence(reinterpret_cast<volatile T*>(addr), value);
169 }
170 
171 template <DecoratorSet decorators>
172 template <DecoratorSet ds, typename T>
173 inline typename EnableIf<
174   HasDecorator<ds, MO_RELEASE>::value>::type
175 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
176   Atomic::release_store(reinterpret_cast<volatile T*>(addr), value);
177 }
178 
179 template <DecoratorSet decorators>
180 template <DecoratorSet ds, typename T>
181 inline typename EnableIf<
182   HasDecorator<ds, MO_RELAXED>::value>::type
183 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
184   Atomic::store(reinterpret_cast<volatile T*>(addr), value);
185 }
186 
187 template <DecoratorSet decorators>
188 template <DecoratorSet ds, typename T>
189 inline typename EnableIf<
190   HasDecorator<ds, MO_RELAXED>::value, T>::type
191 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
192   return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr),
193                          compare_value,
194                          new_value,
195                          memory_order_relaxed);
196 }
197 
198 template <DecoratorSet decorators>
199 template <DecoratorSet ds, typename T>
200 inline typename EnableIf<
201   HasDecorator<ds, MO_SEQ_CST>::value, T>::type
202 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
203   return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr),
204                          compare_value,
205                          new_value,
206                          memory_order_conservative);
207 }
208 
209 template <DecoratorSet decorators>
210 template <DecoratorSet ds, typename T>
211 inline typename EnableIf<
212   HasDecorator<ds, MO_SEQ_CST>::value, T>::type
213 RawAccessBarrier<decorators>::atomic_xchg_internal(void* addr, T new_value) {
214   return Atomic::xchg(reinterpret_cast<volatile T*>(addr),
215                       new_value);
216 }
217 
218 // For platforms that do not have native support for wide atomics,
219 // we can emulate the atomicity using a lock. So here we check
220 // whether that is necessary or not.
221 
222 template <DecoratorSet ds>
223 template <DecoratorSet decorators, typename T>
224 inline typename EnableIf<
225   AccessInternal::PossiblyLockedAccess<T>::value, T>::type
226 RawAccessBarrier<ds>::atomic_xchg_maybe_locked(void* addr, T new_value) {
227   if (!AccessInternal::wide_atomic_needs_locking()) {
228     return atomic_xchg_internal<ds>(addr, new_value);
229   } else {
230     AccessInternal::AccessLocker access_lock;
231     volatile T* p = reinterpret_cast<volatile T*>(addr);
232     T old_val = RawAccess<>::load(p);
233     RawAccess<>::store(p, new_value);
234     return old_val;
235   }
236 }
237 
238 template <DecoratorSet ds>
239 template <DecoratorSet decorators, typename T>
240 inline typename EnableIf<
241   AccessInternal::PossiblyLockedAccess<T>::value, T>::type
242 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(void* addr, T compare_value, T new_value) {
243   if (!AccessInternal::wide_atomic_needs_locking()) {
244     return atomic_cmpxchg_internal<ds>(addr, compare_value, new_value);
245   } else {
246     AccessInternal::AccessLocker access_lock;
247     volatile T* p = reinterpret_cast<volatile T*>(addr);
248     T old_val = RawAccess<>::load(p);
249     if (old_val == compare_value) {
250       RawAccess<>::store(p, new_value);
251     }
252     return old_val;
253   }
254 }
255 
256 class RawAccessBarrierArrayCopy: public AllStatic {
257   template<typename T> struct IsHeapWordSized: public std::integral_constant<bool, sizeof(T) == HeapWordSize> { };
258 public:
259   template <DecoratorSet decorators, typename T>
260   static inline typename EnableIf<
261     HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
262   arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
263             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
264             size_t length) {
265     src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
266     dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
267 
268     // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic.
269     if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
270       AccessInternal::arraycopy_arrayof_conjoint_oops(src_raw, dst_raw, length);
271     } else {
272       typedef typename HeapOopType<decorators>::type OopType;
273       AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src_raw),
274                                               reinterpret_cast<OopType*>(dst_raw), length);
275     }
276   }
277 
278   template <DecoratorSet decorators, typename T>
279   static inline typename EnableIf<
280     !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
281     HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value>::type
282   arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
283             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
284             size_t length) {
285     src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
286     dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
287 
288     AccessInternal::arraycopy_arrayof_conjoint(src_raw, dst_raw, length);
289   }
290 
291   template <DecoratorSet decorators, typename T>
292   static inline typename EnableIf<
293     !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
294     HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value>::type
295   arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
296             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
297             size_t length) {
298     src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
299     dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
300 
301     // There is only a disjoint optimization for word granularity copying
302     if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
303       AccessInternal::arraycopy_disjoint_words_atomic(src_raw, dst_raw, length);
304     } else {
305       AccessInternal::arraycopy_disjoint_words(src_raw, dst_raw, length);
306     }
307   }
308 
309   template <DecoratorSet decorators, typename T>
310   static inline typename EnableIf<
311     !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
312     !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
313     !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
314     !HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
315   arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
316             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
317             size_t length) {
318     src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
319     dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
320 
321     AccessInternal::arraycopy_conjoint(src_raw, dst_raw, length);
322   }
323 
324   template <DecoratorSet decorators, typename T>
325   static inline typename EnableIf<
326     !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
327     !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
328     !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
329     HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
330   arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
331             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
332             size_t length) {
333     src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
334     dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
335 
336     AccessInternal::arraycopy_conjoint_atomic(src_raw, dst_raw, length);
337   }
338 };
339 
340 template<> struct RawAccessBarrierArrayCopy::IsHeapWordSized<void>: public std::false_type { };
341 
342 template <DecoratorSet decorators>
343 template <typename T>
344 inline void RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
345                                                     arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
346                                                     size_t length) {
347   RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
348                                                    dst_obj, dst_offset_in_bytes, dst_raw,
349                                                    length);
350 }
351 
352 template <DecoratorSet decorators>
353 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
354   // 4839641 (4840070): We must do an oop-atomic copy, because if another thread
355   // is modifying a reference field in the clonee, a non-oop-atomic copy might
356   // be suspended in the middle of copying the pointer and end up with parts
357   // of two different pointers in the field.  Subsequent dereferences will crash.
358   // 4846409: an oop-copy of objects with long or double fields or arrays of same
359   // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead
360   // of oops.  We know objects are aligned on a minimum of an jlong boundary.
361   // The same is true of StubRoutines::object_copy and the various oop_copy
362   // variants, and of the code generated by the inline_native_clone intrinsic.
363 
364   assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
365   AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src),
366                                             reinterpret_cast<jlong*>((oopDesc*)dst),
367                                             align_object_size(size) / HeapWordsPerLong);
368   // Clear the header
369   dst->init_mark();
370 }
371 
372 template <DecoratorSet decorators>
373 inline void RawAccessBarrier<decorators>::value_copy(void* src, void* dst, InlineKlass* md) {
374   assert(is_aligned(src, md->get_alignment()) && is_aligned(dst, md->get_alignment()), "Unalign value_copy");
375   AccessInternal::arraycopy_conjoint_atomic(src, dst, static_cast<size_t>(md->get_exact_size_in_bytes()));
376 }
377 #endif // SHARE_OOPS_ACCESSBACKEND_INLINE_HPP