1 /*
  2  * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
 26 #define SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
 27 
 28 #include "oops/accessBackend.hpp"
 29 
 30 #include "oops/access.hpp"
 31 #include "oops/arrayOop.hpp"
 32 #include "oops/compressedOops.inline.hpp"
 33 #include "oops/oopsHierarchy.hpp"
 34 #include "runtime/atomic.hpp"
 35 #include "runtime/orderAccess.hpp"
 36 
 37 template <DecoratorSet decorators>
 38 template <DecoratorSet idecorators, typename T>
 39 inline typename EnableIf<
 40   AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
 41 RawAccessBarrier<decorators>::decode_internal(typename HeapOopType<idecorators>::type value) {
 42   if (HasDecorator<decorators, IS_NOT_NULL>::value) {
 43     return CompressedOops::decode_not_null(value);
 44   } else {
 45     return CompressedOops::decode(value);
 46   }
 47 }
 48 
 49 template <DecoratorSet decorators>
 50 template <DecoratorSet idecorators, typename T>
 51 inline typename EnableIf<
 52   AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
 53   typename HeapOopType<idecorators>::type>::type
 54 RawAccessBarrier<decorators>::encode_internal(T value) {
 55   if (HasDecorator<decorators, IS_NOT_NULL>::value) {
 56     return CompressedOops::encode_not_null(value);
 57   } else {
 58     return CompressedOops::encode(value);
 59   }
 60 }
 61 
 62 template <DecoratorSet decorators>
 63 template <typename T>
 64 inline void RawAccessBarrier<decorators>::oop_store(void* addr, T value) {
 65   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
 66   Encoded encoded = encode(value);
 67   store(reinterpret_cast<Encoded*>(addr), encoded);
 68 }
 69 
 70 template <DecoratorSet decorators>
 71 template <typename T>
 72 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) {
 73   oop_store(field_addr(base, offset), value);
 74 }
 75 
 76 template <DecoratorSet decorators>
 77 template <typename T>
 78 inline T RawAccessBarrier<decorators>::oop_load(void* addr) {
 79   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
 80   Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr));
 81   return decode<T>(encoded);
 82 }
 83 
 84 template <DecoratorSet decorators>
 85 template <typename T>
 86 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) {
 87   return oop_load<T>(field_addr(base, offset));
 88 }
 89 
 90 template <DecoratorSet decorators>
 91 template <typename T>
 92 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 93   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
 94   Encoded encoded_new = encode(new_value);
 95   Encoded encoded_compare = encode(compare_value);
 96   Encoded encoded_result = atomic_cmpxchg(reinterpret_cast<Encoded*>(addr),
 97                                           encoded_compare,
 98                                           encoded_new);
 99   return decode<T>(encoded_result);
100 }
101 
102 template <DecoratorSet decorators>
103 template <typename T>
104 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
105   return oop_atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
106 }
107 
108 template <DecoratorSet decorators>
109 template <typename T>
110 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(void* addr, T new_value) {
111   typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
112   Encoded encoded_new = encode(new_value);
113   Encoded encoded_result = atomic_xchg(reinterpret_cast<Encoded*>(addr), encoded_new);
114   return decode<T>(encoded_result);
115 }
116 
117 template <DecoratorSet decorators>
118 template <typename T>
119 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
120   return oop_atomic_xchg(field_addr(base, offset), new_value);
121 }
122 
123 template <DecoratorSet decorators>
124 template <typename T>
125 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
126                                                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
127                                                         size_t length) {
128   return arraycopy(src_obj, src_offset_in_bytes, src_raw,
129                    dst_obj, dst_offset_in_bytes, dst_raw,
130                    length);
131 }
132 
133 template <DecoratorSet decorators>
134 template <DecoratorSet ds, typename T>
135 inline typename EnableIf<
136   HasDecorator<ds, MO_SEQ_CST>::value, T>::type
137 RawAccessBarrier<decorators>::load_internal(void* addr) {
138   if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
139     OrderAccess::fence();
140   }
141   return Atomic::load_acquire(reinterpret_cast<const volatile T*>(addr));
142 }
143 
144 template <DecoratorSet decorators>
145 template <DecoratorSet ds, typename T>
146 inline typename EnableIf<
147   HasDecorator<ds, MO_ACQUIRE>::value, T>::type
148 RawAccessBarrier<decorators>::load_internal(void* addr) {
149   return Atomic::load_acquire(reinterpret_cast<const volatile T*>(addr));
150 }
151 
152 template <DecoratorSet decorators>
153 template <DecoratorSet ds, typename T>
154 inline typename EnableIf<
155   HasDecorator<ds, MO_RELAXED>::value, T>::type
156 RawAccessBarrier<decorators>::load_internal(void* addr) {
157   return Atomic::load(reinterpret_cast<const volatile T*>(addr));
158 }
159 
160 template <DecoratorSet decorators>
161 template <DecoratorSet ds, typename T>
162 inline typename EnableIf<
163   HasDecorator<ds, MO_SEQ_CST>::value>::type
164 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
165   Atomic::release_store_fence(reinterpret_cast<volatile T*>(addr), value);
166 }
167 
168 template <DecoratorSet decorators>
169 template <DecoratorSet ds, typename T>
170 inline typename EnableIf<
171   HasDecorator<ds, MO_RELEASE>::value>::type
172 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
173   Atomic::release_store(reinterpret_cast<volatile T*>(addr), value);
174 }
175 
176 template <DecoratorSet decorators>
177 template <DecoratorSet ds, typename T>
178 inline typename EnableIf<
179   HasDecorator<ds, MO_RELAXED>::value>::type
180 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
181   Atomic::store(reinterpret_cast<volatile T*>(addr), value);
182 }
183 
184 template <DecoratorSet decorators>
185 template <DecoratorSet ds, typename T>
186 inline typename EnableIf<
187   HasDecorator<ds, MO_RELAXED>::value, T>::type
188 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
189   return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr),
190                          compare_value,
191                          new_value,
192                          memory_order_relaxed);
193 }
194 
195 template <DecoratorSet decorators>
196 template <DecoratorSet ds, typename T>
197 inline typename EnableIf<
198   HasDecorator<ds, MO_SEQ_CST>::value, T>::type
199 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
200   return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr),
201                          compare_value,
202                          new_value,
203                          memory_order_conservative);
204 }
205 
206 template <DecoratorSet decorators>
207 template <DecoratorSet ds, typename T>
208 inline typename EnableIf<
209   HasDecorator<ds, MO_SEQ_CST>::value, T>::type
210 RawAccessBarrier<decorators>::atomic_xchg_internal(void* addr, T new_value) {
211   return Atomic::xchg(reinterpret_cast<volatile T*>(addr),
212                       new_value);
213 }
214 
215 // For platforms that do not have native support for wide atomics,
216 // we can emulate the atomicity using a lock. So here we check
217 // whether that is necessary or not.
218 
219 template <DecoratorSet ds>
220 template <DecoratorSet decorators, typename T>
221 inline typename EnableIf<
222   AccessInternal::PossiblyLockedAccess<T>::value, T>::type
223 RawAccessBarrier<ds>::atomic_xchg_maybe_locked(void* addr, T new_value) {
224   if (!AccessInternal::wide_atomic_needs_locking()) {
225     return atomic_xchg_internal<ds>(addr, new_value);
226   } else {
227     AccessInternal::AccessLocker access_lock;
228     volatile T* p = reinterpret_cast<volatile T*>(addr);
229     T old_val = RawAccess<>::load(p);
230     RawAccess<>::store(p, new_value);
231     return old_val;
232   }
233 }
234 
235 template <DecoratorSet ds>
236 template <DecoratorSet decorators, typename T>
237 inline typename EnableIf<
238   AccessInternal::PossiblyLockedAccess<T>::value, T>::type
239 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(void* addr, T compare_value, T new_value) {
240   if (!AccessInternal::wide_atomic_needs_locking()) {
241     return atomic_cmpxchg_internal<ds>(addr, compare_value, new_value);
242   } else {
243     AccessInternal::AccessLocker access_lock;
244     volatile T* p = reinterpret_cast<volatile T*>(addr);
245     T old_val = RawAccess<>::load(p);
246     if (old_val == compare_value) {
247       RawAccess<>::store(p, new_value);
248     }
249     return old_val;
250   }
251 }
252 
253 class RawAccessBarrierArrayCopy: public AllStatic {
254   template<typename T> struct IsHeapWordSized: public IntegralConstant<bool, sizeof(T) == HeapWordSize> { };
255 public:
256   template <DecoratorSet decorators, typename T>
257   static inline typename EnableIf<
258     HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
259   arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
260             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
261             size_t length) {
262     src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
263     dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
264 
265     // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic.
266     if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
267       AccessInternal::arraycopy_arrayof_conjoint_oops(src_raw, dst_raw, length);
268     } else {
269       typedef typename HeapOopType<decorators>::type OopType;
270       AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src_raw),
271                                               reinterpret_cast<OopType*>(dst_raw), length);
272     }
273   }
274 
275   template <DecoratorSet decorators, typename T>
276   static inline typename EnableIf<
277     !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
278     HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value>::type
279   arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
280             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
281             size_t length) {
282     src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
283     dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
284 
285     AccessInternal::arraycopy_arrayof_conjoint(src_raw, dst_raw, length);
286   }
287 
288   template <DecoratorSet decorators, typename T>
289   static inline typename EnableIf<
290     !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
291     HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value>::type
292   arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
293             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
294             size_t length) {
295     src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
296     dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
297 
298     // There is only a disjoint optimization for word granularity copying
299     if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
300       AccessInternal::arraycopy_disjoint_words_atomic(src_raw, dst_raw, length);
301     } else {
302       AccessInternal::arraycopy_disjoint_words(src_raw, dst_raw, length);
303     }
304   }
305 
306   template <DecoratorSet decorators, typename T>
307   static inline typename EnableIf<
308     !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
309     !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
310     !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
311     !HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
312   arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
313             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
314             size_t length) {
315     src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
316     dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
317 
318     AccessInternal::arraycopy_conjoint(src_raw, dst_raw, length);
319   }
320 
321   template <DecoratorSet decorators, typename T>
322   static inline typename EnableIf<
323     !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
324     !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
325     !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
326     HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
327   arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
328             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
329             size_t length) {
330     src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
331     dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
332 
333     AccessInternal::arraycopy_conjoint_atomic(src_raw, dst_raw, length);
334   }
335 };
336 
337 template<> struct RawAccessBarrierArrayCopy::IsHeapWordSized<void>: public IntegralConstant<bool, false> { };
338 
339 template <DecoratorSet decorators>
340 template <typename T>
341 inline bool RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
342                                                     arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
343                                                     size_t length) {
344   RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
345                                                    dst_obj, dst_offset_in_bytes, dst_raw,
346                                                    length);
347   return true;
348 }
349 
350 template <DecoratorSet decorators>
351 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
352   // 4839641 (4840070): We must do an oop-atomic copy, because if another thread
353   // is modifying a reference field in the clonee, a non-oop-atomic copy might
354   // be suspended in the middle of copying the pointer and end up with parts
355   // of two different pointers in the field.  Subsequent dereferences will crash.
356   // 4846409: an oop-copy of objects with long or double fields or arrays of same
357   // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead
358   // of oops.  We know objects are aligned on a minimum of an jlong boundary.
359   // The same is true of StubRoutines::object_copy and the various oop_copy
360   // variants, and of the code generated by the inline_native_clone intrinsic.
361 
362   assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
363   AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src),
364                                             reinterpret_cast<jlong*>((oopDesc*)dst),
365                                             align_object_size(size) / HeapWordsPerLong);
366   // Clear the header
367   dst->init_mark();
368 }
369 
370 #endif // SHARE_OOPS_ACCESSBACKEND_INLINE_HPP