1 /*
2 * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
26 #define SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
27
28 #include "oops/accessBackend.hpp"
29
30 #include "cppstdlib/type_traits.hpp"
31 #include "oops/access.hpp"
32 #include "oops/arrayOop.hpp"
33 #include "oops/compressedOops.inline.hpp"
34 #include "oops/inlineKlass.hpp"
35 #include "oops/layoutKind.hpp"
36 #include "oops/oopsHierarchy.hpp"
37 #include "oops/valuePayload.inline.hpp"
38 #include "runtime/atomicAccess.hpp"
39 #include "runtime/orderAccess.hpp"
40
41 template <DecoratorSet decorators>
42 template <DecoratorSet idecorators, typename T>
43 inline typename EnableIf<
44 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
45 RawAccessBarrier<decorators>::decode_internal(typename HeapOopType<idecorators>::type value) {
46 if (HasDecorator<decorators, IS_NOT_NULL>::value) {
47 return CompressedOops::decode_not_null(value);
48 } else {
49 return CompressedOops::decode(value);
50 }
51 }
52
53 template <DecoratorSet decorators>
54 template <DecoratorSet idecorators, typename T>
55 inline typename EnableIf<
56 AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
57 typename HeapOopType<idecorators>::type>::type
58 RawAccessBarrier<decorators>::encode_internal(T value) {
59 if (HasDecorator<decorators, IS_NOT_NULL>::value) {
60 return CompressedOops::encode_not_null(value);
61 } else {
62 return CompressedOops::encode(value);
63 }
64 }
65
66 template <DecoratorSet decorators>
67 template <typename T>
68 inline void RawAccessBarrier<decorators>::oop_store(void* addr, T value) {
69 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
70 Encoded encoded = encode(value);
71 store(reinterpret_cast<Encoded*>(addr), encoded);
72 }
73
74 template <DecoratorSet decorators>
75 template <typename T>
76 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) {
77 oop_store(field_addr(base, offset), value);
78 }
79
80 template <DecoratorSet decorators>
81 template <typename T>
82 inline T RawAccessBarrier<decorators>::oop_load(void* addr) {
83 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
84 Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr));
85 return decode<T>(encoded);
86 }
87
88 template <DecoratorSet decorators>
89 template <typename T>
90 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) {
91 return oop_load<T>(field_addr(base, offset));
92 }
93
94 template <DecoratorSet decorators>
95 template <typename T>
96 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(void* addr, T compare_value, T new_value) {
97 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
98 Encoded encoded_new = encode(new_value);
99 Encoded encoded_compare = encode(compare_value);
100 Encoded encoded_result = atomic_cmpxchg(reinterpret_cast<Encoded*>(addr),
101 encoded_compare,
102 encoded_new);
103 return decode<T>(encoded_result);
104 }
105
106 template <DecoratorSet decorators>
107 template <typename T>
108 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
109 return oop_atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
110 }
111
112 template <DecoratorSet decorators>
113 template <typename T>
114 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(void* addr, T new_value) {
115 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
116 Encoded encoded_new = encode(new_value);
117 Encoded encoded_result = atomic_xchg(reinterpret_cast<Encoded*>(addr), encoded_new);
118 return decode<T>(encoded_result);
119 }
120
121 template <DecoratorSet decorators>
122 template <typename T>
123 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
124 return oop_atomic_xchg(field_addr(base, offset), new_value);
125 }
126
127 template <DecoratorSet decorators>
128 template <typename T>
129 inline void RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
130 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
131 size_t length) {
132 arraycopy(src_obj, src_offset_in_bytes, src_raw,
133 dst_obj, dst_offset_in_bytes, dst_raw,
134 length);
135 }
136
137 template <DecoratorSet decorators>
138 template <DecoratorSet ds, typename T>
139 inline typename EnableIf<
140 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
141 RawAccessBarrier<decorators>::load_internal(void* addr) {
142 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
143 OrderAccess::fence();
144 }
145 return AtomicAccess::load_acquire(reinterpret_cast<const volatile T*>(addr));
146 }
147
148 template <DecoratorSet decorators>
149 template <DecoratorSet ds, typename T>
150 inline typename EnableIf<
151 HasDecorator<ds, MO_ACQUIRE>::value, T>::type
152 RawAccessBarrier<decorators>::load_internal(void* addr) {
153 return AtomicAccess::load_acquire(reinterpret_cast<const volatile T*>(addr));
154 }
155
156 template <DecoratorSet decorators>
157 template <DecoratorSet ds, typename T>
158 inline typename EnableIf<
159 HasDecorator<ds, MO_RELAXED>::value, T>::type
160 RawAccessBarrier<decorators>::load_internal(void* addr) {
161 return AtomicAccess::load(reinterpret_cast<const volatile T*>(addr));
162 }
163
164 template <DecoratorSet decorators>
165 template <DecoratorSet ds, typename T>
166 inline typename EnableIf<
167 HasDecorator<ds, MO_SEQ_CST>::value>::type
168 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
169 AtomicAccess::release_store_fence(reinterpret_cast<volatile T*>(addr), value);
170 }
171
172 template <DecoratorSet decorators>
173 template <DecoratorSet ds, typename T>
174 inline typename EnableIf<
175 HasDecorator<ds, MO_RELEASE>::value>::type
176 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
177 AtomicAccess::release_store(reinterpret_cast<volatile T*>(addr), value);
178 }
179
180 template <DecoratorSet decorators>
181 template <DecoratorSet ds, typename T>
182 inline typename EnableIf<
183 HasDecorator<ds, MO_RELAXED>::value>::type
184 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
185 AtomicAccess::store(reinterpret_cast<volatile T*>(addr), value);
186 }
187
188 template <DecoratorSet decorators>
189 template <DecoratorSet ds, typename T>
190 inline typename EnableIf<
191 HasDecorator<ds, MO_RELAXED>::value, T>::type
192 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
193 return AtomicAccess::cmpxchg(reinterpret_cast<volatile T*>(addr),
194 compare_value,
195 new_value,
196 memory_order_relaxed);
197 }
198
199 template <DecoratorSet decorators>
200 template <DecoratorSet ds, typename T>
201 inline typename EnableIf<
202 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
203 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
204 return AtomicAccess::cmpxchg(reinterpret_cast<volatile T*>(addr),
205 compare_value,
206 new_value,
207 memory_order_conservative);
208 }
209
210 template <DecoratorSet decorators>
211 template <DecoratorSet ds, typename T>
212 inline typename EnableIf<
213 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
214 RawAccessBarrier<decorators>::atomic_xchg_internal(void* addr, T new_value) {
215 return AtomicAccess::xchg(reinterpret_cast<volatile T*>(addr),
216 new_value);
217 }
218
219 class RawAccessBarrierArrayCopy: public AllStatic {
220 template<typename T> struct IsHeapWordSized: public std::integral_constant<bool, sizeof(T) == HeapWordSize> { };
221 public:
222 template <DecoratorSet decorators, typename T>
223 static inline typename EnableIf<
224 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
225 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
226 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
227 size_t length) {
228 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
229 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
230
231 // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic.
232 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
233 AccessInternal::arraycopy_arrayof_conjoint_oops(src_raw, dst_raw, length);
234 } else {
235 typedef typename HeapOopType<decorators>::type OopType;
236 AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src_raw),
237 reinterpret_cast<OopType*>(dst_raw), length);
238 }
239 }
240
241 template <DecoratorSet decorators, typename T>
242 static inline typename EnableIf<
243 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
244 HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value>::type
245 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
246 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
247 size_t length) {
248 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
249 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
250
251 AccessInternal::arraycopy_arrayof_conjoint(src_raw, dst_raw, length);
252 }
253
254 template <DecoratorSet decorators, typename T>
255 static inline typename EnableIf<
256 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
257 HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value>::type
258 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
259 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
260 size_t length) {
261 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
262 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
263
264 // There is only a disjoint optimization for word granularity copying
265 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
266 AccessInternal::arraycopy_disjoint_words_atomic(src_raw, dst_raw, length);
267 } else {
268 AccessInternal::arraycopy_disjoint_words(src_raw, dst_raw, length);
269 }
270 }
271
272 template <DecoratorSet decorators, typename T>
273 static inline typename EnableIf<
274 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
275 !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
276 !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
277 !HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
278 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
279 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
280 size_t length) {
281 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
282 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
283
284 AccessInternal::arraycopy_conjoint(src_raw, dst_raw, length);
285 }
286
287 template <DecoratorSet decorators, typename T>
288 static inline typename EnableIf<
289 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
290 !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
291 !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
292 HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
293 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
294 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
295 size_t length) {
296 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
297 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
298
299 AccessInternal::arraycopy_conjoint_atomic(src_raw, dst_raw, length);
300 }
301 };
302
303 template<> struct RawAccessBarrierArrayCopy::IsHeapWordSized<void>: public std::false_type { };
304
305 template <DecoratorSet decorators>
306 template <typename T>
307 inline void RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
308 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
309 size_t length) {
310 RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
311 dst_obj, dst_offset_in_bytes, dst_raw,
312 length);
313 }
314
315 template <DecoratorSet decorators>
316 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
317 // 4839641 (4840070): We must do an oop-atomic copy, because if another thread
318 // is modifying a reference field in the clonee, a non-oop-atomic copy might
319 // be suspended in the middle of copying the pointer and end up with parts
320 // of two different pointers in the field. Subsequent dereferences will crash.
321 // 4846409: an oop-copy of objects with long or double fields or arrays of same
322 // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead
323 // of oops. We know objects are aligned on a minimum of an jlong boundary.
324 // The same is true of StubRoutines::object_copy and the various oop_copy
325 // variants, and of the code generated by the inline_native_clone intrinsic.
326
327 assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
328 AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src),
329 reinterpret_cast<jlong*>((oopDesc*)dst),
330 align_object_size(size) / HeapWordsPerLong);
331 // Clear the header
332 dst->init_mark();
333 }
334
335 template <DecoratorSet decorators>
336 inline void RawAccessBarrier<decorators>::value_copy(const ValuePayload& src, const ValuePayload& dst) {
337 precond(src.klass() == dst.klass());
338
339 const InlineKlass* klass = src.klass();
340 const LayoutKind copy_layout = LayoutKindHelper::get_copy_layout(
341 src.layout_kind(), dst.layout_kind());
342 const int size = klass->layout_size_in_bytes(copy_layout);
343
344 AccessInternal::value_copy_internal(src.addr(), dst.addr(),
345 static_cast<size_t>(size));
346 }
347
348 template <DecoratorSet decorators>
349 inline void RawAccessBarrier<decorators>::value_store_null(const ValuePayload& dst) {
350 address dst_addr = dst.addr();
351 const LayoutKind lk = dst.layout_kind();
352 const InlineKlass* klass = dst.klass();
353 const int size = klass->layout_size_in_bytes(lk);
354
355 AccessInternal::value_store_null(dst_addr, static_cast<size_t>(size));
356 }
357
358 #endif // SHARE_OOPS_ACCESSBACKEND_INLINE_HPP