1 /*
2 * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
26 #define SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
27
28 #include "oops/accessBackend.hpp"
29
30 #include "oops/access.hpp"
31 #include "oops/arrayOop.hpp"
32 #include "oops/compressedOops.inline.hpp"
33 #include "oops/oopsHierarchy.hpp"
34 #include "runtime/atomic.hpp"
35 #include "runtime/orderAccess.hpp"
36 #include "oops/inlineKlass.hpp"
37
38 template <DecoratorSet decorators>
39 template <DecoratorSet idecorators, typename T>
40 inline typename EnableIf<
41 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
42 RawAccessBarrier<decorators>::decode_internal(typename HeapOopType<idecorators>::type value) {
43 if (HasDecorator<decorators, IS_NOT_NULL>::value) {
44 return CompressedOops::decode_not_null(value);
45 } else {
46 return CompressedOops::decode(value);
47 }
48 }
49
50 template <DecoratorSet decorators>
51 template <DecoratorSet idecorators, typename T>
52 inline typename EnableIf<
53 AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
54 typename HeapOopType<idecorators>::type>::type
55 RawAccessBarrier<decorators>::encode_internal(T value) {
56 if (HasDecorator<decorators, IS_NOT_NULL>::value) {
57 return CompressedOops::encode_not_null(value);
58 } else {
59 return CompressedOops::encode(value);
60 }
61 }
62
63 template <DecoratorSet decorators>
64 template <typename T>
65 inline void RawAccessBarrier<decorators>::oop_store(void* addr, T value) {
66 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
67 Encoded encoded = encode(value);
68 store(reinterpret_cast<Encoded*>(addr), encoded);
69 }
70
71 template <DecoratorSet decorators>
72 template <typename T>
73 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) {
74 oop_store(field_addr(base, offset), value);
75 }
76
77 template <DecoratorSet decorators>
78 template <typename T>
79 inline T RawAccessBarrier<decorators>::oop_load(void* addr) {
80 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
81 Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr));
82 return decode<T>(encoded);
83 }
84
85 template <DecoratorSet decorators>
86 template <typename T>
87 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) {
88 return oop_load<T>(field_addr(base, offset));
89 }
90
91 template <DecoratorSet decorators>
92 template <typename T>
93 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(void* addr, T compare_value, T new_value) {
94 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
95 Encoded encoded_new = encode(new_value);
96 Encoded encoded_compare = encode(compare_value);
97 Encoded encoded_result = atomic_cmpxchg(reinterpret_cast<Encoded*>(addr),
98 encoded_compare,
99 encoded_new);
100 return decode<T>(encoded_result);
101 }
102
103 template <DecoratorSet decorators>
104 template <typename T>
105 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
106 return oop_atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
107 }
108
109 template <DecoratorSet decorators>
110 template <typename T>
111 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(void* addr, T new_value) {
112 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
113 Encoded encoded_new = encode(new_value);
114 Encoded encoded_result = atomic_xchg(reinterpret_cast<Encoded*>(addr), encoded_new);
115 return decode<T>(encoded_result);
116 }
117
118 template <DecoratorSet decorators>
119 template <typename T>
120 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
121 return oop_atomic_xchg(field_addr(base, offset), new_value);
122 }
123
124 template <DecoratorSet decorators>
125 template <typename T>
126 inline void RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
127 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
128 size_t length) {
129 arraycopy(src_obj, src_offset_in_bytes, src_raw,
130 dst_obj, dst_offset_in_bytes, dst_raw,
131 length);
132 }
133
134 template <DecoratorSet decorators>
135 template <DecoratorSet ds, typename T>
136 inline typename EnableIf<
137 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
138 RawAccessBarrier<decorators>::load_internal(void* addr) {
139 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
140 OrderAccess::fence();
141 }
142 return Atomic::load_acquire(reinterpret_cast<const volatile T*>(addr));
143 }
144
145 template <DecoratorSet decorators>
146 template <DecoratorSet ds, typename T>
147 inline typename EnableIf<
148 HasDecorator<ds, MO_ACQUIRE>::value, T>::type
149 RawAccessBarrier<decorators>::load_internal(void* addr) {
150 return Atomic::load_acquire(reinterpret_cast<const volatile T*>(addr));
151 }
152
153 template <DecoratorSet decorators>
154 template <DecoratorSet ds, typename T>
155 inline typename EnableIf<
156 HasDecorator<ds, MO_RELAXED>::value, T>::type
157 RawAccessBarrier<decorators>::load_internal(void* addr) {
158 return Atomic::load(reinterpret_cast<const volatile T*>(addr));
159 }
160
161 template <DecoratorSet decorators>
162 template <DecoratorSet ds, typename T>
163 inline typename EnableIf<
164 HasDecorator<ds, MO_SEQ_CST>::value>::type
165 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
166 Atomic::release_store_fence(reinterpret_cast<volatile T*>(addr), value);
167 }
168
169 template <DecoratorSet decorators>
170 template <DecoratorSet ds, typename T>
171 inline typename EnableIf<
172 HasDecorator<ds, MO_RELEASE>::value>::type
173 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
174 Atomic::release_store(reinterpret_cast<volatile T*>(addr), value);
175 }
176
177 template <DecoratorSet decorators>
178 template <DecoratorSet ds, typename T>
179 inline typename EnableIf<
180 HasDecorator<ds, MO_RELAXED>::value>::type
181 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
182 Atomic::store(reinterpret_cast<volatile T*>(addr), value);
183 }
184
185 template <DecoratorSet decorators>
186 template <DecoratorSet ds, typename T>
187 inline typename EnableIf<
188 HasDecorator<ds, MO_RELAXED>::value, T>::type
189 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
190 return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr),
191 compare_value,
192 new_value,
193 memory_order_relaxed);
194 }
195
196 template <DecoratorSet decorators>
197 template <DecoratorSet ds, typename T>
198 inline typename EnableIf<
199 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
200 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
201 return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr),
202 compare_value,
203 new_value,
204 memory_order_conservative);
205 }
206
207 template <DecoratorSet decorators>
208 template <DecoratorSet ds, typename T>
209 inline typename EnableIf<
210 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
211 RawAccessBarrier<decorators>::atomic_xchg_internal(void* addr, T new_value) {
212 return Atomic::xchg(reinterpret_cast<volatile T*>(addr),
213 new_value);
214 }
215
216 // For platforms that do not have native support for wide atomics,
217 // we can emulate the atomicity using a lock. So here we check
218 // whether that is necessary or not.
219
220 template <DecoratorSet ds>
221 template <DecoratorSet decorators, typename T>
222 inline typename EnableIf<
223 AccessInternal::PossiblyLockedAccess<T>::value, T>::type
224 RawAccessBarrier<ds>::atomic_xchg_maybe_locked(void* addr, T new_value) {
225 if (!AccessInternal::wide_atomic_needs_locking()) {
226 return atomic_xchg_internal<ds>(addr, new_value);
227 } else {
228 AccessInternal::AccessLocker access_lock;
229 volatile T* p = reinterpret_cast<volatile T*>(addr);
230 T old_val = RawAccess<>::load(p);
231 RawAccess<>::store(p, new_value);
232 return old_val;
233 }
234 }
235
236 template <DecoratorSet ds>
237 template <DecoratorSet decorators, typename T>
238 inline typename EnableIf<
239 AccessInternal::PossiblyLockedAccess<T>::value, T>::type
240 RawAccessBarrier<ds>::atomic_cmpxchg_maybe_locked(void* addr, T compare_value, T new_value) {
241 if (!AccessInternal::wide_atomic_needs_locking()) {
242 return atomic_cmpxchg_internal<ds>(addr, compare_value, new_value);
243 } else {
244 AccessInternal::AccessLocker access_lock;
245 volatile T* p = reinterpret_cast<volatile T*>(addr);
246 T old_val = RawAccess<>::load(p);
247 if (old_val == compare_value) {
248 RawAccess<>::store(p, new_value);
249 }
250 return old_val;
251 }
252 }
253
254 class RawAccessBarrierArrayCopy: public AllStatic {
255 template<typename T> struct IsHeapWordSized: public IntegralConstant<bool, sizeof(T) == HeapWordSize> { };
256 public:
257 template <DecoratorSet decorators, typename T>
258 static inline typename EnableIf<
259 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
260 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
261 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
262 size_t length) {
263 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
264 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
265
266 // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic.
267 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
268 AccessInternal::arraycopy_arrayof_conjoint_oops(src_raw, dst_raw, length);
269 } else {
270 typedef typename HeapOopType<decorators>::type OopType;
271 AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src_raw),
272 reinterpret_cast<OopType*>(dst_raw), length);
273 }
274 }
275
276 template <DecoratorSet decorators, typename T>
277 static inline typename EnableIf<
278 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
279 HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value>::type
280 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
281 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
282 size_t length) {
283 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
284 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
285
286 AccessInternal::arraycopy_arrayof_conjoint(src_raw, dst_raw, length);
287 }
288
289 template <DecoratorSet decorators, typename T>
290 static inline typename EnableIf<
291 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
292 HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value>::type
293 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
294 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
295 size_t length) {
296 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
297 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
298
299 // There is only a disjoint optimization for word granularity copying
300 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
301 AccessInternal::arraycopy_disjoint_words_atomic(src_raw, dst_raw, length);
302 } else {
303 AccessInternal::arraycopy_disjoint_words(src_raw, dst_raw, length);
304 }
305 }
306
307 template <DecoratorSet decorators, typename T>
308 static inline typename EnableIf<
309 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
310 !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
311 !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
312 !HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
313 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
314 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
315 size_t length) {
316 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
317 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
318
319 AccessInternal::arraycopy_conjoint(src_raw, dst_raw, length);
320 }
321
322 template <DecoratorSet decorators, typename T>
323 static inline typename EnableIf<
324 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
325 !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
326 !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
327 HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
328 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
329 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
330 size_t length) {
331 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
332 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
333
334 AccessInternal::arraycopy_conjoint_atomic(src_raw, dst_raw, length);
335 }
336 };
337
338 template<> struct RawAccessBarrierArrayCopy::IsHeapWordSized<void>: public IntegralConstant<bool, false> { };
339
340 template <DecoratorSet decorators>
341 template <typename T>
342 inline void RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
343 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
344 size_t length) {
345 RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
346 dst_obj, dst_offset_in_bytes, dst_raw,
347 length);
348 }
349
350 template <DecoratorSet decorators>
351 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
352 // 4839641 (4840070): We must do an oop-atomic copy, because if another thread
353 // is modifying a reference field in the clonee, a non-oop-atomic copy might
354 // be suspended in the middle of copying the pointer and end up with parts
355 // of two different pointers in the field. Subsequent dereferences will crash.
356 // 4846409: an oop-copy of objects with long or double fields or arrays of same
357 // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead
358 // of oops. We know objects are aligned on a minimum of an jlong boundary.
359 // The same is true of StubRoutines::object_copy and the various oop_copy
360 // variants, and of the code generated by the inline_native_clone intrinsic.
361
362 assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
363 AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src),
364 reinterpret_cast<jlong*>((oopDesc*)dst),
365 align_object_size(size) / HeapWordsPerLong);
366 // Clear the header
367 dst->init_mark();
368 }
369
370 template <DecoratorSet decorators>
371 inline void RawAccessBarrier<decorators>::value_copy(void* src, void* dst, InlineKlass* md) {
372 assert(is_aligned(src, md->get_alignment()) && is_aligned(dst, md->get_alignment()), "Unalign value_copy");
373 AccessInternal::arraycopy_conjoint_atomic(src, dst, static_cast<size_t>(md->get_exact_size_in_bytes()));
374 }
375 #endif // SHARE_OOPS_ACCESSBACKEND_INLINE_HPP