1 /*
2 * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
26 #define SHARE_OOPS_ACCESSBACKEND_INLINE_HPP
27
28 #include "oops/accessBackend.hpp"
29
30 #include "oops/access.hpp"
31 #include "oops/arrayOop.hpp"
32 #include "oops/compressedOops.inline.hpp"
33 #include "oops/oopsHierarchy.hpp"
34 #include "runtime/atomic.hpp"
35 #include "runtime/orderAccess.hpp"
36
37 #include <type_traits>
38
39 template <DecoratorSet decorators>
40 template <DecoratorSet idecorators, typename T>
41 inline typename EnableIf<
42 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
43 RawAccessBarrier<decorators>::decode_internal(typename HeapOopType<idecorators>::type value) {
44 if (HasDecorator<decorators, IS_NOT_NULL>::value) {
45 return CompressedOops::decode_not_null(value);
46 } else {
47 return CompressedOops::decode(value);
48 }
49 }
50
51 template <DecoratorSet decorators>
52 template <DecoratorSet idecorators, typename T>
53 inline typename EnableIf<
54 AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
55 typename HeapOopType<idecorators>::type>::type
56 RawAccessBarrier<decorators>::encode_internal(T value) {
57 if (HasDecorator<decorators, IS_NOT_NULL>::value) {
58 return CompressedOops::encode_not_null(value);
59 } else {
60 return CompressedOops::encode(value);
61 }
62 }
63
64 template <DecoratorSet decorators>
65 template <typename T>
66 inline void RawAccessBarrier<decorators>::oop_store(void* addr, T value) {
67 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
68 Encoded encoded = encode(value);
69 store(reinterpret_cast<Encoded*>(addr), encoded);
70 }
71
72 template <DecoratorSet decorators>
73 template <typename T>
74 inline void RawAccessBarrier<decorators>::oop_store_at(oop base, ptrdiff_t offset, T value) {
75 oop_store(field_addr(base, offset), value);
76 }
77
78 template <DecoratorSet decorators>
79 template <typename T>
80 inline T RawAccessBarrier<decorators>::oop_load(void* addr) {
81 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
82 Encoded encoded = load<Encoded>(reinterpret_cast<Encoded*>(addr));
83 return decode<T>(encoded);
84 }
85
86 template <DecoratorSet decorators>
87 template <typename T>
88 inline T RawAccessBarrier<decorators>::oop_load_at(oop base, ptrdiff_t offset) {
89 return oop_load<T>(field_addr(base, offset));
90 }
91
92 template <DecoratorSet decorators>
93 template <typename T>
94 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg(void* addr, T compare_value, T new_value) {
95 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
96 Encoded encoded_new = encode(new_value);
97 Encoded encoded_compare = encode(compare_value);
98 Encoded encoded_result = atomic_cmpxchg(reinterpret_cast<Encoded*>(addr),
99 encoded_compare,
100 encoded_new);
101 return decode<T>(encoded_result);
102 }
103
104 template <DecoratorSet decorators>
105 template <typename T>
106 inline T RawAccessBarrier<decorators>::oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
107 return oop_atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
108 }
109
110 template <DecoratorSet decorators>
111 template <typename T>
112 inline T RawAccessBarrier<decorators>::oop_atomic_xchg(void* addr, T new_value) {
113 typedef typename AccessInternal::EncodedType<decorators, T>::type Encoded;
114 Encoded encoded_new = encode(new_value);
115 Encoded encoded_result = atomic_xchg(reinterpret_cast<Encoded*>(addr), encoded_new);
116 return decode<T>(encoded_result);
117 }
118
119 template <DecoratorSet decorators>
120 template <typename T>
121 inline T RawAccessBarrier<decorators>::oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
122 return oop_atomic_xchg(field_addr(base, offset), new_value);
123 }
124
125 template <DecoratorSet decorators>
126 template <typename T>
127 inline bool RawAccessBarrier<decorators>::oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
128 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
129 size_t length) {
130 return arraycopy(src_obj, src_offset_in_bytes, src_raw,
131 dst_obj, dst_offset_in_bytes, dst_raw,
132 length);
133 }
134
135 template <DecoratorSet decorators>
136 template <DecoratorSet ds, typename T>
137 inline typename EnableIf<
138 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
139 RawAccessBarrier<decorators>::load_internal(void* addr) {
140 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
141 OrderAccess::fence();
142 }
143 return Atomic::load_acquire(reinterpret_cast<const volatile T*>(addr));
144 }
145
146 template <DecoratorSet decorators>
147 template <DecoratorSet ds, typename T>
148 inline typename EnableIf<
149 HasDecorator<ds, MO_ACQUIRE>::value, T>::type
150 RawAccessBarrier<decorators>::load_internal(void* addr) {
151 return Atomic::load_acquire(reinterpret_cast<const volatile T*>(addr));
152 }
153
154 template <DecoratorSet decorators>
155 template <DecoratorSet ds, typename T>
156 inline typename EnableIf<
157 HasDecorator<ds, MO_RELAXED>::value, T>::type
158 RawAccessBarrier<decorators>::load_internal(void* addr) {
159 return Atomic::load(reinterpret_cast<const volatile T*>(addr));
160 }
161
162 template <DecoratorSet decorators>
163 template <DecoratorSet ds, typename T>
164 inline typename EnableIf<
165 HasDecorator<ds, MO_SEQ_CST>::value>::type
166 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
167 Atomic::release_store_fence(reinterpret_cast<volatile T*>(addr), value);
168 }
169
170 template <DecoratorSet decorators>
171 template <DecoratorSet ds, typename T>
172 inline typename EnableIf<
173 HasDecorator<ds, MO_RELEASE>::value>::type
174 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
175 Atomic::release_store(reinterpret_cast<volatile T*>(addr), value);
176 }
177
178 template <DecoratorSet decorators>
179 template <DecoratorSet ds, typename T>
180 inline typename EnableIf<
181 HasDecorator<ds, MO_RELAXED>::value>::type
182 RawAccessBarrier<decorators>::store_internal(void* addr, T value) {
183 Atomic::store(reinterpret_cast<volatile T*>(addr), value);
184 }
185
186 template <DecoratorSet decorators>
187 template <DecoratorSet ds, typename T>
188 inline typename EnableIf<
189 HasDecorator<ds, MO_RELAXED>::value, T>::type
190 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
191 return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr),
192 compare_value,
193 new_value,
194 memory_order_relaxed);
195 }
196
197 template <DecoratorSet decorators>
198 template <DecoratorSet ds, typename T>
199 inline typename EnableIf<
200 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
201 RawAccessBarrier<decorators>::atomic_cmpxchg_internal(void* addr, T compare_value, T new_value) {
202 return Atomic::cmpxchg(reinterpret_cast<volatile T*>(addr),
203 compare_value,
204 new_value,
205 memory_order_conservative);
206 }
207
208 template <DecoratorSet decorators>
209 template <DecoratorSet ds, typename T>
210 inline typename EnableIf<
211 HasDecorator<ds, MO_SEQ_CST>::value, T>::type
212 RawAccessBarrier<decorators>::atomic_xchg_internal(void* addr, T new_value) {
213 return Atomic::xchg(reinterpret_cast<volatile T*>(addr),
214 new_value);
215 }
216
217 class RawAccessBarrierArrayCopy: public AllStatic {
218 template<typename T> struct IsHeapWordSized: public std::integral_constant<bool, sizeof(T) == HeapWordSize> { };
219 public:
220 template <DecoratorSet decorators, typename T>
221 static inline typename EnableIf<
222 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value>::type
223 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
224 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
225 size_t length) {
226 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
227 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
228
229 // We do not check for ARRAYCOPY_ATOMIC for oops, because they are unconditionally always atomic.
230 if (HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value) {
231 AccessInternal::arraycopy_arrayof_conjoint_oops(src_raw, dst_raw, length);
232 } else {
233 typedef typename HeapOopType<decorators>::type OopType;
234 AccessInternal::arraycopy_conjoint_oops(reinterpret_cast<OopType*>(src_raw),
235 reinterpret_cast<OopType*>(dst_raw), length);
236 }
237 }
238
239 template <DecoratorSet decorators, typename T>
240 static inline typename EnableIf<
241 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
242 HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value>::type
243 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
244 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
245 size_t length) {
246 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
247 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
248
249 AccessInternal::arraycopy_arrayof_conjoint(src_raw, dst_raw, length);
250 }
251
252 template <DecoratorSet decorators, typename T>
253 static inline typename EnableIf<
254 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
255 HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value>::type
256 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
257 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
258 size_t length) {
259 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
260 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
261
262 // There is only a disjoint optimization for word granularity copying
263 if (HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value) {
264 AccessInternal::arraycopy_disjoint_words_atomic(src_raw, dst_raw, length);
265 } else {
266 AccessInternal::arraycopy_disjoint_words(src_raw, dst_raw, length);
267 }
268 }
269
270 template <DecoratorSet decorators, typename T>
271 static inline typename EnableIf<
272 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
273 !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
274 !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
275 !HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
276 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
277 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
278 size_t length) {
279 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
280 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
281
282 AccessInternal::arraycopy_conjoint(src_raw, dst_raw, length);
283 }
284
285 template <DecoratorSet decorators, typename T>
286 static inline typename EnableIf<
287 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
288 !(HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value && IsHeapWordSized<T>::value) &&
289 !HasDecorator<decorators, ARRAYCOPY_ARRAYOF>::value &&
290 HasDecorator<decorators, ARRAYCOPY_ATOMIC>::value>::type
291 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
292 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
293 size_t length) {
294 src_raw = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
295 dst_raw = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
296
297 AccessInternal::arraycopy_conjoint_atomic(src_raw, dst_raw, length);
298 }
299 };
300
301 template<> struct RawAccessBarrierArrayCopy::IsHeapWordSized<void>: public std::false_type { };
302
303 template <DecoratorSet decorators>
304 template <typename T>
305 inline bool RawAccessBarrier<decorators>::arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
306 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
307 size_t length) {
308 RawAccessBarrierArrayCopy::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
309 dst_obj, dst_offset_in_bytes, dst_raw,
310 length);
311 return true;
312 }
313
314 template <DecoratorSet decorators>
315 inline void RawAccessBarrier<decorators>::clone(oop src, oop dst, size_t size) {
316 // 4839641 (4840070): We must do an oop-atomic copy, because if another thread
317 // is modifying a reference field in the clonee, a non-oop-atomic copy might
318 // be suspended in the middle of copying the pointer and end up with parts
319 // of two different pointers in the field. Subsequent dereferences will crash.
320 // 4846409: an oop-copy of objects with long or double fields or arrays of same
321 // won't copy the longs/doubles atomically in 32-bit vm's, so we copy jlongs instead
322 // of oops. We know objects are aligned on a minimum of an jlong boundary.
323 // The same is true of StubRoutines::object_copy and the various oop_copy
324 // variants, and of the code generated by the inline_native_clone intrinsic.
325
326 assert(MinObjAlignmentInBytes >= BytesPerLong, "objects misaligned");
327 AccessInternal::arraycopy_conjoint_atomic(reinterpret_cast<jlong*>((oopDesc*)src),
328 reinterpret_cast<jlong*>((oopDesc*)dst),
329 align_object_size(size) / HeapWordsPerLong);
330 // Clear the header
331 dst->init_mark();
332 }
333
334 #endif // SHARE_OOPS_ACCESSBACKEND_INLINE_HPP