1 /*
2 * Copyright (c) 2017, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
26
27 #include "gc/z/zBarrierSet.hpp"
28
29 #include "gc/shared/accessBarrierSupport.inline.hpp"
30 #include "gc/z/zAddress.inline.hpp"
31 #include "gc/z/zHeap.hpp"
32 #include "gc/z/zNMethod.hpp"
33 #include "oops/objArrayOop.hpp"
34 #include "utilities/debug.hpp"
35
36 template <DecoratorSet decorators, typename BarrierSetT>
37 template <DecoratorSet expected>
38 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
39 if ((decorators & expected) == 0) {
40 fatal("Using unsupported access decorators");
41 }
42 }
43
44 template <DecoratorSet decorators, typename BarrierSetT>
45 template <DecoratorSet expected>
46 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
47 if ((decorators & expected) != 0) {
48 fatal("Using unsupported access decorators");
49 }
50 }
51
52 template <DecoratorSet decorators, typename BarrierSetT>
53 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
54 ShouldNotReachHere();
55 }
56
57 template <DecoratorSet decorators, typename BarrierSetT>
58 inline zpointer* ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::field_addr(oop base, ptrdiff_t offset) {
59 assert(base != nullptr, "Invalid base");
60 return reinterpret_cast<zpointer*>(reinterpret_cast<intptr_t>((void*)base) + offset);
61 }
62
63 template <DecoratorSet decorators, typename BarrierSetT>
64 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier(zpointer* p, zpointer o) {
65 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
66
67 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
68 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
69 // Load barriers on strong oop refs don't keep objects alive
70 return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
71 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
72 return ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
73 } else {
74 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
75 return ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
76 }
77 } else {
78 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
79 return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
80 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
81 return ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(p, o);
82 } else {
83 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
84 return ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(p, o);
85 }
86 }
87 }
88
89 template <DecoratorSet decorators, typename BarrierSetT>
90 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_unknown_oop_ref(oop base, ptrdiff_t offset, zpointer* p, zpointer o) {
91 verify_decorators_present<ON_UNKNOWN_OOP_REF>();
92
93 const DecoratorSet decorators_known_strength =
94 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
95
96 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
97 if (decorators_known_strength & ON_STRONG_OOP_REF) {
98 // Load barriers on strong oop refs don't keep objects alive
99 return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
100 } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
101 return ZBarrierSet::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
102 } else {
103 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
104 return ZBarrierSet::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
105 }
106 } else {
107 if (decorators_known_strength & ON_STRONG_OOP_REF) {
108 return ZBarrierSet::load_barrier_on_oop_field_preloaded(p, o);
109 } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
110 return ZBarrierSet::load_barrier_on_weak_oop_field_preloaded(p, o);
111 } else {
112 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
113 return ZBarrierSet::load_barrier_on_phantom_oop_field_preloaded(p, o);
114 }
115 }
116 }
117
118 inline zpointer ZBarrierSet::store_good(oop obj) {
119 assert(ZPointerStoreGoodMask != 0, "sanity");
120
121 const zaddress addr = to_zaddress(obj);
122 return ZAddress::store_good(addr);
123 }
124
125 template <DecoratorSet decorators, typename BarrierSetT>
126 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_with_healing(zpointer* p) {
127 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
128 ZBarrierSet::store_barrier_on_heap_oop_field(p, true /* heal */);
129 } else {
130 assert(false, "Should not be used on uninitialized memory");
131 }
132 }
133
134 template <DecoratorSet decorators, typename BarrierSetT>
135 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_without_healing(zpointer* p) {
136 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
137 ZBarrierSet::store_barrier_on_heap_oop_field(p, false /* heal */);
138 }
139 }
140
141 template <DecoratorSet decorators, typename BarrierSetT>
142 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::no_keep_alive_store_barrier_heap(zpointer* p) {
143 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
144 ZBarrierSet::no_keep_alive_store_barrier_on_heap_oop_field(p);
145 }
146 }
147
148 template <DecoratorSet decorators, typename BarrierSetT>
149 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_with_healing(zpointer* p) {
150 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
151 ZBarrierSet::store_barrier_on_native_oop_field(p, true /* heal */);
152 } else {
153 assert(false, "Should not be used on uninitialized memory");
154 }
155 }
156
157 template <DecoratorSet decorators, typename BarrierSetT>
158 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_without_healing(zpointer* p) {
159 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
160 ZBarrierSet::store_barrier_on_native_oop_field(p, false /* heal */);
161 }
162 }
163
164 //
165 // In heap
166 //
167 template <DecoratorSet decorators, typename BarrierSetT>
168 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(zpointer* p) {
169 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
170
171 const zpointer o = Raw::load_in_heap(p);
172 assert_is_valid(o);
173
174 return to_oop(load_barrier(p, o));
175 }
176
177 template <DecoratorSet decorators, typename BarrierSetT>
178 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
179 zpointer* const p = field_addr(base, offset);
180
181 const zpointer o = Raw::load_in_heap(p);
182 assert_is_valid(o);
183
184 if (HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
185 return to_oop(load_barrier_on_unknown_oop_ref(base, offset, p, o));
186 }
187
188 return to_oop(load_barrier(p, o));
189 }
190
191 template <DecoratorSet decorators>
192 bool is_store_barrier_no_keep_alive() {
193 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
194 return HasDecorator<decorators, AS_NO_KEEPALIVE>::value;
195 }
196
197 if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
198 return true;
199 }
200
201 assert((decorators & ON_PHANTOM_OOP_REF) != 0, "Must be");
202 return true;
203 }
204
205 template <DecoratorSet decorators>
206 inline bool is_store_barrier_no_keep_alive(oop base, ptrdiff_t offset) {
207 if (!HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
208 return is_store_barrier_no_keep_alive<decorators>();
209 }
210
211 const DecoratorSet decorators_known_strength =
212 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
213
214 if ((decorators_known_strength & ON_STRONG_OOP_REF) != 0) {
215 return (decorators & AS_NO_KEEPALIVE) != 0;
216 }
217
218 if ((decorators_known_strength & ON_WEAK_OOP_REF) != 0) {
219 return true;
220 }
221
222 assert((decorators_known_strength & ON_PHANTOM_OOP_REF) != 0, "Must be");
223 return true;
224 }
225
226 template <DecoratorSet decorators, typename BarrierSetT>
227 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(zpointer* p, oop value) {
228 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
229
230 if (is_store_barrier_no_keep_alive<decorators>()) {
231 no_keep_alive_store_barrier_heap(p);
232 } else {
233 store_barrier_heap_without_healing(p);
234 }
235
236 Raw::store_in_heap(p, store_good(value));
237 }
238
239 template <DecoratorSet decorators, typename BarrierSetT>
240 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
241 zpointer* const p = field_addr(base, offset);
242
243 if (is_store_barrier_no_keep_alive<decorators>(base, offset)) {
244 no_keep_alive_store_barrier_heap(p);
245 } else {
246 store_barrier_heap_without_healing(p);
247 }
248
249 Raw::store_in_heap(p, store_good(value));
250 }
251
252 template <DecoratorSet decorators, typename BarrierSetT>
253 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(zpointer* p, oop value) {
254 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
255
256 if (!is_store_barrier_no_keep_alive<decorators>()) {
257 store_barrier_native_without_healing(p);
258 }
259
260 Raw::store(p, store_good(value));
261 }
262
263 template <DecoratorSet decorators, typename BarrierSetT>
264 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(zpointer* p, oop compare_value, oop new_value) {
265 verify_decorators_present<ON_STRONG_OOP_REF>();
266 verify_decorators_absent<AS_NO_KEEPALIVE>();
267
268 store_barrier_heap_with_healing(p);
269
270 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
271 assert_is_valid(o);
272
273 return to_oop(ZPointer::uncolor_store_good(o));
274 }
275
276 template <DecoratorSet decorators, typename BarrierSetT>
277 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) {
278 verify_decorators_present<ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF>();
279 verify_decorators_absent<AS_NO_KEEPALIVE>();
280
281 // Through Unsafe.CompareAndExchangeObject()/CompareAndSetObject() we can receive
282 // calls with ON_UNKNOWN_OOP_REF set. However, we treat these as ON_STRONG_OOP_REF,
283 // with the motivation that if you're doing Unsafe operations on a Reference.referent
284 // field, then you're on your own anyway.
285 zpointer* const p = field_addr(base, offset);
286
287 store_barrier_heap_with_healing(p);
288
289 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
290 assert_is_valid(o);
291
292 return to_oop(ZPointer::uncolor_store_good(o));
293 }
294
295 template <DecoratorSet decorators, typename BarrierSetT>
296 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(zpointer* p, oop new_value) {
297 verify_decorators_present<ON_STRONG_OOP_REF>();
298 verify_decorators_absent<AS_NO_KEEPALIVE>();
299
300 store_barrier_heap_with_healing(p);
301
302 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
303 assert_is_valid(o);
304
305 return to_oop(ZPointer::uncolor_store_good(o));
306 }
307
308 template <DecoratorSet decorators, typename BarrierSetT>
309 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
310 verify_decorators_present<ON_STRONG_OOP_REF>();
311 verify_decorators_absent<AS_NO_KEEPALIVE>();
312
313 zpointer* const p = field_addr(base, offset);
314
315 store_barrier_heap_with_healing(p);
316
317 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
318 assert_is_valid(o);
319
320 return to_oop(ZPointer::uncolor_store_good(o));
321 }
322
323 template <DecoratorSet decorators, typename BarrierSetT>
324 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
325 store_barrier_heap_without_healing(dst);
326
327 return ZBarrierSet::load_barrier_on_oop_field(src);
328 }
329
330 template <DecoratorSet decorators, typename BarrierSetT>
331 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
332 const zaddress obj = oop_copy_one_barriers(dst, src);
333
334 AtomicAccess::store(dst, ZAddress::store_good(obj));
335 }
336
337 template <DecoratorSet decorators, typename BarrierSetT>
338 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
339 const zaddress obj = oop_copy_one_barriers(dst, src);
340
341 if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
342 // Check cast failed
343 return false;
344 }
345
346 AtomicAccess::store(dst, ZAddress::store_good(obj));
347
348 return true;
349 }
350
351 template <DecoratorSet decorators, typename BarrierSetT>
352 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
353 // Check cast and copy each elements
354 for (const zpointer* const end = src + length; src < end; src++, dst++) {
355 if (!oop_copy_one_check_cast(dst, src, dst_klass)) {
356 // Check cast failed
357 return false;
358 }
359 }
360
361 return true;
362 }
363
364 template <DecoratorSet decorators, typename BarrierSetT>
365 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
366 const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
367
368 if (is_disjoint || src > dst) {
369 for (const zpointer* const end = src + length; src < end; src++, dst++) {
370 oop_copy_one(dst, src);
371 }
372 return true;
373 }
374
375 if (src < dst) {
376 const zpointer* const end = src;
377 src += length - 1;
378 dst += length - 1;
379 for ( ; src >= end; src--, dst--) {
380 oop_copy_one(dst, src);
381 }
382 return true;
383 }
384
385 // src and dst are the same; nothing to do
386 return true;
387 }
388
389 template <DecoratorSet decorators, typename BarrierSetT>
390 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
391 arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
392 size_t length) {
393 zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
394 zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
395
396 if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
397 Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
398 return oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
399 }
400
401 return oop_arraycopy_in_heap_no_check_cast(dst, src, length);
402 }
403
404 template <DecoratorSet decorators, typename BarrierSetT>
405 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
406 check_is_valid_zaddress(src);
407
408 if (dst->is_objArray()) {
409 // Cloning an object array is similar to performing array copy.
410 // If an array is large enough to have its allocation segmented,
411 // this operation might require GC barriers. However, the intrinsics
412 // for cloning arrays transform the clone to an optimized allocation
413 // and arraycopy sequence, so the performance of this runtime call
414 // does not matter for object arrays.
415 clone_obj_array(objArrayOop(src), objArrayOop(dst));
416 return;
417 }
418
419 // Fix the oops
420 ZBarrierSet::load_barrier_all(src, size);
421
422 // Clone the object
423 Raw::clone_in_heap(src, dst, size);
424
425 // Color store good before handing out
426 ZBarrierSet::color_store_good_all(dst, size);
427 }
428
429 //
430 // Not in heap
431 //
432 template <DecoratorSet decorators, typename BarrierSetT>
433 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
434 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
435
436 const zpointer o = Raw::template load<zpointer>(p);
437 assert_is_valid(o);
438 return to_oop(load_barrier(p, o));
439 }
440
441 template <DecoratorSet decorators, typename BarrierSetT>
442 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
443 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
444
445 return oop_load_not_in_heap((zpointer*)p);
446 }
447
448 template <DecoratorSet decorators, typename BarrierSetT>
449 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(zpointer* p, oop compare_value, oop new_value) {
450 verify_decorators_present<ON_STRONG_OOP_REF>();
451 verify_decorators_absent<AS_NO_KEEPALIVE>();
452
453 store_barrier_native_with_healing(p);
454
455 const zpointer o = Raw::atomic_cmpxchg(p, store_good(compare_value), store_good(new_value));
456 assert_is_valid(o);
457
458 return to_oop(ZPointer::uncolor_store_good(o));
459 }
460
461 template <DecoratorSet decorators, typename BarrierSetT>
462 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(zpointer* p, oop new_value) {
463 verify_decorators_present<ON_STRONG_OOP_REF>();
464 verify_decorators_absent<AS_NO_KEEPALIVE>();
465
466 store_barrier_native_with_healing(p);
467
468 const zpointer o = Raw::atomic_xchg(p, store_good(new_value));
469 assert_is_valid(o);
470
471 return to_oop(ZPointer::uncolor_store_good(o));
472 }
473
474 #endif // SHARE_GC_Z_ZBARRIERSET_INLINE_HPP