1 /*
2 * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
26
27 #include "gc/z/zBarrierSet.hpp"
28
29 #include "gc/shared/accessBarrierSupport.inline.hpp"
30 #include "gc/z/zAddress.inline.hpp"
31 #include "gc/z/zBarrier.inline.hpp"
32 #include "gc/z/zIterator.inline.hpp"
33 #include "gc/z/zNMethod.hpp"
34 #include "memory/iterator.inline.hpp"
35 #include "utilities/debug.hpp"
36
37 template <DecoratorSet decorators, typename BarrierSetT>
38 template <DecoratorSet expected>
39 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
40 if ((decorators & expected) == 0) {
41 fatal("Using unsupported access decorators");
42 }
43 }
44
45 template <DecoratorSet decorators, typename BarrierSetT>
46 template <DecoratorSet expected>
47 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
48 if ((decorators & expected) != 0) {
49 fatal("Using unsupported access decorators");
50 }
51 }
52
53 template <DecoratorSet decorators, typename BarrierSetT>
54 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
55 ShouldNotReachHere();
56 }
57
58 template <DecoratorSet decorators, typename BarrierSetT>
59 inline zpointer* ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::field_addr(oop base, ptrdiff_t offset) {
60 assert(base != nullptr, "Invalid base");
61 return reinterpret_cast<zpointer*>(reinterpret_cast<intptr_t>((void*)base) + offset);
62 }
63
64 template <DecoratorSet decorators, typename BarrierSetT>
65 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier(zpointer* p, zpointer o) {
66 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
67
68 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
69 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
70 // Load barriers on strong oop refs don't keep objects alive
71 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
72 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
73 return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
74 } else {
75 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
76 return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
77 }
78 } else {
79 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
80 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
81 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
82 return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
83 } else {
84 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
85 return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
86 }
87 }
88 }
89
90 template <DecoratorSet decorators, typename BarrierSetT>
91 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_unknown_oop_ref(oop base, ptrdiff_t offset, zpointer* p, zpointer o) {
92 verify_decorators_present<ON_UNKNOWN_OOP_REF>();
93
94 const DecoratorSet decorators_known_strength =
95 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
96
97 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
98 if (decorators_known_strength & ON_STRONG_OOP_REF) {
99 // Load barriers on strong oop refs don't keep objects alive
100 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
101 } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
102 return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
103 } else {
104 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
105 return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
106 }
107 } else {
108 if (decorators_known_strength & ON_STRONG_OOP_REF) {
109 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
110 } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
111 return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
112 } else {
113 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
114 return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
115 }
116 }
117 }
118
119 inline zpointer ZBarrierSet::store_good(oop obj) {
120 assert(ZPointerStoreGoodMask != 0, "sanity");
121
122 const zaddress addr = to_zaddress(obj);
123 return ZAddress::store_good(addr);
124 }
125
126 template <DecoratorSet decorators, typename BarrierSetT>
127 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_with_healing(zpointer* p) {
128 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
129 ZBarrier::store_barrier_on_heap_oop_field(p, true /* heal */);
130 } else {
131 assert(false, "Should not be used on uninitialized memory");
132 }
133 }
134
135 template <DecoratorSet decorators, typename BarrierSetT>
136 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_without_healing(zpointer* p) {
137 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
138 ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
139 }
140 }
141
142 template <DecoratorSet decorators, typename BarrierSetT>
143 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::no_keep_alive_store_barrier_heap(zpointer* p) {
144 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
145 ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p);
146 }
147 }
148
149 template <DecoratorSet decorators, typename BarrierSetT>
150 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_with_healing(zpointer* p) {
151 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
152 ZBarrier::store_barrier_on_native_oop_field(p, true /* heal */);
153 } else {
154 assert(false, "Should not be used on uninitialized memory");
155 }
156 }
157
158 template <DecoratorSet decorators, typename BarrierSetT>
159 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_without_healing(zpointer* p) {
160 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
161 ZBarrier::store_barrier_on_native_oop_field(p, false /* heal */);
162 }
163 }
164
165 //
166 // In heap
167 //
168 template <DecoratorSet decorators, typename BarrierSetT>
169 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(zpointer* p) {
170 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
171
172 const zpointer o = Raw::load_in_heap(p);
173 assert_is_valid(o);
174
175 return to_oop(load_barrier(p, o));
176 }
177
178 template <DecoratorSet decorators, typename BarrierSetT>
179 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
180 zpointer* const p = field_addr(base, offset);
181
182 const zpointer o = Raw::load_in_heap(p);
183 assert_is_valid(o);
184
185 if (HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
186 return to_oop(load_barrier_on_unknown_oop_ref(base, offset, p, o));
187 }
188
189 return to_oop(load_barrier(p, o));
190 }
191
192 template <DecoratorSet decorators>
193 bool is_store_barrier_no_keep_alive() {
194 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
195 return HasDecorator<decorators, AS_NO_KEEPALIVE>::value;
196 }
197
198 if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
199 return true;
200 }
201
202 assert((decorators & ON_PHANTOM_OOP_REF) != 0, "Must be");
203 return true;
204 }
205
206 template <DecoratorSet decorators>
207 inline bool is_store_barrier_no_keep_alive(oop base, ptrdiff_t offset) {
208 if (!HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
209 return is_store_barrier_no_keep_alive<decorators>();
210 }
211
212 const DecoratorSet decorators_known_strength =
213 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
214
215 if ((decorators_known_strength & ON_STRONG_OOP_REF) != 0) {
216 return (decorators & AS_NO_KEEPALIVE) != 0;
217 }
218
219 if ((decorators_known_strength & ON_WEAK_OOP_REF) != 0) {
220 return true;
221 }
222
223 assert((decorators_known_strength & ON_PHANTOM_OOP_REF) != 0, "Must be");
224 return true;
225 }
226
227 template <DecoratorSet decorators, typename BarrierSetT>
228 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(zpointer* p, oop value) {
229 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
230
231 if (is_store_barrier_no_keep_alive<decorators>()) {
232 no_keep_alive_store_barrier_heap(p);
233 } else {
234 store_barrier_heap_without_healing(p);
235 }
236
237 Raw::store_in_heap(p, store_good(value));
238 }
239
240 template <DecoratorSet decorators, typename BarrierSetT>
241 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
242 zpointer* const p = field_addr(base, offset);
243
244 if (is_store_barrier_no_keep_alive<decorators>(base, offset)) {
245 no_keep_alive_store_barrier_heap(p);
246 } else {
247 store_barrier_heap_without_healing(p);
248 }
249
250 Raw::store_in_heap(p, store_good(value));
251 }
252
253 template <DecoratorSet decorators, typename BarrierSetT>
254 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(zpointer* p, oop value) {
255 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
256
257 if (!is_store_barrier_no_keep_alive<decorators>()) {
258 store_barrier_native_without_healing(p);
259 }
260
261 Raw::store(p, store_good(value));
262 }
263
264 template <DecoratorSet decorators, typename BarrierSetT>
265 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(zpointer* p, oop compare_value, oop new_value) {
266 verify_decorators_present<ON_STRONG_OOP_REF>();
267 verify_decorators_absent<AS_NO_KEEPALIVE>();
268
269 store_barrier_heap_with_healing(p);
270
271 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
272 assert_is_valid(o);
273
274 return to_oop(ZPointer::uncolor_store_good(o));
275 }
276
277 template <DecoratorSet decorators, typename BarrierSetT>
278 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) {
279 verify_decorators_present<ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF>();
280 verify_decorators_absent<AS_NO_KEEPALIVE>();
281
282 // Through Unsafe.CompareAndExchangeObject()/CompareAndSetObject() we can receive
283 // calls with ON_UNKNOWN_OOP_REF set. However, we treat these as ON_STRONG_OOP_REF,
284 // with the motivation that if you're doing Unsafe operations on a Reference.referent
285 // field, then you're on your own anyway.
286 zpointer* const p = field_addr(base, offset);
287
288 store_barrier_heap_with_healing(p);
289
290 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
291 assert_is_valid(o);
292
293 return to_oop(ZPointer::uncolor_store_good(o));
294 }
295
296 template <DecoratorSet decorators, typename BarrierSetT>
297 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(zpointer* p, oop new_value) {
298 verify_decorators_present<ON_STRONG_OOP_REF>();
299 verify_decorators_absent<AS_NO_KEEPALIVE>();
300
301 store_barrier_heap_with_healing(p);
302
303 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
304 assert_is_valid(o);
305
306 return to_oop(ZPointer::uncolor_store_good(o));
307 }
308
309 template <DecoratorSet decorators, typename BarrierSetT>
310 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
311 verify_decorators_present<ON_STRONG_OOP_REF>();
312 verify_decorators_absent<AS_NO_KEEPALIVE>();
313
314 zpointer* const p = field_addr(base, offset);
315
316 store_barrier_heap_with_healing(p);
317
318 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
319 assert_is_valid(o);
320
321 return to_oop(ZPointer::uncolor_store_good(o));
322 }
323
324 template <DecoratorSet decorators, typename BarrierSetT>
325 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
326 store_barrier_heap_without_healing(dst);
327
328 return ZBarrier::load_barrier_on_oop_field(src);
329 }
330
331 template <DecoratorSet decorators, typename BarrierSetT>
332 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
333 const zaddress obj = oop_copy_one_barriers(dst, src);
334
335 Atomic::store(dst, ZAddress::store_good(obj));
336 }
337
338 template <DecoratorSet decorators, typename BarrierSetT>
339 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
340 const zaddress obj = oop_copy_one_barriers(dst, src);
341
342 if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
343 // Check cast failed
344 return false;
345 }
346
347 Atomic::store(dst, ZAddress::store_good(obj));
348
349 return true;
350 }
351
352
353 template <DecoratorSet decorators, typename BarrierSetT>
354 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
355 // Check cast and copy each elements
356 for (const zpointer* const end = src + length; src < end; src++, dst++) {
357 if (!oop_copy_one_check_cast(dst, src, dst_klass)) {
358 // Check cast failed
359 return false;
360 }
361 }
362
363 return true;
364 }
365
366 template <DecoratorSet decorators, typename BarrierSetT>
367 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
368 const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
369
370 if (is_disjoint || src > dst) {
371 for (const zpointer* const end = src + length; src < end; src++, dst++) {
372 oop_copy_one(dst, src);
373 }
374 return true;
375 }
376
377 if (src < dst) {
378 const zpointer* const end = src;
379 src += length - 1;
380 dst += length - 1;
381 for ( ; src >= end; src--, dst--) {
382 oop_copy_one(dst, src);
383 }
384 return true;
385 }
386
387 // src and dst are the same; nothing to do
388 return true;
389 }
390
391 template <DecoratorSet decorators, typename BarrierSetT>
392 inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
393 arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
394 size_t length) {
395 zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
396 zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
397
398 if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
399 Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
400 return oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
401 }
402
403 return oop_arraycopy_in_heap_no_check_cast(dst, src, length);
404 }
405
406 class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
407 public:
408 virtual void do_oop(oop* p_) {
409 volatile zpointer* const p = (volatile zpointer*)p_;
410 const zpointer ptr = ZBarrier::load_atomic(p);
411 const zaddress addr = ZPointer::uncolor(ptr);
412 Atomic::store(p, ZAddress::store_good(addr));
413 }
414
415 virtual void do_oop(narrowOop* p) {
416 ShouldNotReachHere();
417 }
418 };
419
420 class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
421 public:
422 virtual void do_oop(oop* p) {
423 ZBarrier::load_barrier_on_oop_field((zpointer*)p);
424 }
425
426 virtual void do_oop(narrowOop* p) {
427 ShouldNotReachHere();
428 }
429 };
430
431 template <DecoratorSet decorators, typename BarrierSetT>
432 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
433 check_is_valid_zaddress(src);
434
435 if (dst->is_objArray()) {
436 // Cloning an object array is similar to performing array copy.
437 // If an array is large enough to have its allocation segmented,
438 // this operation might require GC barriers. However, the intrinsics
439 // for cloning arrays transform the clone to an optimized allocation
440 // and arraycopy sequence, so the performance of this runtime call
441 // does not matter for object arrays.
442 clone_obj_array(objArrayOop(src), objArrayOop(dst));
443 return;
444 }
445
446 // Fix the oops
447 ZLoadBarrierOopClosure cl;
448 ZIterator::oop_iterate(src, &cl);
449
450 // Clone the object
451 Raw::clone_in_heap(src, dst, size);
452
453 assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
454
455 // Color store good before handing out
456 ZColorStoreGoodOopClosure cl_sg;
457 ZIterator::oop_iterate(dst, &cl_sg);
458 }
459
460 //
461 // Not in heap
462 //
463 template <DecoratorSet decorators, typename BarrierSetT>
464 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
465 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
466
467 const zpointer o = Raw::template load<zpointer>(p);
468 assert_is_valid(o);
469 return to_oop(load_barrier(p, o));
470 }
471
472 template <DecoratorSet decorators, typename BarrierSetT>
473 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
474 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
475
476 if (HasDecorator<decorators, IN_NMETHOD>::value) {
477 return ZNMethod::load_oop(p, decorators);
478 } else {
479 return oop_load_not_in_heap((zpointer*)p);
480 }
481 }
482
483 template <DecoratorSet decorators, typename BarrierSetT>
484 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(zpointer* p, oop compare_value, oop new_value) {
485 verify_decorators_present<ON_STRONG_OOP_REF>();
486 verify_decorators_absent<AS_NO_KEEPALIVE>();
487
488 store_barrier_native_with_healing(p);
489
490 const zpointer o = Raw::atomic_cmpxchg(p, store_good(compare_value), store_good(new_value));
491 assert_is_valid(o);
492
493 return to_oop(ZPointer::uncolor_store_good(o));
494 }
495
496 template <DecoratorSet decorators, typename BarrierSetT>
497 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(zpointer* p, oop new_value) {
498 verify_decorators_present<ON_STRONG_OOP_REF>();
499 verify_decorators_absent<AS_NO_KEEPALIVE>();
500
501 store_barrier_native_with_healing(p);
502
503 const zpointer o = Raw::atomic_xchg(p, store_good(new_value));
504 assert_is_valid(o);
505
506 return to_oop(ZPointer::uncolor_store_good(o));
507 }
508
509 #endif // SHARE_GC_Z_ZBARRIERSET_INLINE_HPP