1 /*
2 * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
26
27 #include "gc/z/zBarrierSet.hpp"
28
29 #include "gc/shared/accessBarrierSupport.inline.hpp"
30 #include "gc/z/zAddress.inline.hpp"
31 #include "gc/z/zBarrier.inline.hpp"
32 #include "gc/z/zIterator.inline.hpp"
33 #include "gc/z/zNMethod.hpp"
34 #include "memory/iterator.inline.hpp"
35 #include "oops/inlineKlass.inline.hpp"
36 #include "utilities/debug.hpp"
37
38 template <DecoratorSet decorators, typename BarrierSetT>
39 template <DecoratorSet expected>
40 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
41 if ((decorators & expected) == 0) {
42 fatal("Using unsupported access decorators");
43 }
44 }
45
46 template <DecoratorSet decorators, typename BarrierSetT>
47 template <DecoratorSet expected>
48 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
49 if ((decorators & expected) != 0) {
50 fatal("Using unsupported access decorators");
51 }
52 }
53
54 template <DecoratorSet decorators, typename BarrierSetT>
55 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
56 ShouldNotReachHere();
57 }
58
59 template <DecoratorSet decorators, typename BarrierSetT>
60 inline zpointer* ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::field_addr(oop base, ptrdiff_t offset) {
61 assert(base != nullptr, "Invalid base");
62 return reinterpret_cast<zpointer*>(reinterpret_cast<intptr_t>((void*)base) + offset);
63 }
64
65 template <DecoratorSet decorators, typename BarrierSetT>
66 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier(zpointer* p, zpointer o) {
67 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
68
69 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
70 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
71 // Load barriers on strong oop refs don't keep objects alive
72 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
73 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
74 return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
75 } else {
76 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
77 return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
78 }
79 } else {
80 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
81 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
82 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
83 return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
84 } else {
85 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
86 return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
87 }
88 }
89 }
90
91 template <DecoratorSet decorators, typename BarrierSetT>
92 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_unknown_oop_ref(oop base, ptrdiff_t offset, zpointer* p, zpointer o) {
93 verify_decorators_present<ON_UNKNOWN_OOP_REF>();
94
95 const DecoratorSet decorators_known_strength =
96 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
97
98 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
99 if (decorators_known_strength & ON_STRONG_OOP_REF) {
100 // Load barriers on strong oop refs don't keep objects alive
101 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
102 } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
103 return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
104 } else {
105 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
106 return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
107 }
108 } else {
109 if (decorators_known_strength & ON_STRONG_OOP_REF) {
110 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
111 } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
112 return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
113 } else {
114 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
115 return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
116 }
117 }
118 }
119
120 inline zpointer ZBarrierSet::store_good(oop obj) {
121 assert(ZPointerStoreGoodMask != 0, "sanity");
122
123 const zaddress addr = to_zaddress(obj);
124 return ZAddress::store_good(addr);
125 }
126
127 template <DecoratorSet decorators, typename BarrierSetT>
128 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_with_healing(zpointer* p) {
129 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
130 ZBarrier::store_barrier_on_heap_oop_field(p, true /* heal */);
131 } else {
132 assert(false, "Should not be used on uninitialized memory");
133 }
134 }
135
136 template <DecoratorSet decorators, typename BarrierSetT>
137 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_without_healing(zpointer* p) {
138 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
139 ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
140 }
141 }
142
143 template <DecoratorSet decorators, typename BarrierSetT>
144 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::no_keep_alive_store_barrier_heap(zpointer* p) {
145 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
146 ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p);
147 }
148 }
149
150 template <DecoratorSet decorators, typename BarrierSetT>
151 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_with_healing(zpointer* p) {
152 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
153 ZBarrier::store_barrier_on_native_oop_field(p, true /* heal */);
154 } else {
155 assert(false, "Should not be used on uninitialized memory");
156 }
157 }
158
159 template <DecoratorSet decorators, typename BarrierSetT>
160 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_without_healing(zpointer* p) {
161 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
162 ZBarrier::store_barrier_on_native_oop_field(p, false /* heal */);
163 }
164 }
165
166 //
167 // In heap
168 //
169 template <DecoratorSet decorators, typename BarrierSetT>
170 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(zpointer* p) {
171 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
172
173 const zpointer o = Raw::load_in_heap(p);
174 assert_is_valid(o);
175
176 return to_oop(load_barrier(p, o));
177 }
178
179 template <DecoratorSet decorators, typename BarrierSetT>
180 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
181 zpointer* const p = field_addr(base, offset);
182
183 const zpointer o = Raw::load_in_heap(p);
184 assert_is_valid(o);
185
186 if (HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
187 return to_oop(load_barrier_on_unknown_oop_ref(base, offset, p, o));
188 }
189
190 return to_oop(load_barrier(p, o));
191 }
192
193 template <DecoratorSet decorators>
194 bool is_store_barrier_no_keep_alive() {
195 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
196 return HasDecorator<decorators, AS_NO_KEEPALIVE>::value;
197 }
198
199 if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
200 return true;
201 }
202
203 assert((decorators & ON_PHANTOM_OOP_REF) != 0, "Must be");
204 return true;
205 }
206
207 template <DecoratorSet decorators>
208 inline bool is_store_barrier_no_keep_alive(oop base, ptrdiff_t offset) {
209 if (!HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
210 return is_store_barrier_no_keep_alive<decorators>();
211 }
212
213 const DecoratorSet decorators_known_strength =
214 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
215
216 if ((decorators_known_strength & ON_STRONG_OOP_REF) != 0) {
217 return (decorators & AS_NO_KEEPALIVE) != 0;
218 }
219
220 if ((decorators_known_strength & ON_WEAK_OOP_REF) != 0) {
221 return true;
222 }
223
224 assert((decorators_known_strength & ON_PHANTOM_OOP_REF) != 0, "Must be");
225 return true;
226 }
227
228 template <DecoratorSet decorators, typename BarrierSetT>
229 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(zpointer* p, oop value) {
230 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
231
232 if (is_store_barrier_no_keep_alive<decorators>()) {
233 no_keep_alive_store_barrier_heap(p);
234 } else {
235 store_barrier_heap_without_healing(p);
236 }
237
238 Raw::store_in_heap(p, store_good(value));
239 }
240
241 template <DecoratorSet decorators, typename BarrierSetT>
242 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
243 zpointer* const p = field_addr(base, offset);
244
245 if (is_store_barrier_no_keep_alive<decorators>(base, offset)) {
246 no_keep_alive_store_barrier_heap(p);
247 } else {
248 store_barrier_heap_without_healing(p);
249 }
250
251 Raw::store_in_heap(p, store_good(value));
252 }
253
254 template <DecoratorSet decorators, typename BarrierSetT>
255 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(zpointer* p, oop value) {
256 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
257
258 if (!is_store_barrier_no_keep_alive<decorators>()) {
259 store_barrier_native_without_healing(p);
260 }
261
262 Raw::store(p, store_good(value));
263 }
264
265 template <DecoratorSet decorators, typename BarrierSetT>
266 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(zpointer* p, oop compare_value, oop new_value) {
267 verify_decorators_present<ON_STRONG_OOP_REF>();
268 verify_decorators_absent<AS_NO_KEEPALIVE>();
269
270 store_barrier_heap_with_healing(p);
271
272 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
273 assert_is_valid(o);
274
275 return to_oop(ZPointer::uncolor_store_good(o));
276 }
277
278 template <DecoratorSet decorators, typename BarrierSetT>
279 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) {
280 verify_decorators_present<ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF>();
281 verify_decorators_absent<AS_NO_KEEPALIVE>();
282
283 // Through Unsafe.CompareAndExchangeObject()/CompareAndSetObject() we can receive
284 // calls with ON_UNKNOWN_OOP_REF set. However, we treat these as ON_STRONG_OOP_REF,
285 // with the motivation that if you're doing Unsafe operations on a Reference.referent
286 // field, then you're on your own anyway.
287 zpointer* const p = field_addr(base, offset);
288
289 store_barrier_heap_with_healing(p);
290
291 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
292 assert_is_valid(o);
293
294 return to_oop(ZPointer::uncolor_store_good(o));
295 }
296
297 template <DecoratorSet decorators, typename BarrierSetT>
298 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(zpointer* p, oop new_value) {
299 verify_decorators_present<ON_STRONG_OOP_REF>();
300 verify_decorators_absent<AS_NO_KEEPALIVE>();
301
302 store_barrier_heap_with_healing(p);
303
304 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
305 assert_is_valid(o);
306
307 return to_oop(ZPointer::uncolor_store_good(o));
308 }
309
310 template <DecoratorSet decorators, typename BarrierSetT>
311 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
312 verify_decorators_present<ON_STRONG_OOP_REF>();
313 verify_decorators_absent<AS_NO_KEEPALIVE>();
314
315 zpointer* const p = field_addr(base, offset);
316
317 store_barrier_heap_with_healing(p);
318
319 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
320 assert_is_valid(o);
321
322 return to_oop(ZPointer::uncolor_store_good(o));
323 }
324
325 template <DecoratorSet decorators, typename BarrierSetT>
326 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
327 store_barrier_heap_without_healing(dst);
328
329 return ZBarrier::load_barrier_on_oop_field(src);
330 }
331
332 template <DecoratorSet decorators, typename BarrierSetT>
333 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
334 const zaddress obj = oop_copy_one_barriers(dst, src);
335
336 if (HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value && is_null(obj)) {
337 return oop_copy_check_null;
338 }
339
340 Atomic::store(dst, ZAddress::store_good(obj));
341 return oop_copy_check_ok;
342 }
343
344 template <DecoratorSet decorators, typename BarrierSetT>
345 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
346 const zaddress obj = oop_copy_one_barriers(dst, src);
347 const bool null_check = HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value;
348
349 if (null_check && is_null(obj)) {
350 return oop_copy_check_null;
351 }
352 else if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
353 // Check cast failed
354 return oop_copy_check_class_cast;
355 }
356
357 Atomic::store(dst, ZAddress::store_good(obj));
358
359 return oop_copy_check_ok;
360 }
361
362
363 template <DecoratorSet decorators, typename BarrierSetT>
364 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
365 // Check cast and copy each elements
366 OopCopyCheckStatus check_status = oop_copy_check_ok;
367 for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
368 check_status = oop_copy_one_check_cast(dst, src, dst_klass);
369 }
370 return check_status;
371 }
372
373 template <DecoratorSet decorators, typename BarrierSetT>
374 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
375 const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
376 OopCopyCheckStatus check_status = oop_copy_check_ok;
377 if (is_disjoint || src > dst) {
378 for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
379 check_status = oop_copy_one(dst, src);
380 }
381 return check_status;
382 }
383
384 if (src < dst) {
385 const zpointer* const end = src;
386 src += length - 1;
387 dst += length - 1;
388 for ( ; (check_status == oop_copy_check_ok) && (src >= end); src--, dst--) {
389 check_status = oop_copy_one(dst, src);
390 }
391 return check_status;
392 }
393
394 // src and dst are the same; nothing to do
395 return check_status;
396 }
397
398 template <DecoratorSet decorators, typename BarrierSetT>
399 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
400 arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
401 size_t length) {
402 zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
403 zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
404 OopCopyCheckStatus check_status;
405
406 if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
407 Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
408 check_status = oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
409 } else {
410 check_status = oop_arraycopy_in_heap_no_check_cast(dst, src, length);
411 }
412
413 switch (check_status) {
414 case oop_copy_check_ok:
415 return;
416 case oop_copy_check_class_cast:
417 throw_array_store_exception(src_obj, dst_obj, JavaThread::current());
418 break;
419 case oop_copy_check_null:
420 throw_array_null_pointer_store_exception(src_obj, dst_obj, JavaThread::current());
421 break;
422 default:
423 ShouldNotReachHere();
424 return;
425 }
426 }
427
428 class ZStoreBarrierOopClosure : public BasicOopIterateClosure {
429 public:
430 virtual void do_oop(oop* p_) {
431 volatile zpointer* const p = (volatile zpointer*)p_;
432 const zpointer ptr = ZBarrier::load_atomic(p);
433 const zaddress addr = ZPointer::uncolor(ptr);
434 ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
435 *p = ZAddress::store_good(addr);
436 }
437
438 virtual void do_oop(narrowOop* p) {
439 ShouldNotReachHere();
440 }
441 };
442
443 class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
444 public:
445 virtual void do_oop(oop* p) {
446 ZBarrier::load_barrier_on_oop_field((zpointer*)p);
447 }
448
449 virtual void do_oop(narrowOop* p) {
450 ShouldNotReachHere();
451 }
452 };
453
454 template <DecoratorSet decorators, typename BarrierSetT>
455 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
456 assert_is_valid(to_zaddress(src));
457
458 // Fix the oops
459 ZLoadBarrierOopClosure cl;
460 ZIterator::oop_iterate(src, &cl);
461
462 // Clone the object
463 Raw::clone_in_heap(src, dst, size);
464
465 assert(ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
466
467 // Color store good before handing out
468 ZStoreBarrierOopClosure cl_sg;
469 ZIterator::oop_iterate(dst, &cl_sg);
470 }
471
472 template <DecoratorSet decorators, typename BarrierSetT>
473 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(void* src, void* dst, InlineKlass* md) {
474 if (md->contains_oops()) {
475 // src/dst aren't oops, need offset to adjust oop map offset
476 const address src_oop_addr_offset = ((address) src) - md->first_field_offset();
477
478 OopMapBlock* map = md->start_of_nonstatic_oop_maps();
479 const OopMapBlock* const end = map + md->nonstatic_oop_map_count();
480 while (map != end) {
481 const address soop_address = src_oop_addr_offset + map->offset();
482 zpointer *p = (zpointer*) soop_address;
483 for (const zpointer* const end = p + map->count(); p < end; p++) {
484 ZBarrier::load_barrier_on_oop_field(p);
485 }
486 map++;
487 }
488 }
489 Raw::value_copy_in_heap(src, dst, md);
490 }
491
492 //
493 // Not in heap
494 //
495 template <DecoratorSet decorators, typename BarrierSetT>
496 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
497 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
498
499 const zpointer o = Raw::template load<zpointer>(p);
500 assert_is_valid(o);
501 return to_oop(load_barrier(p, o));
502 }
503
504 template <DecoratorSet decorators, typename BarrierSetT>
505 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
506 verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
507
508 if (HasDecorator<decorators, IN_NMETHOD>::value) {
509 return ZNMethod::load_oop(p, decorators);
510 } else {
511 return oop_load_not_in_heap((zpointer*)p);
512 }
513 }
514
515 template <DecoratorSet decorators, typename BarrierSetT>
516 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(zpointer* p, oop compare_value, oop new_value) {
517 verify_decorators_present<ON_STRONG_OOP_REF>();
518 verify_decorators_absent<AS_NO_KEEPALIVE>();
519
520 store_barrier_native_with_healing(p);
521
522 const zpointer o = Raw::atomic_cmpxchg(p, store_good(compare_value), store_good(new_value));
523 assert_is_valid(o);
524
525 return to_oop(ZPointer::uncolor_store_good(o));
526 }
527
528 template <DecoratorSet decorators, typename BarrierSetT>
529 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(zpointer* p, oop new_value) {
530 verify_decorators_present<ON_STRONG_OOP_REF>();
531 verify_decorators_absent<AS_NO_KEEPALIVE>();
532
533 store_barrier_native_with_healing(p);
534
535 const zpointer o = Raw::atomic_xchg(p, store_good(new_value));
536 assert_is_valid(o);
537
538 return to_oop(ZPointer::uncolor_store_good(o));
539 }
540
541 #endif // SHARE_GC_Z_ZBARRIERSET_INLINE_HPP