1 /*
  2  * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  */
 23 
 24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
 25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
 26 
 27 #include "gc/z/zBarrierSet.hpp"
 28 
 29 #include "gc/shared/accessBarrierSupport.inline.hpp"
 30 #include "gc/z/zAddress.inline.hpp"
 31 #include "gc/z/zBarrier.inline.hpp"
 32 #include "gc/z/zIterator.inline.hpp"
 33 #include "gc/z/zNMethod.hpp"
 34 #include "memory/iterator.inline.hpp"
 35 #include "oops/inlineKlass.inline.hpp"
 36 #include "utilities/debug.hpp"
 37 
 38 template <DecoratorSet decorators, typename BarrierSetT>
 39 template <DecoratorSet expected>
 40 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
 41   if ((decorators & expected) == 0) {
 42     fatal("Using unsupported access decorators");
 43   }
 44 }
 45 
 46 template <DecoratorSet decorators, typename BarrierSetT>
 47 template <DecoratorSet expected>
 48 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
 49   if ((decorators & expected) != 0) {
 50     fatal("Using unsupported access decorators");
 51   }
 52 }
 53 
 54 template <DecoratorSet decorators, typename BarrierSetT>
 55 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
 56   ShouldNotReachHere();
 57 }
 58 
 59 template <DecoratorSet decorators, typename BarrierSetT>
 60 inline zpointer* ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::field_addr(oop base, ptrdiff_t offset) {
 61   assert(base != nullptr, "Invalid base");
 62   return reinterpret_cast<zpointer*>(reinterpret_cast<intptr_t>((void*)base) + offset);
 63 }
 64 
 65 template <DecoratorSet decorators, typename BarrierSetT>
 66 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier(zpointer* p, zpointer o) {
 67   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
 68 
 69   if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
 70     if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
 71       // Load barriers on strong oop refs don't keep objects alive
 72       return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
 73     } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
 74       return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
 75     } else {
 76       assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
 77       return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
 78     }
 79   } else {
 80     if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
 81       return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
 82     } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
 83       return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
 84     } else {
 85       assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
 86       return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
 87     }
 88   }
 89 }
 90 
 91 template <DecoratorSet decorators, typename BarrierSetT>
 92 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_unknown_oop_ref(oop base, ptrdiff_t offset, zpointer* p, zpointer o) {
 93   verify_decorators_present<ON_UNKNOWN_OOP_REF>();
 94 
 95   const DecoratorSet decorators_known_strength =
 96     AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
 97 
 98   if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
 99     if (decorators_known_strength & ON_STRONG_OOP_REF) {
100       // Load barriers on strong oop refs don't keep objects alive
101       return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
102     } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
103       return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
104     } else {
105       assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
106       return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
107     }
108   } else {
109     if (decorators_known_strength & ON_STRONG_OOP_REF) {
110       return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
111     } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
112       return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
113     } else {
114       assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
115       return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
116     }
117   }
118 }
119 
120 inline zpointer ZBarrierSet::store_good(oop obj) {
121   assert(ZPointerStoreGoodMask != 0, "sanity");
122 
123   const zaddress addr = to_zaddress(obj);
124   return ZAddress::store_good(addr);
125 }
126 
127 template <DecoratorSet decorators, typename BarrierSetT>
128 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_with_healing(zpointer* p) {
129   if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
130     ZBarrier::store_barrier_on_heap_oop_field(p, true /* heal */);
131   } else {
132     assert(false, "Should not be used on uninitialized memory");
133   }
134 }
135 
136 template <DecoratorSet decorators, typename BarrierSetT>
137 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_without_healing(zpointer* p) {
138   if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
139     ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
140   }
141 }
142 
143 template <DecoratorSet decorators, typename BarrierSetT>
144 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::no_keep_alive_store_barrier_heap(zpointer* p) {
145   if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
146     ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p);
147   }
148 }
149 
150 template <DecoratorSet decorators, typename BarrierSetT>
151 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_with_healing(zpointer* p) {
152   if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
153     ZBarrier::store_barrier_on_native_oop_field(p, true /* heal */);
154   } else {
155     assert(false, "Should not be used on uninitialized memory");
156   }
157 }
158 
159 template <DecoratorSet decorators, typename BarrierSetT>
160 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_without_healing(zpointer* p) {
161   if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
162     ZBarrier::store_barrier_on_native_oop_field(p, false /* heal */);
163   }
164 }
165 
166 //
167 // In heap
168 //
169 template <DecoratorSet decorators, typename BarrierSetT>
170 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(zpointer* p) {
171   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
172 
173   const zpointer o = Raw::load_in_heap(p);
174   assert_is_valid(o);
175 
176   return to_oop(load_barrier(p, o));
177 }
178 
179 template <DecoratorSet decorators, typename BarrierSetT>
180 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
181   zpointer* const p = field_addr(base, offset);
182 
183   const zpointer o = Raw::load_in_heap(p);
184   assert_is_valid(o);
185 
186   if (HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
187     return to_oop(load_barrier_on_unknown_oop_ref(base, offset, p, o));
188   }
189 
190   return to_oop(load_barrier(p, o));
191 }
192 
193 template <DecoratorSet decorators>
194 bool is_store_barrier_no_keep_alive() {
195   if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
196     return HasDecorator<decorators, AS_NO_KEEPALIVE>::value;
197   }
198 
199   if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
200     return true;
201   }
202 
203   assert((decorators & ON_PHANTOM_OOP_REF) != 0, "Must be");
204   return true;
205 }
206 
207 template <DecoratorSet decorators>
208 inline bool is_store_barrier_no_keep_alive(oop base, ptrdiff_t offset) {
209   if (!HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
210     return is_store_barrier_no_keep_alive<decorators>();
211   }
212 
213   const DecoratorSet decorators_known_strength =
214       AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
215 
216   if ((decorators_known_strength & ON_STRONG_OOP_REF) != 0) {
217     return (decorators & AS_NO_KEEPALIVE) != 0;
218   }
219 
220   if ((decorators_known_strength & ON_WEAK_OOP_REF) != 0) {
221     return true;
222   }
223 
224   assert((decorators_known_strength & ON_PHANTOM_OOP_REF) != 0, "Must be");
225   return true;
226 }
227 
228 template <DecoratorSet decorators, typename BarrierSetT>
229 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(zpointer* p, oop value) {
230   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
231 
232   if (is_store_barrier_no_keep_alive<decorators>()) {
233     no_keep_alive_store_barrier_heap(p);
234   } else {
235     store_barrier_heap_without_healing(p);
236   }
237 
238   Raw::store_in_heap(p, store_good(value));
239 }
240 
241 template <DecoratorSet decorators, typename BarrierSetT>
242 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
243   zpointer* const p = field_addr(base, offset);
244 
245   if (is_store_barrier_no_keep_alive<decorators>(base, offset)) {
246     no_keep_alive_store_barrier_heap(p);
247   } else {
248     store_barrier_heap_without_healing(p);
249   }
250 
251   Raw::store_in_heap(p, store_good(value));
252 }
253 
254 template <DecoratorSet decorators, typename BarrierSetT>
255 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(zpointer* p, oop value) {
256   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
257 
258   if (!is_store_barrier_no_keep_alive<decorators>()) {
259     store_barrier_native_without_healing(p);
260   }
261 
262   Raw::store(p, store_good(value));
263 }
264 
265 template <DecoratorSet decorators, typename BarrierSetT>
266 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(zpointer* p, oop compare_value, oop new_value) {
267   verify_decorators_present<ON_STRONG_OOP_REF>();
268   verify_decorators_absent<AS_NO_KEEPALIVE>();
269 
270   store_barrier_heap_with_healing(p);
271 
272   const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
273   assert_is_valid(o);
274 
275   return to_oop(ZPointer::uncolor_store_good(o));
276 }
277 
278 template <DecoratorSet decorators, typename BarrierSetT>
279 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) {
280   verify_decorators_present<ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF>();
281   verify_decorators_absent<AS_NO_KEEPALIVE>();
282 
283   // Through Unsafe.CompareAndExchangeObject()/CompareAndSetObject() we can receive
284   // calls with ON_UNKNOWN_OOP_REF set. However, we treat these as ON_STRONG_OOP_REF,
285   // with the motivation that if you're doing Unsafe operations on a Reference.referent
286   // field, then you're on your own anyway.
287   zpointer* const p = field_addr(base, offset);
288 
289   store_barrier_heap_with_healing(p);
290 
291   const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
292   assert_is_valid(o);
293 
294   return to_oop(ZPointer::uncolor_store_good(o));
295 }
296 
297 template <DecoratorSet decorators, typename BarrierSetT>
298 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(zpointer* p, oop new_value) {
299   verify_decorators_present<ON_STRONG_OOP_REF>();
300   verify_decorators_absent<AS_NO_KEEPALIVE>();
301 
302   store_barrier_heap_with_healing(p);
303 
304   const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
305   assert_is_valid(o);
306 
307   return to_oop(ZPointer::uncolor_store_good(o));
308 }
309 
310 template <DecoratorSet decorators, typename BarrierSetT>
311 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
312   verify_decorators_present<ON_STRONG_OOP_REF>();
313   verify_decorators_absent<AS_NO_KEEPALIVE>();
314 
315   zpointer* const p = field_addr(base, offset);
316 
317   store_barrier_heap_with_healing(p);
318 
319   const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
320   assert_is_valid(o);
321 
322   return to_oop(ZPointer::uncolor_store_good(o));
323 }
324 
325 template <DecoratorSet decorators, typename BarrierSetT>
326 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
327   store_barrier_heap_without_healing(dst);
328 
329   return ZBarrier::load_barrier_on_oop_field(src);
330 }
331 
332 template <DecoratorSet decorators, typename BarrierSetT>
333 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
334   const zaddress obj = oop_copy_one_barriers(dst, src);
335 
336   if (HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value && is_null(obj)) {
337     return oop_copy_check_null;
338   }
339 
340   Atomic::store(dst, ZAddress::store_good(obj));
341   return oop_copy_check_ok;
342 }
343 
344 template <DecoratorSet decorators, typename BarrierSetT>
345 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
346   const zaddress obj = oop_copy_one_barriers(dst, src);
347   const bool null_check = HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value;
348 
349   if (null_check && is_null(obj)) {
350     return oop_copy_check_null;
351   }
352   else if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
353     // Check cast failed
354     return oop_copy_check_class_cast;
355   }
356 
357   Atomic::store(dst, ZAddress::store_good(obj));
358 
359   return oop_copy_check_ok;
360 }
361 
362 
363 template <DecoratorSet decorators, typename BarrierSetT>
364 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
365   // Check cast and copy each elements
366   OopCopyCheckStatus check_status = oop_copy_check_ok;
367   for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
368     check_status = oop_copy_one_check_cast(dst, src, dst_klass);
369   }
370   return check_status;
371 }
372 
373 template <DecoratorSet decorators, typename BarrierSetT>
374 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
375   const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
376   OopCopyCheckStatus check_status = oop_copy_check_ok;
377   if (is_disjoint || src > dst) {
378     for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) {
379       check_status = oop_copy_one(dst, src);
380     }
381     return check_status;
382   }
383 
384   if (src < dst) {
385     const zpointer* const end = src;
386     src += length - 1;
387     dst += length - 1;
388     for ( ; (check_status == oop_copy_check_ok) && (src >= end); src--, dst--) {
389       check_status = oop_copy_one(dst, src);
390     }
391     return check_status;
392   }
393 
394   // src and dst are the same; nothing to do
395   return check_status;
396 }
397 
398 template <DecoratorSet decorators, typename BarrierSetT>
399 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
400                                                                                        arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
401                                                                                        size_t length) {
402   zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
403   zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
404   OopCopyCheckStatus check_status;
405 
406   if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
407     Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
408     check_status = oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
409   } else {
410     check_status = oop_arraycopy_in_heap_no_check_cast(dst, src, length);
411   }
412 
413   switch (check_status) {
414   case oop_copy_check_ok:
415     return;
416   case oop_copy_check_class_cast:
417     throw_array_store_exception(src_obj, dst_obj, JavaThread::current());
418     break;
419   case oop_copy_check_null:
420     throw_array_null_pointer_store_exception(src_obj, dst_obj, JavaThread::current());
421     break;
422   default:
423     ShouldNotReachHere();
424     return;
425   }
426 }
427 
428 class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
429 public:
430   virtual void do_oop(oop* p_) {
431     volatile zpointer* const p = (volatile zpointer*)p_;
432     const zpointer ptr = ZBarrier::load_atomic(p);
433     const zaddress addr = ZPointer::uncolor(ptr);
434     Atomic::store(p, ZAddress::store_good(addr));
435   }
436 
437   virtual void do_oop(narrowOop* p) {
438     ShouldNotReachHere();
439   }
440 };
441 
442 class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
443 public:
444   virtual void do_oop(oop* p) {
445     ZBarrier::load_barrier_on_oop_field((zpointer*)p);
446   }
447 
448   virtual void do_oop(narrowOop* p) {
449     ShouldNotReachHere();
450   }
451 };
452 
453 template <DecoratorSet decorators, typename BarrierSetT>
454 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
455   assert_is_valid(to_zaddress(src));
456 
457   if (dst->is_objArray()) {
458     // Cloning an object array is similar to performing array copy.
459     // If an array is large enough to have its allocation segmented,
460     // this operation might require GC barriers. However, the intrinsics
461     // for cloning arrays transform the clone to an optimized allocation
462     // and arraycopy sequence, so the performance of this runtime call
463     // does not matter for object arrays.
464     clone_obj_array(objArrayOop(src), objArrayOop(dst));
465     return;
466   }
467 
468   // Fix the oops
469   ZLoadBarrierOopClosure cl;
470   ZIterator::oop_iterate(src, &cl);
471 
472   // Clone the object
473   Raw::clone_in_heap(src, dst, size);
474 
475   assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
476 
477   // Color store good before handing out
478   ZColorStoreGoodOopClosure cl_sg;
479   ZIterator::oop_iterate(dst, &cl_sg);
480 }
481 
482 template <DecoratorSet decorators, typename BarrierSetT>
483 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(void* src, void* dst, InlineKlass* md) {
484   if (md->contains_oops()) {
485     // src/dst aren't oops, need offset to adjust oop map offset
486     const address src_oop_addr_offset = ((address) src) - md->first_field_offset();
487 
488     OopMapBlock* map = md->start_of_nonstatic_oop_maps();
489     const OopMapBlock* const end = map + md->nonstatic_oop_map_count();
490     while (map != end) {
491       const address soop_address = src_oop_addr_offset + map->offset();
492       zpointer *p = (zpointer*) soop_address;
493       for (const zpointer* const end = p + map->count(); p < end; p++) {
494         ZBarrier::load_barrier_on_oop_field(p);
495       }
496       map++;
497     }
498   }
499   Raw::value_copy_in_heap(src, dst, md);
500 }
501 
502 //
503 // Not in heap
504 //
505 template <DecoratorSet decorators, typename BarrierSetT>
506 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
507   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
508 
509   const zpointer o = Raw::template load<zpointer>(p);
510   assert_is_valid(o);
511   return to_oop(load_barrier(p, o));
512 }
513 
514 template <DecoratorSet decorators, typename BarrierSetT>
515 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
516   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
517 
518   if (HasDecorator<decorators, IN_NMETHOD>::value) {
519     return ZNMethod::load_oop(p, decorators);
520   } else {
521     return oop_load_not_in_heap((zpointer*)p);
522   }
523 }
524 
525 template <DecoratorSet decorators, typename BarrierSetT>
526 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(zpointer* p, oop compare_value, oop new_value) {
527   verify_decorators_present<ON_STRONG_OOP_REF>();
528   verify_decorators_absent<AS_NO_KEEPALIVE>();
529 
530   store_barrier_native_with_healing(p);
531 
532   const zpointer o = Raw::atomic_cmpxchg(p, store_good(compare_value), store_good(new_value));
533   assert_is_valid(o);
534 
535   return to_oop(ZPointer::uncolor_store_good(o));
536 }
537 
538 template <DecoratorSet decorators, typename BarrierSetT>
539 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(zpointer* p, oop new_value) {
540   verify_decorators_present<ON_STRONG_OOP_REF>();
541   verify_decorators_absent<AS_NO_KEEPALIVE>();
542 
543   store_barrier_native_with_healing(p);
544 
545   const zpointer o = Raw::atomic_xchg(p, store_good(new_value));
546   assert_is_valid(o);
547 
548   return to_oop(ZPointer::uncolor_store_good(o));
549 }
550 
551 #endif // SHARE_GC_Z_ZBARRIERSET_INLINE_HPP