1 /* 2 * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #ifndef SHARE_GC_Z_ZBARRIERSET_INLINE_HPP 25 #define SHARE_GC_Z_ZBARRIERSET_INLINE_HPP 26 27 #include "gc/z/zBarrierSet.hpp" 28 29 #include "gc/shared/accessBarrierSupport.inline.hpp" 30 #include "gc/z/zAddress.inline.hpp" 31 #include "gc/z/zBarrier.inline.hpp" 32 #include "gc/z/zIterator.inline.hpp" 33 #include "gc/z/zNMethod.hpp" 34 #include "memory/iterator.inline.hpp" 35 #include "oops/inlineKlass.inline.hpp" 36 #include "utilities/debug.hpp" 37 38 template <DecoratorSet decorators, typename BarrierSetT> 39 template <DecoratorSet expected> 40 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() { 41 if ((decorators & expected) == 0) { 42 fatal("Using unsupported access decorators"); 43 } 44 } 45 46 template <DecoratorSet decorators, typename BarrierSetT> 47 template <DecoratorSet expected> 48 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() { 49 if ((decorators & expected) != 0) { 50 fatal("Using unsupported access decorators"); 51 } 52 } 53 54 template <DecoratorSet decorators, typename BarrierSetT> 55 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() { 56 ShouldNotReachHere(); 57 } 58 59 template <DecoratorSet decorators, typename BarrierSetT> 60 inline zpointer* ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::field_addr(oop base, ptrdiff_t offset) { 61 assert(base != nullptr, "Invalid base"); 62 return reinterpret_cast<zpointer*>(reinterpret_cast<intptr_t>((void*)base) + offset); 63 } 64 65 template <DecoratorSet decorators, typename BarrierSetT> 66 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier(zpointer* p, zpointer o) { 67 verify_decorators_absent<ON_UNKNOWN_OOP_REF>(); 68 69 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) { 70 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) { 71 // Load barriers on strong oop refs don't keep objects alive 72 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o); 73 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) { 74 return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o); 75 } else { 76 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be"); 77 return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o); 78 } 79 } else { 80 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) { 81 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o); 82 } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) { 83 return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o); 84 } else { 85 assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be"); 86 return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o); 87 } 88 } 89 } 90 91 template <DecoratorSet decorators, typename BarrierSetT> 92 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_unknown_oop_ref(oop base, ptrdiff_t offset, zpointer* p, zpointer o) { 93 verify_decorators_present<ON_UNKNOWN_OOP_REF>(); 94 95 const DecoratorSet decorators_known_strength = 96 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset); 97 98 if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) { 99 if (decorators_known_strength & ON_STRONG_OOP_REF) { 100 // Load barriers on strong oop refs don't keep objects alive 101 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o); 102 } else if (decorators_known_strength & ON_WEAK_OOP_REF) { 103 return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o); 104 } else { 105 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be"); 106 return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o); 107 } 108 } else { 109 if (decorators_known_strength & ON_STRONG_OOP_REF) { 110 return ZBarrier::load_barrier_on_oop_field_preloaded(p, o); 111 } else if (decorators_known_strength & ON_WEAK_OOP_REF) { 112 return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o); 113 } else { 114 assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be"); 115 return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o); 116 } 117 } 118 } 119 120 inline zpointer ZBarrierSet::store_good(oop obj) { 121 assert(ZPointerStoreGoodMask != 0, "sanity"); 122 123 const zaddress addr = to_zaddress(obj); 124 return ZAddress::store_good(addr); 125 } 126 127 template <DecoratorSet decorators, typename BarrierSetT> 128 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_with_healing(zpointer* p) { 129 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) { 130 ZBarrier::store_barrier_on_heap_oop_field(p, true /* heal */); 131 } else { 132 assert(false, "Should not be used on uninitialized memory"); 133 } 134 } 135 136 template <DecoratorSet decorators, typename BarrierSetT> 137 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_without_healing(zpointer* p) { 138 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) { 139 ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */); 140 } 141 } 142 143 template <DecoratorSet decorators, typename BarrierSetT> 144 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::no_keep_alive_store_barrier_heap(zpointer* p) { 145 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) { 146 ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p); 147 } 148 } 149 150 template <DecoratorSet decorators, typename BarrierSetT> 151 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_with_healing(zpointer* p) { 152 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) { 153 ZBarrier::store_barrier_on_native_oop_field(p, true /* heal */); 154 } else { 155 assert(false, "Should not be used on uninitialized memory"); 156 } 157 } 158 159 template <DecoratorSet decorators, typename BarrierSetT> 160 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_without_healing(zpointer* p) { 161 if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) { 162 ZBarrier::store_barrier_on_native_oop_field(p, false /* heal */); 163 } 164 } 165 166 // 167 // In heap 168 // 169 template <DecoratorSet decorators, typename BarrierSetT> 170 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(zpointer* p) { 171 verify_decorators_absent<ON_UNKNOWN_OOP_REF>(); 172 173 const zpointer o = Raw::load_in_heap(p); 174 assert_is_valid(o); 175 176 return to_oop(load_barrier(p, o)); 177 } 178 179 template <DecoratorSet decorators, typename BarrierSetT> 180 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) { 181 zpointer* const p = field_addr(base, offset); 182 183 const zpointer o = Raw::load_in_heap(p); 184 assert_is_valid(o); 185 186 if (HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) { 187 return to_oop(load_barrier_on_unknown_oop_ref(base, offset, p, o)); 188 } 189 190 return to_oop(load_barrier(p, o)); 191 } 192 193 template <DecoratorSet decorators> 194 bool is_store_barrier_no_keep_alive() { 195 if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) { 196 return HasDecorator<decorators, AS_NO_KEEPALIVE>::value; 197 } 198 199 if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) { 200 return true; 201 } 202 203 assert((decorators & ON_PHANTOM_OOP_REF) != 0, "Must be"); 204 return true; 205 } 206 207 template <DecoratorSet decorators> 208 inline bool is_store_barrier_no_keep_alive(oop base, ptrdiff_t offset) { 209 if (!HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) { 210 return is_store_barrier_no_keep_alive<decorators>(); 211 } 212 213 const DecoratorSet decorators_known_strength = 214 AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset); 215 216 if ((decorators_known_strength & ON_STRONG_OOP_REF) != 0) { 217 return (decorators & AS_NO_KEEPALIVE) != 0; 218 } 219 220 if ((decorators_known_strength & ON_WEAK_OOP_REF) != 0) { 221 return true; 222 } 223 224 assert((decorators_known_strength & ON_PHANTOM_OOP_REF) != 0, "Must be"); 225 return true; 226 } 227 228 template <DecoratorSet decorators, typename BarrierSetT> 229 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(zpointer* p, oop value) { 230 verify_decorators_absent<ON_UNKNOWN_OOP_REF>(); 231 232 if (is_store_barrier_no_keep_alive<decorators>()) { 233 no_keep_alive_store_barrier_heap(p); 234 } else { 235 store_barrier_heap_without_healing(p); 236 } 237 238 Raw::store_in_heap(p, store_good(value)); 239 } 240 241 template <DecoratorSet decorators, typename BarrierSetT> 242 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { 243 zpointer* const p = field_addr(base, offset); 244 245 if (is_store_barrier_no_keep_alive<decorators>(base, offset)) { 246 no_keep_alive_store_barrier_heap(p); 247 } else { 248 store_barrier_heap_without_healing(p); 249 } 250 251 Raw::store_in_heap(p, store_good(value)); 252 } 253 254 template <DecoratorSet decorators, typename BarrierSetT> 255 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(zpointer* p, oop value) { 256 verify_decorators_absent<ON_UNKNOWN_OOP_REF>(); 257 258 if (!is_store_barrier_no_keep_alive<decorators>()) { 259 store_barrier_native_without_healing(p); 260 } 261 262 Raw::store(p, store_good(value)); 263 } 264 265 template <DecoratorSet decorators, typename BarrierSetT> 266 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(zpointer* p, oop compare_value, oop new_value) { 267 verify_decorators_present<ON_STRONG_OOP_REF>(); 268 verify_decorators_absent<AS_NO_KEEPALIVE>(); 269 270 store_barrier_heap_with_healing(p); 271 272 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value)); 273 assert_is_valid(o); 274 275 return to_oop(ZPointer::uncolor_store_good(o)); 276 } 277 278 template <DecoratorSet decorators, typename BarrierSetT> 279 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) { 280 verify_decorators_present<ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF>(); 281 verify_decorators_absent<AS_NO_KEEPALIVE>(); 282 283 // Through Unsafe.CompareAndExchangeObject()/CompareAndSetObject() we can receive 284 // calls with ON_UNKNOWN_OOP_REF set. However, we treat these as ON_STRONG_OOP_REF, 285 // with the motivation that if you're doing Unsafe operations on a Reference.referent 286 // field, then you're on your own anyway. 287 zpointer* const p = field_addr(base, offset); 288 289 store_barrier_heap_with_healing(p); 290 291 const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value)); 292 assert_is_valid(o); 293 294 return to_oop(ZPointer::uncolor_store_good(o)); 295 } 296 297 template <DecoratorSet decorators, typename BarrierSetT> 298 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(zpointer* p, oop new_value) { 299 verify_decorators_present<ON_STRONG_OOP_REF>(); 300 verify_decorators_absent<AS_NO_KEEPALIVE>(); 301 302 store_barrier_heap_with_healing(p); 303 304 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value)); 305 assert_is_valid(o); 306 307 return to_oop(ZPointer::uncolor_store_good(o)); 308 } 309 310 template <DecoratorSet decorators, typename BarrierSetT> 311 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) { 312 verify_decorators_present<ON_STRONG_OOP_REF>(); 313 verify_decorators_absent<AS_NO_KEEPALIVE>(); 314 315 zpointer* const p = field_addr(base, offset); 316 317 store_barrier_heap_with_healing(p); 318 319 const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value)); 320 assert_is_valid(o); 321 322 return to_oop(ZPointer::uncolor_store_good(o)); 323 } 324 325 template <DecoratorSet decorators, typename BarrierSetT> 326 inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) { 327 store_barrier_heap_without_healing(dst); 328 329 return ZBarrier::load_barrier_on_oop_field(src); 330 } 331 332 template <DecoratorSet decorators, typename BarrierSetT> 333 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) { 334 const zaddress obj = oop_copy_one_barriers(dst, src); 335 336 if (HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value && is_null(obj)) { 337 return oop_copy_check_null; 338 } 339 340 Atomic::store(dst, ZAddress::store_good(obj)); 341 return oop_copy_check_ok; 342 } 343 344 template <DecoratorSet decorators, typename BarrierSetT> 345 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) { 346 const zaddress obj = oop_copy_one_barriers(dst, src); 347 const bool null_check = HasDecorator<decorators, ARRAYCOPY_NOTNULL>::value; 348 349 if (null_check && is_null(obj)) { 350 return oop_copy_check_null; 351 } 352 else if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) { 353 // Check cast failed 354 return oop_copy_check_class_cast; 355 } 356 357 Atomic::store(dst, ZAddress::store_good(obj)); 358 359 return oop_copy_check_ok; 360 } 361 362 363 template <DecoratorSet decorators, typename BarrierSetT> 364 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) { 365 // Check cast and copy each elements 366 OopCopyCheckStatus check_status = oop_copy_check_ok; 367 for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) { 368 check_status = oop_copy_one_check_cast(dst, src, dst_klass); 369 } 370 return check_status; 371 } 372 373 template <DecoratorSet decorators, typename BarrierSetT> 374 inline ZBarrierSet::OopCopyCheckStatus ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) { 375 const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value; 376 OopCopyCheckStatus check_status = oop_copy_check_ok; 377 if (is_disjoint || src > dst) { 378 for (const zpointer* const end = src + length; (check_status == oop_copy_check_ok) && (src < end); src++, dst++) { 379 check_status = oop_copy_one(dst, src); 380 } 381 return check_status; 382 } 383 384 if (src < dst) { 385 const zpointer* const end = src; 386 src += length - 1; 387 dst += length - 1; 388 for ( ; (check_status == oop_copy_check_ok) && (src >= end); src--, dst--) { 389 check_status = oop_copy_one(dst, src); 390 } 391 return check_status; 392 } 393 394 // src and dst are the same; nothing to do 395 return check_status; 396 } 397 398 template <DecoratorSet decorators, typename BarrierSetT> 399 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw, 400 arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw, 401 size_t length) { 402 zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); 403 zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); 404 OopCopyCheckStatus check_status; 405 406 if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) { 407 Klass* const dst_klass = objArrayOop(dst_obj)->element_klass(); 408 check_status = oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass); 409 } else { 410 check_status = oop_arraycopy_in_heap_no_check_cast(dst, src, length); 411 } 412 413 switch (check_status) { 414 case oop_copy_check_ok: 415 return; 416 case oop_copy_check_class_cast: 417 throw_array_store_exception(src_obj, dst_obj, JavaThread::current()); 418 break; 419 case oop_copy_check_null: 420 throw_array_null_pointer_store_exception(src_obj, dst_obj, JavaThread::current()); 421 break; 422 default: 423 ShouldNotReachHere(); 424 return; 425 } 426 } 427 428 class ZColorStoreGoodOopClosure : public BasicOopIterateClosure { 429 public: 430 virtual void do_oop(oop* p_) { 431 volatile zpointer* const p = (volatile zpointer*)p_; 432 const zpointer ptr = ZBarrier::load_atomic(p); 433 const zaddress addr = ZPointer::uncolor(ptr); 434 Atomic::store(p, ZAddress::store_good(addr)); 435 } 436 437 virtual void do_oop(narrowOop* p) { 438 ShouldNotReachHere(); 439 } 440 }; 441 442 class ZLoadBarrierOopClosure : public BasicOopIterateClosure { 443 public: 444 virtual void do_oop(oop* p) { 445 ZBarrier::load_barrier_on_oop_field((zpointer*)p); 446 } 447 448 virtual void do_oop(narrowOop* p) { 449 ShouldNotReachHere(); 450 } 451 }; 452 453 template <DecoratorSet decorators, typename BarrierSetT> 454 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) { 455 assert_is_valid(to_zaddress(src)); 456 457 if (dst->is_objArray()) { 458 // Cloning an object array is similar to performing array copy. 459 // If an array is large enough to have its allocation segmented, 460 // this operation might require GC barriers. However, the intrinsics 461 // for cloning arrays transform the clone to an optimized allocation 462 // and arraycopy sequence, so the performance of this runtime call 463 // does not matter for object arrays. 464 clone_obj_array(objArrayOop(src), objArrayOop(dst)); 465 return; 466 } 467 468 // Fix the oops 469 ZLoadBarrierOopClosure cl; 470 ZIterator::oop_iterate(src, &cl); 471 472 // Clone the object 473 Raw::clone_in_heap(src, dst, size); 474 475 assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects"); 476 477 // Color store good before handing out 478 ZColorStoreGoodOopClosure cl_sg; 479 ZIterator::oop_iterate(dst, &cl_sg); 480 } 481 482 template <DecoratorSet decorators, typename BarrierSetT> 483 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::value_copy_in_heap(void* src, void* dst, InlineKlass* md) { 484 if (md->contains_oops()) { 485 // src/dst aren't oops, need offset to adjust oop map offset 486 const address src_oop_addr_offset = ((address) src) - md->first_field_offset(); 487 488 OopMapBlock* map = md->start_of_nonstatic_oop_maps(); 489 const OopMapBlock* const end = map + md->nonstatic_oop_map_count(); 490 while (map != end) { 491 const address soop_address = src_oop_addr_offset + map->offset(); 492 zpointer *p = (zpointer*) soop_address; 493 for (const zpointer* const end = p + map->count(); p < end; p++) { 494 ZBarrier::load_barrier_on_oop_field(p); 495 } 496 map++; 497 } 498 } 499 Raw::value_copy_in_heap(src, dst, md); 500 } 501 502 // 503 // Not in heap 504 // 505 template <DecoratorSet decorators, typename BarrierSetT> 506 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) { 507 verify_decorators_absent<ON_UNKNOWN_OOP_REF>(); 508 509 const zpointer o = Raw::template load<zpointer>(p); 510 assert_is_valid(o); 511 return to_oop(load_barrier(p, o)); 512 } 513 514 template <DecoratorSet decorators, typename BarrierSetT> 515 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) { 516 verify_decorators_absent<ON_UNKNOWN_OOP_REF>(); 517 518 if (HasDecorator<decorators, IN_NMETHOD>::value) { 519 return ZNMethod::load_oop(p, decorators); 520 } else { 521 return oop_load_not_in_heap((zpointer*)p); 522 } 523 } 524 525 template <DecoratorSet decorators, typename BarrierSetT> 526 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(zpointer* p, oop compare_value, oop new_value) { 527 verify_decorators_present<ON_STRONG_OOP_REF>(); 528 verify_decorators_absent<AS_NO_KEEPALIVE>(); 529 530 store_barrier_native_with_healing(p); 531 532 const zpointer o = Raw::atomic_cmpxchg(p, store_good(compare_value), store_good(new_value)); 533 assert_is_valid(o); 534 535 return to_oop(ZPointer::uncolor_store_good(o)); 536 } 537 538 template <DecoratorSet decorators, typename BarrierSetT> 539 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(zpointer* p, oop new_value) { 540 verify_decorators_present<ON_STRONG_OOP_REF>(); 541 verify_decorators_absent<AS_NO_KEEPALIVE>(); 542 543 store_barrier_native_with_healing(p); 544 545 const zpointer o = Raw::atomic_xchg(p, store_good(new_value)); 546 assert_is_valid(o); 547 548 return to_oop(ZPointer::uncolor_store_good(o)); 549 } 550 551 #endif // SHARE_GC_Z_ZBARRIERSET_INLINE_HPP