1 /* 2 * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef SHARE_OOPS_ACCESSBACKEND_HPP 26 #define SHARE_OOPS_ACCESSBACKEND_HPP 27 28 #include "gc/shared/barrierSetConfig.hpp" 29 #include "memory/allocation.hpp" 30 #include "metaprogramming/enableIf.hpp" 31 #include "oops/accessDecorators.hpp" 32 #include "oops/oopsHierarchy.hpp" 33 #include "runtime/globals.hpp" 34 #include "utilities/debug.hpp" 35 #include "utilities/globalDefinitions.hpp" 36 37 #include <type_traits> 38 39 // This metafunction returns either oop or narrowOop depending on whether 40 // an access needs to use compressed oops or not. 41 template <DecoratorSet decorators> 42 struct HeapOopType: AllStatic { 43 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value && 44 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value; 45 using type = std::conditional_t<needs_oop_compress, narrowOop, oop>; 46 }; 47 48 namespace AccessInternal { 49 enum BarrierType { 50 BARRIER_STORE, 51 BARRIER_STORE_AT, 52 BARRIER_LOAD, 53 BARRIER_LOAD_AT, 54 BARRIER_ATOMIC_CMPXCHG, 55 BARRIER_ATOMIC_CMPXCHG_AT, 56 BARRIER_ATOMIC_XCHG, 57 BARRIER_ATOMIC_XCHG_AT, 58 BARRIER_ARRAYCOPY, 59 BARRIER_CLONE 60 }; 61 62 template <DecoratorSet decorators, typename T> 63 struct MustConvertCompressedOop: public std::integral_constant<bool, 64 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value && 65 std::is_same<typename HeapOopType<decorators>::type, narrowOop>::value && 66 std::is_same<T, oop>::value> {}; 67 68 // This metafunction returns an appropriate oop type if the value is oop-like 69 // and otherwise returns the same type T. 70 template <DecoratorSet decorators, typename T> 71 struct EncodedType: AllStatic { 72 using type = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value, 73 typename HeapOopType<decorators>::type, 74 T>; 75 }; 76 77 template <DecoratorSet decorators> 78 inline typename HeapOopType<decorators>::type* 79 oop_field_addr(oop base, ptrdiff_t byte_offset) { 80 return reinterpret_cast<typename HeapOopType<decorators>::type*>( 81 reinterpret_cast<intptr_t>((void*)base) + byte_offset); 82 } 83 84 // This metafunction returns whether it is possible for a type T to require 85 // locking to support wide atomics or not. 86 template <typename T> 87 #ifdef SUPPORTS_NATIVE_CX8 88 struct PossiblyLockedAccess: public std::false_type {}; 89 #else 90 struct PossiblyLockedAccess: public std::integral_constant<bool, (sizeof(T) > 4)> {}; 91 #endif 92 93 template <DecoratorSet decorators, typename T> 94 struct AccessFunctionTypes { 95 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset); 96 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value); 97 typedef T (*atomic_cmpxchg_at_func_t)(oop base, ptrdiff_t offset, T compare_value, T new_value); 98 typedef T (*atomic_xchg_at_func_t)(oop base, ptrdiff_t offset, T new_value); 99 100 typedef T (*load_func_t)(void* addr); 101 typedef void (*store_func_t)(void* addr, T value); 102 typedef T (*atomic_cmpxchg_func_t)(void* addr, T compare_value, T new_value); 103 typedef T (*atomic_xchg_func_t)(void* addr, T new_value); 104 105 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 106 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 107 size_t length); 108 typedef void (*clone_func_t)(oop src, oop dst, size_t size); 109 }; 110 111 template <DecoratorSet decorators> 112 struct AccessFunctionTypes<decorators, void> { 113 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src, 114 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst, 115 size_t length); 116 }; 117 118 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {}; 119 120 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \ 121 template <DecoratorSet decorators, typename T> \ 122 struct AccessFunction<decorators, T, bt>: AllStatic{ \ 123 typedef typename AccessFunctionTypes<decorators, T>::func type; \ 124 } 125 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t); 126 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t); 127 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t); 128 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t); 129 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t); 130 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t); 131 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t); 132 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t); 133 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t); 134 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t); 135 #undef ACCESS_GENERATE_ACCESS_FUNCTION 136 137 template <DecoratorSet decorators, typename T, BarrierType barrier_type> 138 typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier(); 139 140 template <DecoratorSet decorators, typename T, BarrierType barrier_type> 141 typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier(); 142 143 class AccessLocker { 144 public: 145 AccessLocker(); 146 ~AccessLocker(); 147 }; 148 bool wide_atomic_needs_locking(); 149 150 void* field_addr(oop base, ptrdiff_t offset); 151 152 // Forward calls to Copy:: in the cpp file to reduce dependencies and allow 153 // faster build times, given how frequently included access is. 154 void arraycopy_arrayof_conjoint_oops(void* src, void* dst, size_t length); 155 void arraycopy_conjoint_oops(oop* src, oop* dst, size_t length); 156 void arraycopy_conjoint_oops(narrowOop* src, narrowOop* dst, size_t length); 157 158 void arraycopy_disjoint_words(void* src, void* dst, size_t length); 159 void arraycopy_disjoint_words_atomic(void* src, void* dst, size_t length); 160 161 template<typename T> 162 void arraycopy_conjoint(T* src, T* dst, size_t length); 163 template<typename T> 164 void arraycopy_arrayof_conjoint(T* src, T* dst, size_t length); 165 template<typename T> 166 void arraycopy_conjoint_atomic(T* src, T* dst, size_t length); 167 } 168 169 // This mask specifies what decorators are relevant for raw accesses. When passing 170 // accesses to the raw layer, irrelevant decorators are removed. 171 const DecoratorSet RAW_DECORATOR_MASK = INTERNAL_DECORATOR_MASK | MO_DECORATOR_MASK | 172 ARRAYCOPY_DECORATOR_MASK | IS_NOT_NULL; 173 174 // The RawAccessBarrier performs raw accesses with additional knowledge of 175 // memory ordering, so that OrderAccess/Atomic is called when necessary. 176 // It additionally handles compressed oops, and hence is not completely "raw" 177 // strictly speaking. 178 template <DecoratorSet decorators> 179 class RawAccessBarrier: public AllStatic { 180 protected: 181 static inline void* field_addr(oop base, ptrdiff_t byte_offset) { 182 return AccessInternal::field_addr(base, byte_offset); 183 } 184 185 protected: 186 // Only encode if INTERNAL_VALUE_IS_OOP 187 template <DecoratorSet idecorators, typename T> 188 static inline typename EnableIf< 189 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, 190 typename HeapOopType<idecorators>::type>::type 191 encode_internal(T value); 192 193 template <DecoratorSet idecorators, typename T> 194 static inline typename EnableIf< 195 !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type 196 encode_internal(T value) { 197 return value; 198 } 199 200 template <typename T> 201 static inline typename AccessInternal::EncodedType<decorators, T>::type 202 encode(T value) { 203 return encode_internal<decorators, T>(value); 204 } 205 206 // Only decode if INTERNAL_VALUE_IS_OOP 207 template <DecoratorSet idecorators, typename T> 208 static inline typename EnableIf< 209 AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type 210 decode_internal(typename HeapOopType<idecorators>::type value); 211 212 template <DecoratorSet idecorators, typename T> 213 static inline typename EnableIf< 214 !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type 215 decode_internal(T value) { 216 return value; 217 } 218 219 template <typename T> 220 static inline T decode(typename AccessInternal::EncodedType<decorators, T>::type value) { 221 return decode_internal<decorators, T>(value); 222 } 223 224 protected: 225 template <DecoratorSet ds, typename T> 226 static typename EnableIf< 227 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 228 load_internal(void* addr); 229 230 template <DecoratorSet ds, typename T> 231 static typename EnableIf< 232 HasDecorator<ds, MO_ACQUIRE>::value, T>::type 233 load_internal(void* addr); 234 235 template <DecoratorSet ds, typename T> 236 static typename EnableIf< 237 HasDecorator<ds, MO_RELAXED>::value, T>::type 238 load_internal(void* addr); 239 240 template <DecoratorSet ds, typename T> 241 static inline typename EnableIf< 242 HasDecorator<ds, MO_UNORDERED>::value, T>::type 243 load_internal(void* addr) { 244 return *reinterpret_cast<T*>(addr); 245 } 246 247 template <DecoratorSet ds, typename T> 248 static typename EnableIf< 249 HasDecorator<ds, MO_SEQ_CST>::value>::type 250 store_internal(void* addr, T value); 251 252 template <DecoratorSet ds, typename T> 253 static typename EnableIf< 254 HasDecorator<ds, MO_RELEASE>::value>::type 255 store_internal(void* addr, T value); 256 257 template <DecoratorSet ds, typename T> 258 static typename EnableIf< 259 HasDecorator<ds, MO_RELAXED>::value>::type 260 store_internal(void* addr, T value); 261 262 template <DecoratorSet ds, typename T> 263 static inline typename EnableIf< 264 HasDecorator<ds, MO_UNORDERED>::value>::type 265 store_internal(void* addr, T value) { 266 *reinterpret_cast<T*>(addr) = value; 267 } 268 269 template <DecoratorSet ds, typename T> 270 static typename EnableIf< 271 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 272 atomic_cmpxchg_internal(void* addr, T compare_value, T new_value); 273 274 template <DecoratorSet ds, typename T> 275 static typename EnableIf< 276 HasDecorator<ds, MO_RELAXED>::value, T>::type 277 atomic_cmpxchg_internal(void* addr, T compare_value, T new_value); 278 279 template <DecoratorSet ds, typename T> 280 static typename EnableIf< 281 HasDecorator<ds, MO_SEQ_CST>::value, T>::type 282 atomic_xchg_internal(void* addr, T new_value); 283 284 // The following *_locked mechanisms serve the purpose of handling atomic operations 285 // that are larger than a machine can handle, and then possibly opt for using 286 // a slower path using a mutex to perform the operation. 287 288 template <DecoratorSet ds, typename T> 289 static inline typename EnableIf< 290 !AccessInternal::PossiblyLockedAccess<T>::value, T>::type 291 atomic_cmpxchg_maybe_locked(void* addr, T compare_value, T new_value) { 292 return atomic_cmpxchg_internal<ds>(addr, compare_value, new_value); 293 } 294 295 template <DecoratorSet ds, typename T> 296 static typename EnableIf< 297 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 298 atomic_cmpxchg_maybe_locked(void* addr, T compare_value, T new_value); 299 300 template <DecoratorSet ds, typename T> 301 static inline typename EnableIf< 302 !AccessInternal::PossiblyLockedAccess<T>::value, T>::type 303 atomic_xchg_maybe_locked(void* addr, T new_value) { 304 return atomic_xchg_internal<ds>(addr, new_value); 305 } 306 307 template <DecoratorSet ds, typename T> 308 static typename EnableIf< 309 AccessInternal::PossiblyLockedAccess<T>::value, T>::type 310 atomic_xchg_maybe_locked(void* addr, T new_value); 311 312 public: 313 template <typename T> 314 static inline void store(void* addr, T value) { 315 store_internal<decorators>(addr, value); 316 } 317 318 template <typename T> 319 static inline T load(void* addr) { 320 return load_internal<decorators, T>(addr); 321 } 322 323 template <typename T> 324 static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) { 325 return atomic_cmpxchg_maybe_locked<decorators>(addr, compare_value, new_value); 326 } 327 328 template <typename T> 329 static inline T atomic_xchg(void* addr, T new_value) { 330 return atomic_xchg_maybe_locked<decorators>(addr, new_value); 331 } 332 333 template <typename T> 334 static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 335 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 336 size_t length); 337 338 template <typename T> 339 static void oop_store(void* addr, T value); 340 template <typename T> 341 static void oop_store_at(oop base, ptrdiff_t offset, T value); 342 343 template <typename T> 344 static T oop_load(void* addr); 345 template <typename T> 346 static T oop_load_at(oop base, ptrdiff_t offset); 347 348 template <typename T> 349 static T oop_atomic_cmpxchg(void* addr, T compare_value, T new_value); 350 template <typename T> 351 static T oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value); 352 353 template <typename T> 354 static T oop_atomic_xchg(void* addr, T new_value); 355 template <typename T> 356 static T oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value); 357 358 template <typename T> 359 static void store_at(oop base, ptrdiff_t offset, T value) { 360 store(field_addr(base, offset), value); 361 } 362 363 template <typename T> 364 static T load_at(oop base, ptrdiff_t offset) { 365 return load<T>(field_addr(base, offset)); 366 } 367 368 template <typename T> 369 static T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) { 370 return atomic_cmpxchg(field_addr(base, offset), compare_value, new_value); 371 } 372 373 template <typename T> 374 static T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { 375 return atomic_xchg(field_addr(base, offset), new_value); 376 } 377 378 template <typename T> 379 static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 380 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 381 size_t length); 382 383 static void clone(oop src, oop dst, size_t size); 384 }; 385 386 namespace AccessInternal { 387 DEBUG_ONLY(void check_access_thread_state()); 388 #define assert_access_thread_state() DEBUG_ONLY(check_access_thread_state()) 389 } 390 391 // Below is the implementation of the first 4 steps of the template pipeline: 392 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers 393 // and sets default decorators to sensible values. 394 // * Step 2: Reduce types. This step makes sure there is only a single T type and not 395 // multiple types. The P type of the address and T type of the value must 396 // match. 397 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be 398 // avoided, and in that case avoids it (calling raw accesses or 399 // primitive accesses in a build that does not require primitive GC barriers) 400 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding 401 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers 402 // to the access. 403 404 namespace AccessInternal { 405 template <typename T> 406 struct OopOrNarrowOopInternal: AllStatic { 407 typedef oop type; 408 }; 409 410 template <> 411 struct OopOrNarrowOopInternal<narrowOop>: AllStatic { 412 typedef narrowOop type; 413 }; 414 415 // This metafunction returns a canonicalized oop/narrowOop type for a passed 416 // in oop-like types passed in from oop_* overloads where the user has sworn 417 // that the passed in values should be oop-like (e.g. oop, oopDesc*, arrayOop, 418 // narrowOoop, instanceOopDesc*, and random other things). 419 // In the oop_* overloads, it must hold that if the passed in type T is not 420 // narrowOop, then it by contract has to be one of many oop-like types implicitly 421 // convertible to oop, and hence returns oop as the canonical oop type. 422 // If it turns out it was not, then the implicit conversion to oop will fail 423 // to compile, as desired. 424 template <typename T> 425 struct OopOrNarrowOop: AllStatic { 426 typedef typename OopOrNarrowOopInternal<std::decay_t<T>>::type type; 427 }; 428 429 inline void* field_addr(oop base, ptrdiff_t byte_offset) { 430 return reinterpret_cast<void*>(reinterpret_cast<intptr_t>((void*)base) + byte_offset); 431 } 432 // Step 4: Runtime dispatch 433 // The RuntimeDispatch class is responsible for performing a runtime dispatch of the 434 // accessor. This is required when the access either depends on whether compressed oops 435 // is being used, or it depends on which GC implementation was chosen (e.g. requires GC 436 // barriers). The way it works is that a function pointer initially pointing to an 437 // accessor resolution function gets called for each access. Upon first invocation, 438 // it resolves which accessor to be used in future invocations and patches the 439 // function pointer to this new accessor. 440 441 template <DecoratorSet decorators, typename T, BarrierType type> 442 struct RuntimeDispatch: AllStatic {}; 443 444 template <DecoratorSet decorators, typename T> 445 struct RuntimeDispatch<decorators, T, BARRIER_STORE>: AllStatic { 446 typedef typename AccessFunction<decorators, T, BARRIER_STORE>::type func_t; 447 static func_t _store_func; 448 449 static void store_init(void* addr, T value); 450 451 static inline void store(void* addr, T value) { 452 assert_access_thread_state(); 453 _store_func(addr, value); 454 } 455 }; 456 457 template <DecoratorSet decorators, typename T> 458 struct RuntimeDispatch<decorators, T, BARRIER_STORE_AT>: AllStatic { 459 typedef typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type func_t; 460 static func_t _store_at_func; 461 462 static void store_at_init(oop base, ptrdiff_t offset, T value); 463 464 static inline void store_at(oop base, ptrdiff_t offset, T value) { 465 assert_access_thread_state(); 466 _store_at_func(base, offset, value); 467 } 468 }; 469 470 template <DecoratorSet decorators, typename T> 471 struct RuntimeDispatch<decorators, T, BARRIER_LOAD>: AllStatic { 472 typedef typename AccessFunction<decorators, T, BARRIER_LOAD>::type func_t; 473 static func_t _load_func; 474 475 static T load_init(void* addr); 476 477 static inline T load(void* addr) { 478 assert_access_thread_state(); 479 return _load_func(addr); 480 } 481 }; 482 483 template <DecoratorSet decorators, typename T> 484 struct RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>: AllStatic { 485 typedef typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type func_t; 486 static func_t _load_at_func; 487 488 static T load_at_init(oop base, ptrdiff_t offset); 489 490 static inline T load_at(oop base, ptrdiff_t offset) { 491 assert_access_thread_state(); 492 return _load_at_func(base, offset); 493 } 494 }; 495 496 template <DecoratorSet decorators, typename T> 497 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>: AllStatic { 498 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type func_t; 499 static func_t _atomic_cmpxchg_func; 500 501 static T atomic_cmpxchg_init(void* addr, T compare_value, T new_value); 502 503 static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) { 504 assert_access_thread_state(); 505 return _atomic_cmpxchg_func(addr, compare_value, new_value); 506 } 507 }; 508 509 template <DecoratorSet decorators, typename T> 510 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>: AllStatic { 511 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type func_t; 512 static func_t _atomic_cmpxchg_at_func; 513 514 static T atomic_cmpxchg_at_init(oop base, ptrdiff_t offset, T compare_value, T new_value); 515 516 static inline T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) { 517 assert_access_thread_state(); 518 return _atomic_cmpxchg_at_func(base, offset, compare_value, new_value); 519 } 520 }; 521 522 template <DecoratorSet decorators, typename T> 523 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>: AllStatic { 524 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type func_t; 525 static func_t _atomic_xchg_func; 526 527 static T atomic_xchg_init(void* addr, T new_value); 528 529 static inline T atomic_xchg(void* addr, T new_value) { 530 assert_access_thread_state(); 531 return _atomic_xchg_func(addr, new_value); 532 } 533 }; 534 535 template <DecoratorSet decorators, typename T> 536 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic { 537 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t; 538 static func_t _atomic_xchg_at_func; 539 540 static T atomic_xchg_at_init(oop base, ptrdiff_t offset, T new_value); 541 542 static inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { 543 assert_access_thread_state(); 544 return _atomic_xchg_at_func(base, offset, new_value); 545 } 546 }; 547 548 template <DecoratorSet decorators, typename T> 549 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic { 550 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t; 551 static func_t _arraycopy_func; 552 553 static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 554 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 555 size_t length); 556 557 static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 558 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 559 size_t length) { 560 assert_access_thread_state(); 561 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw, 562 dst_obj, dst_offset_in_bytes, dst_raw, 563 length); 564 } 565 }; 566 567 template <DecoratorSet decorators, typename T> 568 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic { 569 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t; 570 static func_t _clone_func; 571 572 static void clone_init(oop src, oop dst, size_t size); 573 574 static inline void clone(oop src, oop dst, size_t size) { 575 assert_access_thread_state(); 576 _clone_func(src, dst, size); 577 } 578 }; 579 580 // Initialize the function pointers to point to the resolving function. 581 template <DecoratorSet decorators, typename T> 582 typename AccessFunction<decorators, T, BARRIER_STORE>::type 583 RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init; 584 585 template <DecoratorSet decorators, typename T> 586 typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type 587 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init; 588 589 template <DecoratorSet decorators, typename T> 590 typename AccessFunction<decorators, T, BARRIER_LOAD>::type 591 RuntimeDispatch<decorators, T, BARRIER_LOAD>::_load_func = &load_init; 592 593 template <DecoratorSet decorators, typename T> 594 typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type 595 RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::_load_at_func = &load_at_init; 596 597 template <DecoratorSet decorators, typename T> 598 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type 599 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::_atomic_cmpxchg_func = &atomic_cmpxchg_init; 600 601 template <DecoratorSet decorators, typename T> 602 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type 603 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init; 604 605 template <DecoratorSet decorators, typename T> 606 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type 607 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init; 608 609 template <DecoratorSet decorators, typename T> 610 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type 611 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init; 612 613 template <DecoratorSet decorators, typename T> 614 typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type 615 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init; 616 617 template <DecoratorSet decorators, typename T> 618 typename AccessFunction<decorators, T, BARRIER_CLONE>::type 619 RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init; 620 621 // Step 3: Pre-runtime dispatching. 622 // The PreRuntimeDispatch class is responsible for filtering the barrier strength 623 // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime 624 // dispatch point. Otherwise it goes through a runtime check if hardwiring was 625 // not possible. 626 struct PreRuntimeDispatch: AllStatic { 627 template<DecoratorSet decorators> 628 struct CanHardwireRaw: public std::integral_constant< 629 bool, 630 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access 631 !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address) 632 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address) 633 {}; 634 635 static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP; 636 637 template<DecoratorSet decorators> 638 static bool is_hardwired_primitive() { 639 return !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value; 640 } 641 642 template <DecoratorSet decorators, typename T> 643 inline static typename EnableIf< 644 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value>::type 645 store(void* addr, T value) { 646 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 647 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 648 Raw::oop_store(addr, value); 649 } else { 650 Raw::store(addr, value); 651 } 652 } 653 654 template <DecoratorSet decorators, typename T> 655 inline static typename EnableIf< 656 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value>::type 657 store(void* addr, T value) { 658 if (UseCompressedOops) { 659 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 660 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 661 } else { 662 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 663 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 664 } 665 } 666 667 template <DecoratorSet decorators, typename T> 668 inline static typename EnableIf< 669 !HasDecorator<decorators, AS_RAW>::value>::type 670 store(void* addr, T value) { 671 if (is_hardwired_primitive<decorators>()) { 672 const DecoratorSet expanded_decorators = decorators | AS_RAW; 673 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 674 } else { 675 RuntimeDispatch<decorators, T, BARRIER_STORE>::store(addr, value); 676 } 677 } 678 679 template <DecoratorSet decorators, typename T> 680 inline static typename EnableIf< 681 HasDecorator<decorators, AS_RAW>::value>::type 682 store_at(oop base, ptrdiff_t offset, T value) { 683 store<decorators>(field_addr(base, offset), value); 684 } 685 686 template <DecoratorSet decorators, typename T> 687 inline static typename EnableIf< 688 !HasDecorator<decorators, AS_RAW>::value>::type 689 store_at(oop base, ptrdiff_t offset, T value) { 690 if (is_hardwired_primitive<decorators>()) { 691 const DecoratorSet expanded_decorators = decorators | AS_RAW; 692 PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, value); 693 } else { 694 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::store_at(base, offset, value); 695 } 696 } 697 698 template <DecoratorSet decorators, typename T> 699 inline static typename EnableIf< 700 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type 701 load(void* addr) { 702 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 703 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 704 return Raw::template oop_load<T>(addr); 705 } else { 706 return Raw::template load<T>(addr); 707 } 708 } 709 710 template <DecoratorSet decorators, typename T> 711 inline static typename EnableIf< 712 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type 713 load(void* addr) { 714 if (UseCompressedOops) { 715 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 716 return PreRuntimeDispatch::load<expanded_decorators, T>(addr); 717 } else { 718 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 719 return PreRuntimeDispatch::load<expanded_decorators, T>(addr); 720 } 721 } 722 723 template <DecoratorSet decorators, typename T> 724 inline static typename EnableIf< 725 !HasDecorator<decorators, AS_RAW>::value, T>::type 726 load(void* addr) { 727 if (is_hardwired_primitive<decorators>()) { 728 const DecoratorSet expanded_decorators = decorators | AS_RAW; 729 return PreRuntimeDispatch::load<expanded_decorators, T>(addr); 730 } else { 731 return RuntimeDispatch<decorators, T, BARRIER_LOAD>::load(addr); 732 } 733 } 734 735 template <DecoratorSet decorators, typename T> 736 inline static typename EnableIf< 737 HasDecorator<decorators, AS_RAW>::value, T>::type 738 load_at(oop base, ptrdiff_t offset) { 739 return load<decorators, T>(field_addr(base, offset)); 740 } 741 742 template <DecoratorSet decorators, typename T> 743 inline static typename EnableIf< 744 !HasDecorator<decorators, AS_RAW>::value, T>::type 745 load_at(oop base, ptrdiff_t offset) { 746 if (is_hardwired_primitive<decorators>()) { 747 const DecoratorSet expanded_decorators = decorators | AS_RAW; 748 return PreRuntimeDispatch::load_at<expanded_decorators, T>(base, offset); 749 } else { 750 return RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::load_at(base, offset); 751 } 752 } 753 754 template <DecoratorSet decorators, typename T> 755 inline static typename EnableIf< 756 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type 757 atomic_cmpxchg(void* addr, T compare_value, T new_value) { 758 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 759 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 760 return Raw::oop_atomic_cmpxchg(addr, compare_value, new_value); 761 } else { 762 return Raw::atomic_cmpxchg(addr, compare_value, new_value); 763 } 764 } 765 766 template <DecoratorSet decorators, typename T> 767 inline static typename EnableIf< 768 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type 769 atomic_cmpxchg(void* addr, T compare_value, T new_value) { 770 if (UseCompressedOops) { 771 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 772 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value); 773 } else { 774 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 775 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value); 776 } 777 } 778 779 template <DecoratorSet decorators, typename T> 780 inline static typename EnableIf< 781 !HasDecorator<decorators, AS_RAW>::value, T>::type 782 atomic_cmpxchg(void* addr, T compare_value, T new_value) { 783 if (is_hardwired_primitive<decorators>()) { 784 const DecoratorSet expanded_decorators = decorators | AS_RAW; 785 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value); 786 } else { 787 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::atomic_cmpxchg(addr, compare_value, new_value); 788 } 789 } 790 791 template <DecoratorSet decorators, typename T> 792 inline static typename EnableIf< 793 HasDecorator<decorators, AS_RAW>::value, T>::type 794 atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) { 795 return atomic_cmpxchg<decorators>(field_addr(base, offset), compare_value, new_value); 796 } 797 798 template <DecoratorSet decorators, typename T> 799 inline static typename EnableIf< 800 !HasDecorator<decorators, AS_RAW>::value, T>::type 801 atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) { 802 if (is_hardwired_primitive<decorators>()) { 803 const DecoratorSet expanded_decorators = decorators | AS_RAW; 804 return PreRuntimeDispatch::atomic_cmpxchg_at<expanded_decorators>(base, offset, compare_value, new_value); 805 } else { 806 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::atomic_cmpxchg_at(base, offset, compare_value, new_value); 807 } 808 } 809 810 template <DecoratorSet decorators, typename T> 811 inline static typename EnableIf< 812 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type 813 atomic_xchg(void* addr, T new_value) { 814 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 815 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 816 return Raw::oop_atomic_xchg(addr, new_value); 817 } else { 818 return Raw::atomic_xchg(addr, new_value); 819 } 820 } 821 822 template <DecoratorSet decorators, typename T> 823 inline static typename EnableIf< 824 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type 825 atomic_xchg(void* addr, T new_value) { 826 if (UseCompressedOops) { 827 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 828 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); 829 } else { 830 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 831 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); 832 } 833 } 834 835 template <DecoratorSet decorators, typename T> 836 inline static typename EnableIf< 837 !HasDecorator<decorators, AS_RAW>::value, T>::type 838 atomic_xchg(void* addr, T new_value) { 839 if (is_hardwired_primitive<decorators>()) { 840 const DecoratorSet expanded_decorators = decorators | AS_RAW; 841 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); 842 } else { 843 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::atomic_xchg(addr, new_value); 844 } 845 } 846 847 template <DecoratorSet decorators, typename T> 848 inline static typename EnableIf< 849 HasDecorator<decorators, AS_RAW>::value, T>::type 850 atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { 851 return atomic_xchg<decorators>(field_addr(base, offset), new_value); 852 } 853 854 template <DecoratorSet decorators, typename T> 855 inline static typename EnableIf< 856 !HasDecorator<decorators, AS_RAW>::value, T>::type 857 atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { 858 if (is_hardwired_primitive<decorators>()) { 859 const DecoratorSet expanded_decorators = decorators | AS_RAW; 860 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(base, offset, new_value); 861 } else { 862 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(base, offset, new_value); 863 } 864 } 865 866 template <DecoratorSet decorators, typename T> 867 inline static typename EnableIf< 868 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type 869 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 870 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 871 size_t length) { 872 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 873 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) { 874 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, 875 dst_obj, dst_offset_in_bytes, dst_raw, 876 length); 877 } else { 878 return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw, 879 dst_obj, dst_offset_in_bytes, dst_raw, 880 length); 881 } 882 } 883 884 template <DecoratorSet decorators, typename T> 885 inline static typename EnableIf< 886 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type 887 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 888 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 889 size_t length) { 890 if (UseCompressedOops) { 891 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops; 892 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, 893 dst_obj, dst_offset_in_bytes, dst_raw, 894 length); 895 } else { 896 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops; 897 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, 898 dst_obj, dst_offset_in_bytes, dst_raw, 899 length); 900 } 901 } 902 903 template <DecoratorSet decorators, typename T> 904 inline static typename EnableIf< 905 !HasDecorator<decorators, AS_RAW>::value, bool>::type 906 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 907 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 908 size_t length) { 909 if (is_hardwired_primitive<decorators>()) { 910 const DecoratorSet expanded_decorators = decorators | AS_RAW; 911 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, 912 dst_obj, dst_offset_in_bytes, dst_raw, 913 length); 914 } else { 915 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw, 916 dst_obj, dst_offset_in_bytes, dst_raw, 917 length); 918 } 919 } 920 921 template <DecoratorSet decorators> 922 inline static typename EnableIf< 923 HasDecorator<decorators, AS_RAW>::value>::type 924 clone(oop src, oop dst, size_t size) { 925 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw; 926 Raw::clone(src, dst, size); 927 } 928 929 template <DecoratorSet decorators> 930 inline static typename EnableIf< 931 !HasDecorator<decorators, AS_RAW>::value>::type 932 clone(oop src, oop dst, size_t size) { 933 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size); 934 } 935 }; 936 937 // Step 2: Reduce types. 938 // Enforce that for non-oop types, T and P have to be strictly the same. 939 // P is the type of the address and T is the type of the values. 940 // As for oop types, it is allow to send T in {narrowOop, oop} and 941 // P in {narrowOop, oop, HeapWord*}. The following rules apply according to 942 // the subsequent table. (columns are P, rows are T) 943 // | | HeapWord | oop | narrowOop | 944 // | oop | rt-comp | hw-none | hw-comp | 945 // | narrowOop | x | x | hw-none | 946 // 947 // x means not allowed 948 // rt-comp means it must be checked at runtime whether the oop is compressed. 949 // hw-none means it is statically known the oop will not be compressed. 950 // hw-comp means it is statically known the oop will be compressed. 951 952 template <DecoratorSet decorators, typename T> 953 inline void store_reduce_types(T* addr, T value) { 954 PreRuntimeDispatch::store<decorators>(addr, value); 955 } 956 957 template <DecoratorSet decorators> 958 inline void store_reduce_types(narrowOop* addr, oop value) { 959 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 960 INTERNAL_RT_USE_COMPRESSED_OOPS; 961 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 962 } 963 964 template <DecoratorSet decorators> 965 inline void store_reduce_types(narrowOop* addr, narrowOop value) { 966 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 967 INTERNAL_RT_USE_COMPRESSED_OOPS; 968 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 969 } 970 971 template <DecoratorSet decorators> 972 inline void store_reduce_types(HeapWord* addr, oop value) { 973 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 974 PreRuntimeDispatch::store<expanded_decorators>(addr, value); 975 } 976 977 template <DecoratorSet decorators, typename T> 978 inline T atomic_cmpxchg_reduce_types(T* addr, T compare_value, T new_value) { 979 return PreRuntimeDispatch::atomic_cmpxchg<decorators>(addr, compare_value, new_value); 980 } 981 982 template <DecoratorSet decorators> 983 inline oop atomic_cmpxchg_reduce_types(narrowOop* addr, oop compare_value, oop new_value) { 984 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 985 INTERNAL_RT_USE_COMPRESSED_OOPS; 986 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value); 987 } 988 989 template <DecoratorSet decorators> 990 inline narrowOop atomic_cmpxchg_reduce_types(narrowOop* addr, narrowOop compare_value, narrowOop new_value) { 991 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 992 INTERNAL_RT_USE_COMPRESSED_OOPS; 993 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value); 994 } 995 996 template <DecoratorSet decorators> 997 inline oop atomic_cmpxchg_reduce_types(HeapWord* addr, 998 oop compare_value, 999 oop new_value) { 1000 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1001 return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value); 1002 } 1003 1004 template <DecoratorSet decorators, typename T> 1005 inline T atomic_xchg_reduce_types(T* addr, T new_value) { 1006 const DecoratorSet expanded_decorators = decorators; 1007 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); 1008 } 1009 1010 template <DecoratorSet decorators> 1011 inline oop atomic_xchg_reduce_types(narrowOop* addr, oop new_value) { 1012 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1013 INTERNAL_RT_USE_COMPRESSED_OOPS; 1014 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); 1015 } 1016 1017 template <DecoratorSet decorators> 1018 inline narrowOop atomic_xchg_reduce_types(narrowOop* addr, narrowOop new_value) { 1019 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1020 INTERNAL_RT_USE_COMPRESSED_OOPS; 1021 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); 1022 } 1023 1024 template <DecoratorSet decorators> 1025 inline oop atomic_xchg_reduce_types(HeapWord* addr, oop new_value) { 1026 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1027 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value); 1028 } 1029 1030 template <DecoratorSet decorators, typename T> 1031 inline T load_reduce_types(T* addr) { 1032 return PreRuntimeDispatch::load<decorators, T>(addr); 1033 } 1034 1035 template <DecoratorSet decorators, typename T> 1036 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) { 1037 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1038 INTERNAL_RT_USE_COMPRESSED_OOPS; 1039 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr); 1040 } 1041 1042 template <DecoratorSet decorators, typename T> 1043 inline oop load_reduce_types(HeapWord* addr) { 1044 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1045 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr); 1046 } 1047 1048 template <DecoratorSet decorators, typename T> 1049 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, 1050 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 1051 size_t length) { 1052 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw, 1053 dst_obj, dst_offset_in_bytes, dst_raw, 1054 length); 1055 } 1056 1057 template <DecoratorSet decorators> 1058 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw, 1059 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw, 1060 size_t length) { 1061 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP; 1062 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, 1063 dst_obj, dst_offset_in_bytes, dst_raw, 1064 length); 1065 } 1066 1067 template <DecoratorSet decorators> 1068 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw, 1069 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw, 1070 size_t length) { 1071 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP | 1072 INTERNAL_RT_USE_COMPRESSED_OOPS; 1073 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw, 1074 dst_obj, dst_offset_in_bytes, dst_raw, 1075 length); 1076 } 1077 1078 // Step 1: Set default decorators. This step remembers if a type was volatile 1079 // and then sets the MO_RELAXED decorator by default. Otherwise, a default 1080 // memory ordering is set for the access, and the implied decorator rules 1081 // are applied to select sensible defaults for decorators that have not been 1082 // explicitly set. For example, default object referent strength is set to strong. 1083 // This step also decays the types passed in (e.g. getting rid of CV qualifiers 1084 // and references from the types). This step also perform some type verification 1085 // that the passed in types make sense. 1086 1087 template <DecoratorSet decorators, typename T> 1088 static void verify_types(){ 1089 // If this fails to compile, then you have sent in something that is 1090 // not recognized as a valid primitive type to a primitive Access function. 1091 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated 1092 (std::is_pointer<T>::value || std::is_integral<T>::value) || 1093 std::is_floating_point<T>::value)); // not allowed primitive type 1094 } 1095 1096 template <DecoratorSet decorators, typename P, typename T> 1097 inline void store(P* addr, T value) { 1098 verify_types<decorators, T>(); 1099 using DecayedP = std::decay_t<P>; 1100 using DecayedT = std::decay_t<T>; 1101 DecayedT decayed_value = value; 1102 // If a volatile address is passed in but no memory ordering decorator, 1103 // set the memory ordering to MO_RELAXED by default. 1104 const DecoratorSet expanded_decorators = DecoratorFixup< 1105 (std::is_volatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1106 (MO_RELAXED | decorators) : decorators>::value; 1107 store_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), decayed_value); 1108 } 1109 1110 template <DecoratorSet decorators, typename T> 1111 inline void store_at(oop base, ptrdiff_t offset, T value) { 1112 verify_types<decorators, T>(); 1113 using DecayedT = std::decay_t<T>; 1114 DecayedT decayed_value = value; 1115 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | 1116 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1117 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value; 1118 PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, decayed_value); 1119 } 1120 1121 template <DecoratorSet decorators, typename P, typename T> 1122 inline T load(P* addr) { 1123 verify_types<decorators, T>(); 1124 using DecayedP = std::decay_t<P>; 1125 using DecayedT = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value, 1126 typename OopOrNarrowOop<T>::type, 1127 std::decay_t<T>>; 1128 // If a volatile address is passed in but no memory ordering decorator, 1129 // set the memory ordering to MO_RELAXED by default. 1130 const DecoratorSet expanded_decorators = DecoratorFixup< 1131 (std::is_volatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1132 (MO_RELAXED | decorators) : decorators>::value; 1133 return load_reduce_types<expanded_decorators, DecayedT>(const_cast<DecayedP*>(addr)); 1134 } 1135 1136 template <DecoratorSet decorators, typename T> 1137 inline T load_at(oop base, ptrdiff_t offset) { 1138 verify_types<decorators, T>(); 1139 using DecayedT = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value, 1140 typename OopOrNarrowOop<T>::type, 1141 std::decay_t<T>>; 1142 // Expand the decorators (figure out sensible defaults) 1143 // Potentially remember if we need compressed oop awareness 1144 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | 1145 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1146 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value; 1147 return PreRuntimeDispatch::load_at<expanded_decorators, DecayedT>(base, offset); 1148 } 1149 1150 template <DecoratorSet decorators, typename P, typename T> 1151 inline T atomic_cmpxchg(P* addr, T compare_value, T new_value) { 1152 verify_types<decorators, T>(); 1153 using DecayedP = std::decay_t<P>; 1154 using DecayedT = std::decay_t<T>; 1155 DecayedT new_decayed_value = new_value; 1156 DecayedT compare_decayed_value = compare_value; 1157 const DecoratorSet expanded_decorators = DecoratorFixup< 1158 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1159 (MO_SEQ_CST | decorators) : decorators>::value; 1160 return atomic_cmpxchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), 1161 compare_decayed_value, 1162 new_decayed_value); 1163 } 1164 1165 template <DecoratorSet decorators, typename T> 1166 inline T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) { 1167 verify_types<decorators, T>(); 1168 using DecayedT = std::decay_t<T>; 1169 DecayedT new_decayed_value = new_value; 1170 DecayedT compare_decayed_value = compare_value; 1171 // Determine default memory ordering 1172 const DecoratorSet expanded_decorators = DecoratorFixup< 1173 (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ? 1174 (MO_SEQ_CST | decorators) : decorators>::value; 1175 // Potentially remember that we need compressed oop awareness 1176 const DecoratorSet final_decorators = expanded_decorators | 1177 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1178 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE); 1179 return PreRuntimeDispatch::atomic_cmpxchg_at<final_decorators>(base, offset, compare_decayed_value, 1180 new_decayed_value); 1181 } 1182 1183 template <DecoratorSet decorators, typename P, typename T> 1184 inline T atomic_xchg(P* addr, T new_value) { 1185 verify_types<decorators, T>(); 1186 using DecayedP = std::decay_t<P>; 1187 using DecayedT = std::decay_t<T>; 1188 DecayedT new_decayed_value = new_value; 1189 // atomic_xchg is only available in SEQ_CST flavour. 1190 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value; 1191 return atomic_xchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), 1192 new_decayed_value); 1193 } 1194 1195 template <DecoratorSet decorators, typename T> 1196 inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) { 1197 verify_types<decorators, T>(); 1198 using DecayedT = std::decay_t<T>; 1199 DecayedT new_decayed_value = new_value; 1200 // atomic_xchg is only available in SEQ_CST flavour. 1201 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST | 1202 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ? 1203 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value; 1204 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(base, offset, new_decayed_value); 1205 } 1206 1207 template <DecoratorSet decorators, typename T> 1208 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw, 1209 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, 1210 size_t length) { 1211 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || 1212 (std::is_same<T, void>::value || std::is_integral<T>::value) || 1213 std::is_floating_point<T>::value)); // arraycopy allows type erased void elements 1214 using DecayedT = std::decay_t<T>; 1215 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value; 1216 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw), 1217 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw), 1218 length); 1219 } 1220 1221 template <DecoratorSet decorators> 1222 inline void clone(oop src, oop dst, size_t size) { 1223 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value; 1224 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size); 1225 } 1226 1227 // Infer the type that should be returned from an Access::oop_load. 1228 template <typename P, DecoratorSet decorators> 1229 class OopLoadProxy: public StackObj { 1230 private: 1231 P *const _addr; 1232 public: 1233 explicit OopLoadProxy(P* addr) : _addr(addr) {} 1234 1235 inline operator oop() { 1236 return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr); 1237 } 1238 1239 inline operator narrowOop() { 1240 return load<decorators | INTERNAL_VALUE_IS_OOP, P, narrowOop>(_addr); 1241 } 1242 1243 template <typename T> 1244 inline bool operator ==(const T& other) const { 1245 return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) == other; 1246 } 1247 1248 template <typename T> 1249 inline bool operator !=(const T& other) const { 1250 return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) != other; 1251 } 1252 1253 inline bool operator ==(std::nullptr_t) const { 1254 return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr) == nullptr; 1255 } 1256 1257 inline bool operator !=(std::nullptr_t) const { 1258 return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr) != nullptr; 1259 } 1260 }; 1261 1262 // Infer the type that should be returned from an Access::load_at. 1263 template <DecoratorSet decorators> 1264 class LoadAtProxy: public StackObj { 1265 private: 1266 const oop _base; 1267 const ptrdiff_t _offset; 1268 public: 1269 LoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {} 1270 1271 template <typename T> 1272 inline operator T() const { 1273 return load_at<decorators, T>(_base, _offset); 1274 } 1275 1276 template <typename T> 1277 inline bool operator ==(const T& other) const { return load_at<decorators, T>(_base, _offset) == other; } 1278 1279 template <typename T> 1280 inline bool operator !=(const T& other) const { return load_at<decorators, T>(_base, _offset) != other; } 1281 }; 1282 1283 // Infer the type that should be returned from an Access::oop_load_at. 1284 template <DecoratorSet decorators> 1285 class OopLoadAtProxy: public StackObj { 1286 private: 1287 const oop _base; 1288 const ptrdiff_t _offset; 1289 public: 1290 OopLoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {} 1291 1292 inline operator oop() const { 1293 return load_at<decorators | INTERNAL_VALUE_IS_OOP, oop>(_base, _offset); 1294 } 1295 1296 inline operator narrowOop() const { 1297 return load_at<decorators | INTERNAL_VALUE_IS_OOP, narrowOop>(_base, _offset); 1298 } 1299 1300 template <typename T> 1301 inline bool operator ==(const T& other) const { 1302 return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) == other; 1303 } 1304 1305 template <typename T> 1306 inline bool operator !=(const T& other) const { 1307 return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) != other; 1308 } 1309 }; 1310 } 1311 1312 #endif // SHARE_OOPS_ACCESSBACKEND_HPP