1 /*
   2  * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_OOPS_ACCESSBACKEND_HPP
  26 #define SHARE_OOPS_ACCESSBACKEND_HPP
  27 
  28 #include "gc/shared/barrierSetConfig.hpp"
  29 #include "memory/allocation.hpp"
  30 #include "metaprogramming/enableIf.hpp"
  31 #include "oops/accessDecorators.hpp"
  32 #include "oops/oopsHierarchy.hpp"
  33 #include "runtime/globals.hpp"
  34 #include "utilities/debug.hpp"
  35 #include "utilities/globalDefinitions.hpp"
  36 
  37 #include <type_traits>
  38 
  39 // This metafunction returns either oop or narrowOop depending on whether
  40 // an access needs to use compressed oops or not.
  41 template <DecoratorSet decorators>
  42 struct HeapOopType: AllStatic {
  43   static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
  44                                          HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
  45   using type = std::conditional_t<needs_oop_compress, narrowOop, oop>;
  46 };
  47 
  48 namespace AccessInternal {
  49   enum BarrierType {
  50     BARRIER_STORE,
  51     BARRIER_STORE_AT,
  52     BARRIER_LOAD,
  53     BARRIER_LOAD_AT,
  54     BARRIER_ATOMIC_CMPXCHG,
  55     BARRIER_ATOMIC_CMPXCHG_AT,
  56     BARRIER_ATOMIC_XCHG,
  57     BARRIER_ATOMIC_XCHG_AT,
  58     BARRIER_ARRAYCOPY,
  59     BARRIER_CLONE
  60   };
  61 
  62   template <DecoratorSet decorators, typename T>
  63   struct MustConvertCompressedOop: public std::integral_constant<bool,
  64     HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
  65     std::is_same<typename HeapOopType<decorators>::type, narrowOop>::value &&
  66     std::is_same<T, oop>::value> {};
  67 
  68   // This metafunction returns an appropriate oop type if the value is oop-like
  69   // and otherwise returns the same type T.
  70   template <DecoratorSet decorators, typename T>
  71   struct EncodedType: AllStatic {
  72     using type = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
  73                                     typename HeapOopType<decorators>::type,
  74                                     T>;
  75   };
  76 
  77   template <DecoratorSet decorators>
  78   inline typename HeapOopType<decorators>::type*
  79   oop_field_addr(oop base, ptrdiff_t byte_offset) {
  80     return reinterpret_cast<typename HeapOopType<decorators>::type*>(
  81              reinterpret_cast<intptr_t>((void*)base) + byte_offset);
  82   }
  83 
  84   // This metafunction returns whether it is possible for a type T to require
  85   // locking to support wide atomics or not.
  86   template <typename T>
  87 #ifdef SUPPORTS_NATIVE_CX8
  88   struct PossiblyLockedAccess: public std::false_type {};
  89 #else
  90   struct PossiblyLockedAccess: public std::integral_constant<bool, (sizeof(T) > 4)> {};
  91 #endif
  92 
  93   template <DecoratorSet decorators, typename T>
  94   struct AccessFunctionTypes {
  95     typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
  96     typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
  97     typedef T (*atomic_cmpxchg_at_func_t)(oop base, ptrdiff_t offset, T compare_value, T new_value);
  98     typedef T (*atomic_xchg_at_func_t)(oop base, ptrdiff_t offset, T new_value);
  99 
 100     typedef T (*load_func_t)(void* addr);
 101     typedef void (*store_func_t)(void* addr, T value);
 102     typedef T (*atomic_cmpxchg_func_t)(void* addr, T compare_value, T new_value);
 103     typedef T (*atomic_xchg_func_t)(void* addr, T new_value);
 104 
 105     typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 106                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 107                                      size_t length);
 108     typedef void (*clone_func_t)(oop src, oop dst, size_t size);
 109   };
 110 
 111   template <DecoratorSet decorators>
 112   struct AccessFunctionTypes<decorators, void> {
 113     typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
 114                                      arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
 115                                      size_t length);
 116   };
 117 
 118   template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
 119 
 120 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func)                   \
 121   template <DecoratorSet decorators, typename T>                    \
 122   struct AccessFunction<decorators, T, bt>: AllStatic{              \
 123     typedef typename AccessFunctionTypes<decorators, T>::func type; \
 124   }
 125   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
 126   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
 127   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
 128   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
 129   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
 130   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
 131   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
 132   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
 133   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
 134   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
 135 #undef ACCESS_GENERATE_ACCESS_FUNCTION
 136 
 137   template <DecoratorSet decorators, typename T, BarrierType barrier_type>
 138   typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier();
 139 
 140   template <DecoratorSet decorators, typename T, BarrierType barrier_type>
 141   typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier();
 142 
 143   class AccessLocker {
 144   public:
 145     AccessLocker();
 146     ~AccessLocker();
 147   };
 148   bool wide_atomic_needs_locking();
 149 
 150   void* field_addr(oop base, ptrdiff_t offset);
 151 
 152   // Forward calls to Copy:: in the cpp file to reduce dependencies and allow
 153   // faster build times, given how frequently included access is.
 154   void arraycopy_arrayof_conjoint_oops(void* src, void* dst, size_t length);
 155   void arraycopy_conjoint_oops(oop* src, oop* dst, size_t length);
 156   void arraycopy_conjoint_oops(narrowOop* src, narrowOop* dst, size_t length);
 157 
 158   void arraycopy_disjoint_words(void* src, void* dst, size_t length);
 159   void arraycopy_disjoint_words_atomic(void* src, void* dst, size_t length);
 160 
 161   template<typename T>
 162   void arraycopy_conjoint(T* src, T* dst, size_t length);
 163   template<typename T>
 164   void arraycopy_arrayof_conjoint(T* src, T* dst, size_t length);
 165   template<typename T>
 166   void arraycopy_conjoint_atomic(T* src, T* dst, size_t length);
 167 }
 168 
 169 // This mask specifies what decorators are relevant for raw accesses. When passing
 170 // accesses to the raw layer, irrelevant decorators are removed.
 171 const DecoratorSet RAW_DECORATOR_MASK = INTERNAL_DECORATOR_MASK | MO_DECORATOR_MASK |
 172                                         ARRAYCOPY_DECORATOR_MASK | IS_NOT_NULL;
 173 
 174 // The RawAccessBarrier performs raw accesses with additional knowledge of
 175 // memory ordering, so that OrderAccess/Atomic is called when necessary.
 176 // It additionally handles compressed oops, and hence is not completely "raw"
 177 // strictly speaking.
 178 template <DecoratorSet decorators>
 179 class RawAccessBarrier: public AllStatic {
 180 protected:
 181   static inline void* field_addr(oop base, ptrdiff_t byte_offset) {
 182     return AccessInternal::field_addr(base, byte_offset);
 183   }
 184 
 185 protected:
 186   // Only encode if INTERNAL_VALUE_IS_OOP
 187   template <DecoratorSet idecorators, typename T>
 188   static inline typename EnableIf<
 189     AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
 190     typename HeapOopType<idecorators>::type>::type
 191   encode_internal(T value);
 192 
 193   template <DecoratorSet idecorators, typename T>
 194   static inline typename EnableIf<
 195     !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
 196   encode_internal(T value) {
 197     return value;
 198   }
 199 
 200   template <typename T>
 201   static inline typename AccessInternal::EncodedType<decorators, T>::type
 202   encode(T value) {
 203     return encode_internal<decorators, T>(value);
 204   }
 205 
 206   // Only decode if INTERNAL_VALUE_IS_OOP
 207   template <DecoratorSet idecorators, typename T>
 208   static inline typename EnableIf<
 209     AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
 210   decode_internal(typename HeapOopType<idecorators>::type value);
 211 
 212   template <DecoratorSet idecorators, typename T>
 213   static inline typename EnableIf<
 214     !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
 215   decode_internal(T value) {
 216     return value;
 217   }
 218 
 219   template <typename T>
 220   static inline T decode(typename AccessInternal::EncodedType<decorators, T>::type value) {
 221     return decode_internal<decorators, T>(value);
 222   }
 223 
 224 protected:
 225   template <DecoratorSet ds, typename T>
 226   static typename EnableIf<
 227     HasDecorator<ds, MO_SEQ_CST>::value, T>::type
 228   load_internal(void* addr);
 229 
 230   template <DecoratorSet ds, typename T>
 231   static typename EnableIf<
 232     HasDecorator<ds, MO_ACQUIRE>::value, T>::type
 233   load_internal(void* addr);
 234 
 235   template <DecoratorSet ds, typename T>
 236   static typename EnableIf<
 237     HasDecorator<ds, MO_RELAXED>::value, T>::type
 238   load_internal(void* addr);
 239 
 240   template <DecoratorSet ds, typename T>
 241   static inline typename EnableIf<
 242     HasDecorator<ds, MO_UNORDERED>::value, T>::type
 243   load_internal(void* addr) {
 244     return *reinterpret_cast<T*>(addr);
 245   }
 246 
 247   template <DecoratorSet ds, typename T>
 248   static typename EnableIf<
 249     HasDecorator<ds, MO_SEQ_CST>::value>::type
 250   store_internal(void* addr, T value);
 251 
 252   template <DecoratorSet ds, typename T>
 253   static typename EnableIf<
 254     HasDecorator<ds, MO_RELEASE>::value>::type
 255   store_internal(void* addr, T value);
 256 
 257   template <DecoratorSet ds, typename T>
 258   static typename EnableIf<
 259     HasDecorator<ds, MO_RELAXED>::value>::type
 260   store_internal(void* addr, T value);
 261 
 262   template <DecoratorSet ds, typename T>
 263   static inline typename EnableIf<
 264     HasDecorator<ds, MO_UNORDERED>::value>::type
 265   store_internal(void* addr, T value) {
 266     *reinterpret_cast<T*>(addr) = value;
 267   }
 268 
 269   template <DecoratorSet ds, typename T>
 270   static typename EnableIf<
 271     HasDecorator<ds, MO_SEQ_CST>::value, T>::type
 272   atomic_cmpxchg_internal(void* addr, T compare_value, T new_value);
 273 
 274   template <DecoratorSet ds, typename T>
 275   static typename EnableIf<
 276     HasDecorator<ds, MO_RELAXED>::value, T>::type
 277   atomic_cmpxchg_internal(void* addr, T compare_value, T new_value);
 278 
 279   template <DecoratorSet ds, typename T>
 280   static typename EnableIf<
 281     HasDecorator<ds, MO_SEQ_CST>::value, T>::type
 282   atomic_xchg_internal(void* addr, T new_value);
 283 
 284   // The following *_locked mechanisms serve the purpose of handling atomic operations
 285   // that are larger than a machine can handle, and then possibly opt for using
 286   // a slower path using a mutex to perform the operation.
 287 
 288   template <DecoratorSet ds, typename T>
 289   static inline typename EnableIf<
 290     !AccessInternal::PossiblyLockedAccess<T>::value, T>::type
 291   atomic_cmpxchg_maybe_locked(void* addr, T compare_value, T new_value) {
 292     return atomic_cmpxchg_internal<ds>(addr, compare_value, new_value);
 293   }
 294 
 295   template <DecoratorSet ds, typename T>
 296   static typename EnableIf<
 297     AccessInternal::PossiblyLockedAccess<T>::value, T>::type
 298   atomic_cmpxchg_maybe_locked(void* addr, T compare_value, T new_value);
 299 
 300   template <DecoratorSet ds, typename T>
 301   static inline typename EnableIf<
 302     !AccessInternal::PossiblyLockedAccess<T>::value, T>::type
 303   atomic_xchg_maybe_locked(void* addr, T new_value) {
 304     return atomic_xchg_internal<ds>(addr, new_value);
 305   }
 306 
 307   template <DecoratorSet ds, typename T>
 308   static typename EnableIf<
 309     AccessInternal::PossiblyLockedAccess<T>::value, T>::type
 310   atomic_xchg_maybe_locked(void* addr, T new_value);
 311 
 312 public:
 313   template <typename T>
 314   static inline void store(void* addr, T value) {
 315     store_internal<decorators>(addr, value);
 316   }
 317 
 318   template <typename T>
 319   static inline T load(void* addr) {
 320     return load_internal<decorators, T>(addr);
 321   }
 322 
 323   template <typename T>
 324   static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 325     return atomic_cmpxchg_maybe_locked<decorators>(addr, compare_value, new_value);
 326   }
 327 
 328   template <typename T>
 329   static inline T atomic_xchg(void* addr, T new_value) {
 330     return atomic_xchg_maybe_locked<decorators>(addr, new_value);
 331   }
 332 
 333   template <typename T>
 334   static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 335                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 336                         size_t length);
 337 
 338   template <typename T>
 339   static void oop_store(void* addr, T value);
 340   template <typename T>
 341   static void oop_store_at(oop base, ptrdiff_t offset, T value);
 342 
 343   template <typename T>
 344   static T oop_load(void* addr);
 345   template <typename T>
 346   static T oop_load_at(oop base, ptrdiff_t offset);
 347 
 348   template <typename T>
 349   static T oop_atomic_cmpxchg(void* addr, T compare_value, T new_value);
 350   template <typename T>
 351   static T oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value);
 352 
 353   template <typename T>
 354   static T oop_atomic_xchg(void* addr, T new_value);
 355   template <typename T>
 356   static T oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value);
 357 
 358   template <typename T>
 359   static void store_at(oop base, ptrdiff_t offset, T value) {
 360     store(field_addr(base, offset), value);
 361   }
 362 
 363   template <typename T>
 364   static T load_at(oop base, ptrdiff_t offset) {
 365     return load<T>(field_addr(base, offset));
 366   }
 367 
 368   template <typename T>
 369   static T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
 370     return atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
 371   }
 372 
 373   template <typename T>
 374   static T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 375     return atomic_xchg(field_addr(base, offset), new_value);
 376   }
 377 
 378   template <typename T>
 379   static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 380                             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 381                             size_t length);
 382 
 383   static void clone(oop src, oop dst, size_t size);
 384 };
 385 
 386 namespace AccessInternal {
 387   DEBUG_ONLY(void check_access_thread_state());
 388 #define assert_access_thread_state() DEBUG_ONLY(check_access_thread_state())
 389 }
 390 
 391 // Below is the implementation of the first 4 steps of the template pipeline:
 392 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
 393 //           and sets default decorators to sensible values.
 394 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
 395 //           multiple types. The P type of the address and T type of the value must
 396 //           match.
 397 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
 398 //           avoided, and in that case avoids it (calling raw accesses or
 399 //           primitive accesses in a build that does not require primitive GC barriers)
 400 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
 401 //           BarrierSet::AccessBarrier accessor that attaches GC-required barriers
 402 //           to the access.
 403 
 404 namespace AccessInternal {
 405   template <typename T>
 406   struct OopOrNarrowOopInternal: AllStatic {
 407     typedef oop type;
 408   };
 409 
 410   template <>
 411   struct OopOrNarrowOopInternal<narrowOop>: AllStatic {
 412     typedef narrowOop type;
 413   };
 414 
 415   // This metafunction returns a canonicalized oop/narrowOop type for a passed
 416   // in oop-like types passed in from oop_* overloads where the user has sworn
 417   // that the passed in values should be oop-like (e.g. oop, oopDesc*, arrayOop,
 418   // narrowOoop, instanceOopDesc*, and random other things).
 419   // In the oop_* overloads, it must hold that if the passed in type T is not
 420   // narrowOop, then it by contract has to be one of many oop-like types implicitly
 421   // convertible to oop, and hence returns oop as the canonical oop type.
 422   // If it turns out it was not, then the implicit conversion to oop will fail
 423   // to compile, as desired.
 424   template <typename T>
 425   struct OopOrNarrowOop: AllStatic {
 426     typedef typename OopOrNarrowOopInternal<std::decay_t<T>>::type type;
 427   };
 428 
 429   inline void* field_addr(oop base, ptrdiff_t byte_offset) {
 430     return reinterpret_cast<void*>(reinterpret_cast<intptr_t>((void*)base) + byte_offset);
 431   }
 432   // Step 4: Runtime dispatch
 433   // The RuntimeDispatch class is responsible for performing a runtime dispatch of the
 434   // accessor. This is required when the access either depends on whether compressed oops
 435   // is being used, or it depends on which GC implementation was chosen (e.g. requires GC
 436   // barriers). The way it works is that a function pointer initially pointing to an
 437   // accessor resolution function gets called for each access. Upon first invocation,
 438   // it resolves which accessor to be used in future invocations and patches the
 439   // function pointer to this new accessor.
 440 
 441   template <DecoratorSet decorators, typename T, BarrierType type>
 442   struct RuntimeDispatch: AllStatic {};
 443 
 444   template <DecoratorSet decorators, typename T>
 445   struct RuntimeDispatch<decorators, T, BARRIER_STORE>: AllStatic {
 446     typedef typename AccessFunction<decorators, T, BARRIER_STORE>::type func_t;
 447     static func_t _store_func;
 448 
 449     static void store_init(void* addr, T value);
 450 
 451     static inline void store(void* addr, T value) {
 452       assert_access_thread_state();
 453       _store_func(addr, value);
 454     }
 455   };
 456 
 457   template <DecoratorSet decorators, typename T>
 458   struct RuntimeDispatch<decorators, T, BARRIER_STORE_AT>: AllStatic {
 459     typedef typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type func_t;
 460     static func_t _store_at_func;
 461 
 462     static void store_at_init(oop base, ptrdiff_t offset, T value);
 463 
 464     static inline void store_at(oop base, ptrdiff_t offset, T value) {
 465       assert_access_thread_state();
 466       _store_at_func(base, offset, value);
 467     }
 468   };
 469 
 470   template <DecoratorSet decorators, typename T>
 471   struct RuntimeDispatch<decorators, T, BARRIER_LOAD>: AllStatic {
 472     typedef typename AccessFunction<decorators, T, BARRIER_LOAD>::type func_t;
 473     static func_t _load_func;
 474 
 475     static T load_init(void* addr);
 476 
 477     static inline T load(void* addr) {
 478       assert_access_thread_state();
 479       return _load_func(addr);
 480     }
 481   };
 482 
 483   template <DecoratorSet decorators, typename T>
 484   struct RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>: AllStatic {
 485     typedef typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type func_t;
 486     static func_t _load_at_func;
 487 
 488     static T load_at_init(oop base, ptrdiff_t offset);
 489 
 490     static inline T load_at(oop base, ptrdiff_t offset) {
 491       assert_access_thread_state();
 492       return _load_at_func(base, offset);
 493     }
 494   };
 495 
 496   template <DecoratorSet decorators, typename T>
 497   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>: AllStatic {
 498     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type func_t;
 499     static func_t _atomic_cmpxchg_func;
 500 
 501     static T atomic_cmpxchg_init(void* addr, T compare_value, T new_value);
 502 
 503     static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 504       assert_access_thread_state();
 505       return _atomic_cmpxchg_func(addr, compare_value, new_value);
 506     }
 507   };
 508 
 509   template <DecoratorSet decorators, typename T>
 510   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>: AllStatic {
 511     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type func_t;
 512     static func_t _atomic_cmpxchg_at_func;
 513 
 514     static T atomic_cmpxchg_at_init(oop base, ptrdiff_t offset, T compare_value, T new_value);
 515 
 516     static inline T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
 517       assert_access_thread_state();
 518       return _atomic_cmpxchg_at_func(base, offset, compare_value, new_value);
 519     }
 520   };
 521 
 522   template <DecoratorSet decorators, typename T>
 523   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>: AllStatic {
 524     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type func_t;
 525     static func_t _atomic_xchg_func;
 526 
 527     static T atomic_xchg_init(void* addr, T new_value);
 528 
 529     static inline T atomic_xchg(void* addr, T new_value) {
 530       assert_access_thread_state();
 531       return _atomic_xchg_func(addr, new_value);
 532     }
 533   };
 534 
 535   template <DecoratorSet decorators, typename T>
 536   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
 537     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
 538     static func_t _atomic_xchg_at_func;
 539 
 540     static T atomic_xchg_at_init(oop base, ptrdiff_t offset, T new_value);
 541 
 542     static inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 543       assert_access_thread_state();
 544       return _atomic_xchg_at_func(base, offset, new_value);
 545     }
 546   };
 547 
 548   template <DecoratorSet decorators, typename T>
 549   struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
 550     typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
 551     static func_t _arraycopy_func;
 552 
 553     static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 554                                arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 555                                size_t length);
 556 
 557     static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 558                                  arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 559                                  size_t length) {
 560       assert_access_thread_state();
 561       return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
 562                              dst_obj, dst_offset_in_bytes, dst_raw,
 563                              length);
 564     }
 565   };
 566 
 567   template <DecoratorSet decorators, typename T>
 568   struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
 569     typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
 570     static func_t _clone_func;
 571 
 572     static void clone_init(oop src, oop dst, size_t size);
 573 
 574     static inline void clone(oop src, oop dst, size_t size) {
 575       assert_access_thread_state();
 576       _clone_func(src, dst, size);
 577     }
 578   };
 579 
 580   // Initialize the function pointers to point to the resolving function.
 581   template <DecoratorSet decorators, typename T>
 582   typename AccessFunction<decorators, T, BARRIER_STORE>::type
 583   RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init;
 584 
 585   template <DecoratorSet decorators, typename T>
 586   typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type
 587   RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init;
 588 
 589   template <DecoratorSet decorators, typename T>
 590   typename AccessFunction<decorators, T, BARRIER_LOAD>::type
 591   RuntimeDispatch<decorators, T, BARRIER_LOAD>::_load_func = &load_init;
 592 
 593   template <DecoratorSet decorators, typename T>
 594   typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type
 595   RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::_load_at_func = &load_at_init;
 596 
 597   template <DecoratorSet decorators, typename T>
 598   typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type
 599   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::_atomic_cmpxchg_func = &atomic_cmpxchg_init;
 600 
 601   template <DecoratorSet decorators, typename T>
 602   typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type
 603   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init;
 604 
 605   template <DecoratorSet decorators, typename T>
 606   typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type
 607   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init;
 608 
 609   template <DecoratorSet decorators, typename T>
 610   typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type
 611   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init;
 612 
 613   template <DecoratorSet decorators, typename T>
 614   typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type
 615   RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init;
 616 
 617   template <DecoratorSet decorators, typename T>
 618   typename AccessFunction<decorators, T, BARRIER_CLONE>::type
 619   RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
 620 
 621   // Step 3: Pre-runtime dispatching.
 622   // The PreRuntimeDispatch class is responsible for filtering the barrier strength
 623   // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime
 624   // dispatch point. Otherwise it goes through a runtime check if hardwiring was
 625   // not possible.
 626   struct PreRuntimeDispatch: AllStatic {
 627     template<DecoratorSet decorators>
 628     struct CanHardwireRaw: public std::integral_constant<
 629       bool,
 630       !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access
 631       !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address)
 632       HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address)
 633     {};
 634 
 635     static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP;
 636 
 637     template<DecoratorSet decorators>
 638     static bool is_hardwired_primitive() {
 639       return !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value;
 640     }
 641 
 642     template <DecoratorSet decorators, typename T>
 643     inline static typename EnableIf<
 644       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value>::type
 645     store(void* addr, T value) {
 646       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 647       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 648         Raw::oop_store(addr, value);
 649       } else {
 650         Raw::store(addr, value);
 651       }
 652     }
 653 
 654     template <DecoratorSet decorators, typename T>
 655     inline static typename EnableIf<
 656       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value>::type
 657     store(void* addr, T value) {
 658       if (UseCompressedOops) {
 659         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 660         PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 661       } else {
 662         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 663         PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 664       }
 665     }
 666 
 667     template <DecoratorSet decorators, typename T>
 668     inline static typename EnableIf<
 669       !HasDecorator<decorators, AS_RAW>::value>::type
 670     store(void* addr, T value) {
 671       if (is_hardwired_primitive<decorators>()) {
 672         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 673         PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 674       } else {
 675         RuntimeDispatch<decorators, T, BARRIER_STORE>::store(addr, value);
 676       }
 677     }
 678 
 679     template <DecoratorSet decorators, typename T>
 680     inline static typename EnableIf<
 681       HasDecorator<decorators, AS_RAW>::value>::type
 682     store_at(oop base, ptrdiff_t offset, T value) {
 683       store<decorators>(field_addr(base, offset), value);
 684     }
 685 
 686     template <DecoratorSet decorators, typename T>
 687     inline static typename EnableIf<
 688       !HasDecorator<decorators, AS_RAW>::value>::type
 689     store_at(oop base, ptrdiff_t offset, T value) {
 690       if (is_hardwired_primitive<decorators>()) {
 691         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 692         PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, value);
 693       } else {
 694         RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::store_at(base, offset, value);
 695       }
 696     }
 697 
 698     template <DecoratorSet decorators, typename T>
 699     inline static typename EnableIf<
 700       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type
 701     load(void* addr) {
 702       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 703       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 704         return Raw::template oop_load<T>(addr);
 705       } else {
 706         return Raw::template load<T>(addr);
 707       }
 708     }
 709 
 710     template <DecoratorSet decorators, typename T>
 711     inline static typename EnableIf<
 712       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type
 713     load(void* addr) {
 714       if (UseCompressedOops) {
 715         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 716         return PreRuntimeDispatch::load<expanded_decorators, T>(addr);
 717       } else {
 718         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 719         return PreRuntimeDispatch::load<expanded_decorators, T>(addr);
 720       }
 721     }
 722 
 723     template <DecoratorSet decorators, typename T>
 724     inline static typename EnableIf<
 725       !HasDecorator<decorators, AS_RAW>::value, T>::type
 726     load(void* addr) {
 727       if (is_hardwired_primitive<decorators>()) {
 728         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 729         return PreRuntimeDispatch::load<expanded_decorators, T>(addr);
 730       } else {
 731         return RuntimeDispatch<decorators, T, BARRIER_LOAD>::load(addr);
 732       }
 733     }
 734 
 735     template <DecoratorSet decorators, typename T>
 736     inline static typename EnableIf<
 737       HasDecorator<decorators, AS_RAW>::value, T>::type
 738     load_at(oop base, ptrdiff_t offset) {
 739       return load<decorators, T>(field_addr(base, offset));
 740     }
 741 
 742     template <DecoratorSet decorators, typename T>
 743     inline static typename EnableIf<
 744       !HasDecorator<decorators, AS_RAW>::value, T>::type
 745     load_at(oop base, ptrdiff_t offset) {
 746       if (is_hardwired_primitive<decorators>()) {
 747         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 748         return PreRuntimeDispatch::load_at<expanded_decorators, T>(base, offset);
 749       } else {
 750         return RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::load_at(base, offset);
 751       }
 752     }
 753 
 754     template <DecoratorSet decorators, typename T>
 755     inline static typename EnableIf<
 756       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type
 757     atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 758       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 759       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 760         return Raw::oop_atomic_cmpxchg(addr, compare_value, new_value);
 761       } else {
 762         return Raw::atomic_cmpxchg(addr, compare_value, new_value);
 763       }
 764     }
 765 
 766     template <DecoratorSet decorators, typename T>
 767     inline static typename EnableIf<
 768       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type
 769     atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 770       if (UseCompressedOops) {
 771         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 772         return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
 773       } else {
 774         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 775         return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
 776       }
 777     }
 778 
 779     template <DecoratorSet decorators, typename T>
 780     inline static typename EnableIf<
 781       !HasDecorator<decorators, AS_RAW>::value, T>::type
 782     atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 783       if (is_hardwired_primitive<decorators>()) {
 784         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 785         return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
 786       } else {
 787         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::atomic_cmpxchg(addr, compare_value, new_value);
 788       }
 789     }
 790 
 791     template <DecoratorSet decorators, typename T>
 792     inline static typename EnableIf<
 793       HasDecorator<decorators, AS_RAW>::value, T>::type
 794     atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
 795       return atomic_cmpxchg<decorators>(field_addr(base, offset), compare_value, new_value);
 796     }
 797 
 798     template <DecoratorSet decorators, typename T>
 799     inline static typename EnableIf<
 800       !HasDecorator<decorators, AS_RAW>::value, T>::type
 801     atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
 802       if (is_hardwired_primitive<decorators>()) {
 803         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 804         return PreRuntimeDispatch::atomic_cmpxchg_at<expanded_decorators>(base, offset, compare_value, new_value);
 805       } else {
 806         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::atomic_cmpxchg_at(base, offset, compare_value, new_value);
 807       }
 808     }
 809 
 810     template <DecoratorSet decorators, typename T>
 811     inline static typename EnableIf<
 812       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type
 813     atomic_xchg(void* addr, T new_value) {
 814       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 815       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 816         return Raw::oop_atomic_xchg(addr, new_value);
 817       } else {
 818         return Raw::atomic_xchg(addr, new_value);
 819       }
 820     }
 821 
 822     template <DecoratorSet decorators, typename T>
 823     inline static typename EnableIf<
 824       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type
 825     atomic_xchg(void* addr, T new_value) {
 826       if (UseCompressedOops) {
 827         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 828         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
 829       } else {
 830         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 831         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
 832       }
 833     }
 834 
 835     template <DecoratorSet decorators, typename T>
 836     inline static typename EnableIf<
 837       !HasDecorator<decorators, AS_RAW>::value, T>::type
 838     atomic_xchg(void* addr, T new_value) {
 839       if (is_hardwired_primitive<decorators>()) {
 840         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 841         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
 842       } else {
 843         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::atomic_xchg(addr, new_value);
 844       }
 845     }
 846 
 847     template <DecoratorSet decorators, typename T>
 848     inline static typename EnableIf<
 849       HasDecorator<decorators, AS_RAW>::value, T>::type
 850     atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 851       return atomic_xchg<decorators>(field_addr(base, offset), new_value);
 852     }
 853 
 854     template <DecoratorSet decorators, typename T>
 855     inline static typename EnableIf<
 856       !HasDecorator<decorators, AS_RAW>::value, T>::type
 857     atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 858       if (is_hardwired_primitive<decorators>()) {
 859         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 860         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(base, offset, new_value);
 861       } else {
 862         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(base, offset, new_value);
 863       }
 864     }
 865 
 866     template <DecoratorSet decorators, typename T>
 867     inline static typename EnableIf<
 868       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
 869     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 870               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 871               size_t length) {
 872       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 873       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 874         return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
 875                                   dst_obj, dst_offset_in_bytes, dst_raw,
 876                                   length);
 877       } else {
 878         return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 879                               dst_obj, dst_offset_in_bytes, dst_raw,
 880                               length);
 881       }
 882     }
 883 
 884     template <DecoratorSet decorators, typename T>
 885     inline static typename EnableIf<
 886       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
 887     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 888               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 889               size_t length) {
 890       if (UseCompressedOops) {
 891         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 892         return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 893                                                                   dst_obj, dst_offset_in_bytes, dst_raw,
 894                                                                   length);
 895       } else {
 896         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 897         return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 898                                                                   dst_obj, dst_offset_in_bytes, dst_raw,
 899                                                                   length);
 900       }
 901     }
 902 
 903     template <DecoratorSet decorators, typename T>
 904     inline static typename EnableIf<
 905       !HasDecorator<decorators, AS_RAW>::value, bool>::type
 906     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 907               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 908               size_t length) {
 909       if (is_hardwired_primitive<decorators>()) {
 910         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 911         return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 912                                                                   dst_obj, dst_offset_in_bytes, dst_raw,
 913                                                                   length);
 914       } else {
 915         return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 916                                                                             dst_obj, dst_offset_in_bytes, dst_raw,
 917                                                                             length);
 918       }
 919     }
 920 
 921     template <DecoratorSet decorators>
 922     inline static typename EnableIf<
 923       HasDecorator<decorators, AS_RAW>::value>::type
 924     clone(oop src, oop dst, size_t size) {
 925       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 926       Raw::clone(src, dst, size);
 927     }
 928 
 929     template <DecoratorSet decorators>
 930     inline static typename EnableIf<
 931       !HasDecorator<decorators, AS_RAW>::value>::type
 932     clone(oop src, oop dst, size_t size) {
 933       RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
 934     }
 935   };
 936 
 937   // Step 2: Reduce types.
 938   // Enforce that for non-oop types, T and P have to be strictly the same.
 939   // P is the type of the address and T is the type of the values.
 940   // As for oop types, it is allow to send T in {narrowOop, oop} and
 941   // P in {narrowOop, oop, HeapWord*}. The following rules apply according to
 942   // the subsequent table. (columns are P, rows are T)
 943   // |           | HeapWord  |   oop   | narrowOop |
 944   // |   oop     |  rt-comp  | hw-none |  hw-comp  |
 945   // | narrowOop |     x     |    x    |  hw-none  |
 946   //
 947   // x means not allowed
 948   // rt-comp means it must be checked at runtime whether the oop is compressed.
 949   // hw-none means it is statically known the oop will not be compressed.
 950   // hw-comp means it is statically known the oop will be compressed.
 951 
 952   template <DecoratorSet decorators, typename T>
 953   inline void store_reduce_types(T* addr, T value) {
 954     PreRuntimeDispatch::store<decorators>(addr, value);
 955   }
 956 
 957   template <DecoratorSet decorators>
 958   inline void store_reduce_types(narrowOop* addr, oop value) {
 959     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
 960                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
 961     PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 962   }
 963 
 964   template <DecoratorSet decorators>
 965   inline void store_reduce_types(narrowOop* addr, narrowOop value) {
 966     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
 967                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
 968     PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 969   }
 970 
 971   template <DecoratorSet decorators>
 972   inline void store_reduce_types(HeapWord* addr, oop value) {
 973     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
 974     PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 975   }
 976 
 977   template <DecoratorSet decorators, typename T>
 978   inline T atomic_cmpxchg_reduce_types(T* addr, T compare_value, T new_value) {
 979     return PreRuntimeDispatch::atomic_cmpxchg<decorators>(addr, compare_value, new_value);
 980   }
 981 
 982   template <DecoratorSet decorators>
 983   inline oop atomic_cmpxchg_reduce_types(narrowOop* addr, oop compare_value, oop new_value) {
 984     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
 985                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
 986     return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
 987   }
 988 
 989   template <DecoratorSet decorators>
 990   inline narrowOop atomic_cmpxchg_reduce_types(narrowOop* addr, narrowOop compare_value, narrowOop new_value) {
 991     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
 992                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
 993     return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
 994   }
 995 
 996   template <DecoratorSet decorators>
 997   inline oop atomic_cmpxchg_reduce_types(HeapWord* addr,
 998                                          oop compare_value,
 999                                          oop new_value) {
1000     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1001     return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
1002   }
1003 
1004   template <DecoratorSet decorators, typename T>
1005   inline T atomic_xchg_reduce_types(T* addr, T new_value) {
1006     const DecoratorSet expanded_decorators = decorators;
1007     return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
1008   }
1009 
1010   template <DecoratorSet decorators>
1011   inline oop atomic_xchg_reduce_types(narrowOop* addr, oop new_value) {
1012     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1013                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1014     return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
1015   }
1016 
1017   template <DecoratorSet decorators>
1018   inline narrowOop atomic_xchg_reduce_types(narrowOop* addr, narrowOop new_value) {
1019     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1020                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1021     return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
1022   }
1023 
1024   template <DecoratorSet decorators>
1025   inline oop atomic_xchg_reduce_types(HeapWord* addr, oop new_value) {
1026     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1027     return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
1028   }
1029 
1030   template <DecoratorSet decorators, typename T>
1031   inline T load_reduce_types(T* addr) {
1032     return PreRuntimeDispatch::load<decorators, T>(addr);
1033   }
1034 
1035   template <DecoratorSet decorators, typename T>
1036   inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1037     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1038                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1039     return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1040   }
1041 
1042   template <DecoratorSet decorators, typename T>
1043   inline oop load_reduce_types(HeapWord* addr) {
1044     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1045     return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1046   }
1047 
1048   template <DecoratorSet decorators, typename T>
1049   inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1050                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1051                                      size_t length) {
1052     return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1053                                                      dst_obj, dst_offset_in_bytes, dst_raw,
1054                                                      length);
1055   }
1056 
1057   template <DecoratorSet decorators>
1058   inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1059                                      arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1060                                      size_t length) {
1061     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1062     return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1063                                                               dst_obj, dst_offset_in_bytes, dst_raw,
1064                                                               length);
1065   }
1066 
1067   template <DecoratorSet decorators>
1068   inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1069                                      arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1070                                      size_t length) {
1071     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1072                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1073     return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1074                                                               dst_obj, dst_offset_in_bytes, dst_raw,
1075                                                               length);
1076   }
1077 
1078   // Step 1: Set default decorators. This step remembers if a type was volatile
1079   // and then sets the MO_RELAXED decorator by default. Otherwise, a default
1080   // memory ordering is set for the access, and the implied decorator rules
1081   // are applied to select sensible defaults for decorators that have not been
1082   // explicitly set. For example, default object referent strength is set to strong.
1083   // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1084   // and references from the types). This step also perform some type verification
1085   // that the passed in types make sense.
1086 
1087   template <DecoratorSet decorators, typename T>
1088   static void verify_types(){
1089     // If this fails to compile, then you have sent in something that is
1090     // not recognized as a valid primitive type to a primitive Access function.
1091     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1092                    (std::is_pointer<T>::value || std::is_integral<T>::value) ||
1093                     std::is_floating_point<T>::value)); // not allowed primitive type
1094   }
1095 
1096   template <DecoratorSet decorators, typename P, typename T>
1097   inline void store(P* addr, T value) {
1098     verify_types<decorators, T>();
1099     using DecayedP = std::decay_t<P>;
1100     using DecayedT = std::decay_t<T>;
1101     DecayedT decayed_value = value;
1102     // If a volatile address is passed in but no memory ordering decorator,
1103     // set the memory ordering to MO_RELAXED by default.
1104     const DecoratorSet expanded_decorators = DecoratorFixup<
1105       (std::is_volatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1106       (MO_RELAXED | decorators) : decorators>::value;
1107     store_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), decayed_value);
1108   }
1109 
1110   template <DecoratorSet decorators, typename T>
1111   inline void store_at(oop base, ptrdiff_t offset, T value) {
1112     verify_types<decorators, T>();
1113     using DecayedT = std::decay_t<T>;
1114     DecayedT decayed_value = value;
1115     const DecoratorSet expanded_decorators = DecoratorFixup<decorators |
1116                                              (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1117                                               INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1118     PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, decayed_value);
1119   }
1120 
1121   template <DecoratorSet decorators, typename P, typename T>
1122   inline T load(P* addr) {
1123     verify_types<decorators, T>();
1124     using DecayedP = std::decay_t<P>;
1125     using DecayedT = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
1126                                         typename OopOrNarrowOop<T>::type,
1127                                         std::decay_t<T>>;
1128     // If a volatile address is passed in but no memory ordering decorator,
1129     // set the memory ordering to MO_RELAXED by default.
1130     const DecoratorSet expanded_decorators = DecoratorFixup<
1131       (std::is_volatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1132       (MO_RELAXED | decorators) : decorators>::value;
1133     return load_reduce_types<expanded_decorators, DecayedT>(const_cast<DecayedP*>(addr));
1134   }
1135 
1136   template <DecoratorSet decorators, typename T>
1137   inline T load_at(oop base, ptrdiff_t offset) {
1138     verify_types<decorators, T>();
1139     using DecayedT = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
1140                                         typename OopOrNarrowOop<T>::type,
1141                                         std::decay_t<T>>;
1142     // Expand the decorators (figure out sensible defaults)
1143     // Potentially remember if we need compressed oop awareness
1144     const DecoratorSet expanded_decorators = DecoratorFixup<decorators |
1145                                              (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1146                                               INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1147     return PreRuntimeDispatch::load_at<expanded_decorators, DecayedT>(base, offset);
1148   }
1149 
1150   template <DecoratorSet decorators, typename P, typename T>
1151   inline T atomic_cmpxchg(P* addr, T compare_value, T new_value) {
1152     verify_types<decorators, T>();
1153     using DecayedP = std::decay_t<P>;
1154     using DecayedT = std::decay_t<T>;
1155     DecayedT new_decayed_value = new_value;
1156     DecayedT compare_decayed_value = compare_value;
1157     const DecoratorSet expanded_decorators = DecoratorFixup<
1158       (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1159       (MO_SEQ_CST | decorators) : decorators>::value;
1160     return atomic_cmpxchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr),
1161                                                             compare_decayed_value,
1162                                                             new_decayed_value);
1163   }
1164 
1165   template <DecoratorSet decorators, typename T>
1166   inline T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
1167     verify_types<decorators, T>();
1168     using DecayedT = std::decay_t<T>;
1169     DecayedT new_decayed_value = new_value;
1170     DecayedT compare_decayed_value = compare_value;
1171     // Determine default memory ordering
1172     const DecoratorSet expanded_decorators = DecoratorFixup<
1173       (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1174       (MO_SEQ_CST | decorators) : decorators>::value;
1175     // Potentially remember that we need compressed oop awareness
1176     const DecoratorSet final_decorators = expanded_decorators |
1177                                           (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1178                                            INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE);
1179     return PreRuntimeDispatch::atomic_cmpxchg_at<final_decorators>(base, offset, compare_decayed_value,
1180                                                                    new_decayed_value);
1181   }
1182 
1183   template <DecoratorSet decorators, typename P, typename T>
1184   inline T atomic_xchg(P* addr, T new_value) {
1185     verify_types<decorators, T>();
1186     using DecayedP = std::decay_t<P>;
1187     using DecayedT = std::decay_t<T>;
1188     DecayedT new_decayed_value = new_value;
1189     // atomic_xchg is only available in SEQ_CST flavour.
1190     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1191     return atomic_xchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr),
1192                                                          new_decayed_value);
1193   }
1194 
1195   template <DecoratorSet decorators, typename T>
1196   inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
1197     verify_types<decorators, T>();
1198     using DecayedT = std::decay_t<T>;
1199     DecayedT new_decayed_value = new_value;
1200     // atomic_xchg is only available in SEQ_CST flavour.
1201     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1202                                              (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1203                                               INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1204     return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(base, offset, new_decayed_value);
1205   }
1206 
1207   template <DecoratorSet decorators, typename T>
1208   inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1209                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1210                         size_t length) {
1211     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1212                    (std::is_same<T, void>::value || std::is_integral<T>::value) ||
1213                     std::is_floating_point<T>::value)); // arraycopy allows type erased void elements
1214     using DecayedT = std::decay_t<T>;
1215     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1216     return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1217                                                        dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1218                                                        length);
1219   }
1220 
1221   template <DecoratorSet decorators>
1222   inline void clone(oop src, oop dst, size_t size) {
1223     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1224     PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1225   }
1226 
1227   // Infer the type that should be returned from an Access::oop_load.
1228   template <typename P, DecoratorSet decorators>
1229   class OopLoadProxy: public StackObj {
1230   private:
1231     P *const _addr;
1232   public:
1233     explicit OopLoadProxy(P* addr) : _addr(addr) {}
1234 
1235     inline operator oop() {
1236       return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr);
1237     }
1238 
1239     inline operator narrowOop() {
1240       return load<decorators | INTERNAL_VALUE_IS_OOP, P, narrowOop>(_addr);
1241     }
1242 
1243     template <typename T>
1244     inline bool operator ==(const T& other) const {
1245       return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) == other;
1246     }
1247 
1248     template <typename T>
1249     inline bool operator !=(const T& other) const {
1250       return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) != other;
1251     }
1252 
1253     inline bool operator ==(std::nullptr_t) const {
1254       return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr) == nullptr;
1255     }
1256 
1257     inline bool operator !=(std::nullptr_t) const {
1258       return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr) != nullptr;
1259     }
1260   };
1261 
1262   // Infer the type that should be returned from an Access::load_at.
1263   template <DecoratorSet decorators>
1264   class LoadAtProxy: public StackObj {
1265   private:
1266     const oop _base;
1267     const ptrdiff_t _offset;
1268   public:
1269     LoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {}
1270 
1271     template <typename T>
1272     inline operator T() const {
1273       return load_at<decorators, T>(_base, _offset);
1274     }
1275 
1276     template <typename T>
1277     inline bool operator ==(const T& other) const { return load_at<decorators, T>(_base, _offset) == other; }
1278 
1279     template <typename T>
1280     inline bool operator !=(const T& other) const { return load_at<decorators, T>(_base, _offset) != other; }
1281   };
1282 
1283   // Infer the type that should be returned from an Access::oop_load_at.
1284   template <DecoratorSet decorators>
1285   class OopLoadAtProxy: public StackObj {
1286   private:
1287     const oop _base;
1288     const ptrdiff_t _offset;
1289   public:
1290     OopLoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {}
1291 
1292     inline operator oop() const {
1293       return load_at<decorators | INTERNAL_VALUE_IS_OOP, oop>(_base, _offset);
1294     }
1295 
1296     inline operator narrowOop() const {
1297       return load_at<decorators | INTERNAL_VALUE_IS_OOP, narrowOop>(_base, _offset);
1298     }
1299 
1300     template <typename T>
1301     inline bool operator ==(const T& other) const {
1302       return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) == other;
1303     }
1304 
1305     template <typename T>
1306     inline bool operator !=(const T& other) const {
1307       return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) != other;
1308     }
1309   };
1310 }
1311 
1312 #endif // SHARE_OOPS_ACCESSBACKEND_HPP