1 /*
   2  * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_OOPS_ACCESSBACKEND_HPP
  26 #define SHARE_OOPS_ACCESSBACKEND_HPP
  27 
  28 #include "gc/shared/barrierSetConfig.hpp"
  29 #include "memory/allocation.hpp"
  30 #include "metaprogramming/enableIf.hpp"
  31 #include "oops/accessDecorators.hpp"
  32 #include "oops/inlineKlass.hpp"
  33 #include "oops/oopsHierarchy.hpp"
  34 #include "runtime/globals.hpp"
  35 #include "utilities/debug.hpp"
  36 #include "utilities/globalDefinitions.hpp"
  37 
  38 #include <type_traits>
  39 
  40 // This metafunction returns either oop or narrowOop depending on whether
  41 // an access needs to use compressed oops or not.
  42 template <DecoratorSet decorators>
  43 struct HeapOopType: AllStatic {
  44   static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
  45                                          HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
  46   using type = std::conditional_t<needs_oop_compress, narrowOop, oop>;
  47 };
  48 
  49 // This meta-function returns either oop or narrowOop depending on whether
  50 // a back-end needs to consider compressed oops types or not.
  51 template <DecoratorSet decorators>
  52 struct ValueOopType: AllStatic {
  53   static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
  54   using type = std::conditional_t<needs_oop_compress, narrowOop, oop>;
  55 };
  56 
  57 namespace AccessInternal {
  58   enum BarrierType {
  59     BARRIER_STORE,
  60     BARRIER_STORE_AT,
  61     BARRIER_LOAD,
  62     BARRIER_LOAD_AT,
  63     BARRIER_ATOMIC_CMPXCHG,
  64     BARRIER_ATOMIC_CMPXCHG_AT,
  65     BARRIER_ATOMIC_XCHG,
  66     BARRIER_ATOMIC_XCHG_AT,
  67     BARRIER_ARRAYCOPY,
  68     BARRIER_CLONE,
  69     BARRIER_VALUE_COPY
  70   };
  71 
  72   template <DecoratorSet decorators, typename T>
  73   struct MustConvertCompressedOop: public std::integral_constant<bool,
  74     HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
  75     std::is_same<typename HeapOopType<decorators>::type, narrowOop>::value &&
  76     std::is_same<T, oop>::value> {};
  77 
  78   // This metafunction returns an appropriate oop type if the value is oop-like
  79   // and otherwise returns the same type T.
  80   template <DecoratorSet decorators, typename T>
  81   struct EncodedType: AllStatic {
  82     using type = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
  83                                     typename HeapOopType<decorators>::type,
  84                                     T>;
  85   };
  86 
  87   template <DecoratorSet decorators>
  88   inline typename HeapOopType<decorators>::type*
  89   oop_field_addr(oop base, ptrdiff_t byte_offset) {
  90     return reinterpret_cast<typename HeapOopType<decorators>::type*>(
  91              reinterpret_cast<intptr_t>((void*)base) + byte_offset);
  92   }
  93 
  94   template <DecoratorSet decorators, typename T>
  95   struct AccessFunctionTypes {
  96     typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
  97     typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
  98     typedef T (*atomic_cmpxchg_at_func_t)(oop base, ptrdiff_t offset, T compare_value, T new_value);
  99     typedef T (*atomic_xchg_at_func_t)(oop base, ptrdiff_t offset, T new_value);
 100 
 101     typedef T (*load_func_t)(void* addr);
 102     typedef void (*store_func_t)(void* addr, T value);
 103     typedef T (*atomic_cmpxchg_func_t)(void* addr, T compare_value, T new_value);
 104     typedef T (*atomic_xchg_func_t)(void* addr, T new_value);
 105 
 106     typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 107                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 108                                      size_t length);
 109     typedef void (*clone_func_t)(oop src, oop dst, size_t size);
 110     typedef void (*value_copy_func_t)(void* src, void* dst, InlineKlass* md, LayoutKind lk);
 111   };
 112 
 113   template <DecoratorSet decorators>
 114   struct AccessFunctionTypes<decorators, void> {
 115     typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
 116                                      arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
 117                                      size_t length);
 118   };
 119 
 120   template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
 121 
 122 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func)                   \
 123   template <DecoratorSet decorators, typename T>                    \
 124   struct AccessFunction<decorators, T, bt>: AllStatic{              \
 125     typedef typename AccessFunctionTypes<decorators, T>::func type; \
 126   }
 127   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
 128   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
 129   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
 130   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
 131   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
 132   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
 133   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
 134   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
 135   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
 136   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
 137   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_VALUE_COPY, value_copy_func_t);
 138 #undef ACCESS_GENERATE_ACCESS_FUNCTION
 139 
 140   template <DecoratorSet decorators, typename T, BarrierType barrier_type>
 141   typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier();
 142 
 143   template <DecoratorSet decorators, typename T, BarrierType barrier_type>
 144   typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier();
 145 
 146   void* field_addr(oop base, ptrdiff_t offset);
 147 
 148   // Forward calls to Copy:: in the cpp file to reduce dependencies and allow
 149   // faster build times, given how frequently included access is.
 150   void arraycopy_arrayof_conjoint_oops(void* src, void* dst, size_t length);
 151   void arraycopy_conjoint_oops(oop* src, oop* dst, size_t length);
 152   void arraycopy_conjoint_oops(narrowOop* src, narrowOop* dst, size_t length);
 153 
 154   void arraycopy_disjoint_words(void* src, void* dst, size_t length);
 155   void arraycopy_disjoint_words_atomic(void* src, void* dst, size_t length);
 156 
 157   template<typename T>
 158   void arraycopy_conjoint(T* src, T* dst, size_t length);
 159   template<typename T>
 160   void arraycopy_arrayof_conjoint(T* src, T* dst, size_t length);
 161   template<typename T>
 162   void arraycopy_conjoint_atomic(T* src, T* dst, size_t length);
 163 
 164   void value_copy_internal(void* src, void* dst, size_t length);
 165 }
 166 
 167 // This mask specifies what decorators are relevant for raw accesses. When passing
 168 // accesses to the raw layer, irrelevant decorators are removed.
 169 const DecoratorSet RAW_DECORATOR_MASK = INTERNAL_DECORATOR_MASK | MO_DECORATOR_MASK |
 170                                         ARRAYCOPY_DECORATOR_MASK | IS_NOT_NULL;
 171 
 172 // The RawAccessBarrier performs raw accesses with additional knowledge of
 173 // memory ordering, so that OrderAccess/Atomic is called when necessary.
 174 // It additionally handles compressed oops, and hence is not completely "raw"
 175 // strictly speaking.
 176 template <DecoratorSet decorators>
 177 class RawAccessBarrier: public AllStatic {
 178 protected:
 179   static inline void* field_addr(oop base, ptrdiff_t byte_offset) {
 180     return AccessInternal::field_addr(base, byte_offset);
 181   }
 182 
 183 protected:
 184   // Only encode if INTERNAL_VALUE_IS_OOP
 185   template <DecoratorSet idecorators, typename T>
 186   static inline typename EnableIf<
 187     AccessInternal::MustConvertCompressedOop<idecorators, T>::value,
 188     typename HeapOopType<idecorators>::type>::type
 189   encode_internal(T value);
 190 
 191   template <DecoratorSet idecorators, typename T>
 192   static inline typename EnableIf<
 193     !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
 194   encode_internal(T value) {
 195     return value;
 196   }
 197 
 198   template <typename T>
 199   static inline typename AccessInternal::EncodedType<decorators, T>::type
 200   encode(T value) {
 201     return encode_internal<decorators, T>(value);
 202   }
 203 
 204   // Only decode if INTERNAL_VALUE_IS_OOP
 205   template <DecoratorSet idecorators, typename T>
 206   static inline typename EnableIf<
 207     AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
 208   decode_internal(typename HeapOopType<idecorators>::type value);
 209 
 210   template <DecoratorSet idecorators, typename T>
 211   static inline typename EnableIf<
 212     !AccessInternal::MustConvertCompressedOop<idecorators, T>::value, T>::type
 213   decode_internal(T value) {
 214     return value;
 215   }
 216 
 217   template <typename T>
 218   static inline T decode(typename AccessInternal::EncodedType<decorators, T>::type value) {
 219     return decode_internal<decorators, T>(value);
 220   }
 221 
 222 protected:
 223   template <DecoratorSet ds, typename T>
 224   static typename EnableIf<
 225     HasDecorator<ds, MO_SEQ_CST>::value, T>::type
 226   load_internal(void* addr);
 227 
 228   template <DecoratorSet ds, typename T>
 229   static typename EnableIf<
 230     HasDecorator<ds, MO_ACQUIRE>::value, T>::type
 231   load_internal(void* addr);
 232 
 233   template <DecoratorSet ds, typename T>
 234   static typename EnableIf<
 235     HasDecorator<ds, MO_RELAXED>::value, T>::type
 236   load_internal(void* addr);
 237 
 238   template <DecoratorSet ds, typename T>
 239   static inline typename EnableIf<
 240     HasDecorator<ds, MO_UNORDERED>::value, T>::type
 241   load_internal(void* addr) {
 242     return *reinterpret_cast<T*>(addr);
 243   }
 244 
 245   template <DecoratorSet ds, typename T>
 246   static typename EnableIf<
 247     HasDecorator<ds, MO_SEQ_CST>::value>::type
 248   store_internal(void* addr, T value);
 249 
 250   template <DecoratorSet ds, typename T>
 251   static typename EnableIf<
 252     HasDecorator<ds, MO_RELEASE>::value>::type
 253   store_internal(void* addr, T value);
 254 
 255   template <DecoratorSet ds, typename T>
 256   static typename EnableIf<
 257     HasDecorator<ds, MO_RELAXED>::value>::type
 258   store_internal(void* addr, T value);
 259 
 260   template <DecoratorSet ds, typename T>
 261   static inline typename EnableIf<
 262     HasDecorator<ds, MO_UNORDERED>::value>::type
 263   store_internal(void* addr, T value) {
 264     *reinterpret_cast<T*>(addr) = value;
 265   }
 266 
 267   template <DecoratorSet ds, typename T>
 268   static typename EnableIf<
 269     HasDecorator<ds, MO_SEQ_CST>::value, T>::type
 270   atomic_cmpxchg_internal(void* addr, T compare_value, T new_value);
 271 
 272   template <DecoratorSet ds, typename T>
 273   static typename EnableIf<
 274     HasDecorator<ds, MO_RELAXED>::value, T>::type
 275   atomic_cmpxchg_internal(void* addr, T compare_value, T new_value);
 276 
 277   template <DecoratorSet ds, typename T>
 278   static typename EnableIf<
 279     HasDecorator<ds, MO_SEQ_CST>::value, T>::type
 280   atomic_xchg_internal(void* addr, T new_value);
 281 
 282 public:
 283   template <typename T>
 284   static inline void store(void* addr, T value) {
 285     store_internal<decorators>(addr, value);
 286   }
 287 
 288   template <typename T>
 289   static inline T load(void* addr) {
 290     return load_internal<decorators, T>(addr);
 291   }
 292 
 293   template <typename T>
 294   static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 295     return atomic_cmpxchg_internal<decorators>(addr, compare_value, new_value);
 296   }
 297 
 298   template <typename T>
 299   static inline T atomic_xchg(void* addr, T new_value) {
 300     return atomic_xchg_internal<decorators>(addr, new_value);
 301   }
 302 
 303   template <typename T>
 304   static void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 305                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 306                         size_t length);
 307 
 308   template <typename T>
 309   static void oop_store(void* addr, T value);
 310   template <typename T>
 311   static void oop_store_at(oop base, ptrdiff_t offset, T value);
 312 
 313   template <typename T>
 314   static T oop_load(void* addr);
 315   template <typename T>
 316   static T oop_load_at(oop base, ptrdiff_t offset);
 317 
 318   template <typename T>
 319   static T oop_atomic_cmpxchg(void* addr, T compare_value, T new_value);
 320   template <typename T>
 321   static T oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value);
 322 
 323   template <typename T>
 324   static T oop_atomic_xchg(void* addr, T new_value);
 325   template <typename T>
 326   static T oop_atomic_xchg_at(oop base, ptrdiff_t offset, T new_value);
 327 
 328   template <typename T>
 329   static void store_at(oop base, ptrdiff_t offset, T value) {
 330     store(field_addr(base, offset), value);
 331   }
 332 
 333   template <typename T>
 334   static T load_at(oop base, ptrdiff_t offset) {
 335     return load<T>(field_addr(base, offset));
 336   }
 337 
 338   template <typename T>
 339   static T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
 340     return atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
 341   }
 342 
 343   template <typename T>
 344   static T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 345     return atomic_xchg(field_addr(base, offset), new_value);
 346   }
 347 
 348   template <typename T>
 349   static void oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 350                             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 351                             size_t length);
 352 
 353   static void clone(oop src, oop dst, size_t size);
 354   static void value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk);
 355 
 356 };
 357 
 358 namespace AccessInternal {
 359   DEBUG_ONLY(void check_access_thread_state());
 360 #define assert_access_thread_state() DEBUG_ONLY(check_access_thread_state())
 361 }
 362 
 363 // Below is the implementation of the first 4 steps of the template pipeline:
 364 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
 365 //           and sets default decorators to sensible values.
 366 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
 367 //           multiple types. The P type of the address and T type of the value must
 368 //           match.
 369 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
 370 //           avoided, and in that case avoids it (calling raw accesses or
 371 //           primitive accesses in a build that does not require primitive GC barriers)
 372 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
 373 //           BarrierSet::AccessBarrier accessor that attaches GC-required barriers
 374 //           to the access.
 375 
 376 namespace AccessInternal {
 377   template <typename T>
 378   struct OopOrNarrowOopInternal: AllStatic {
 379     typedef oop type;
 380   };
 381 
 382   template <>
 383   struct OopOrNarrowOopInternal<narrowOop>: AllStatic {
 384     typedef narrowOop type;
 385   };
 386 
 387   // This metafunction returns a canonicalized oop/narrowOop type for a passed
 388   // in oop-like types passed in from oop_* overloads where the user has sworn
 389   // that the passed in values should be oop-like (e.g. oop, oopDesc*, arrayOop,
 390   // narrowOoop, instanceOopDesc*, and random other things).
 391   // In the oop_* overloads, it must hold that if the passed in type T is not
 392   // narrowOop, then it by contract has to be one of many oop-like types implicitly
 393   // convertible to oop, and hence returns oop as the canonical oop type.
 394   // If it turns out it was not, then the implicit conversion to oop will fail
 395   // to compile, as desired.
 396   template <typename T>
 397   struct OopOrNarrowOop: AllStatic {
 398     typedef typename OopOrNarrowOopInternal<std::decay_t<T>>::type type;
 399   };
 400 
 401   inline void* field_addr(oop base, ptrdiff_t byte_offset) {
 402     return reinterpret_cast<void*>(reinterpret_cast<intptr_t>((void*)base) + byte_offset);
 403   }
 404   // Step 4: Runtime dispatch
 405   // The RuntimeDispatch class is responsible for performing a runtime dispatch of the
 406   // accessor. This is required when the access either depends on whether compressed oops
 407   // is being used, or it depends on which GC implementation was chosen (e.g. requires GC
 408   // barriers). The way it works is that a function pointer initially pointing to an
 409   // accessor resolution function gets called for each access. Upon first invocation,
 410   // it resolves which accessor to be used in future invocations and patches the
 411   // function pointer to this new accessor.
 412 
 413   template <DecoratorSet decorators, typename T, BarrierType type>
 414   struct RuntimeDispatch: AllStatic {};
 415 
 416   template <DecoratorSet decorators, typename T>
 417   struct RuntimeDispatch<decorators, T, BARRIER_STORE>: AllStatic {
 418     typedef typename AccessFunction<decorators, T, BARRIER_STORE>::type func_t;
 419     static func_t _store_func;
 420 
 421     static void store_init(void* addr, T value);
 422 
 423     static inline void store(void* addr, T value) {
 424       assert_access_thread_state();
 425       _store_func(addr, value);
 426     }
 427   };
 428 
 429   template <DecoratorSet decorators, typename T>
 430   struct RuntimeDispatch<decorators, T, BARRIER_STORE_AT>: AllStatic {
 431     typedef typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type func_t;
 432     static func_t _store_at_func;
 433 
 434     static void store_at_init(oop base, ptrdiff_t offset, T value);
 435 
 436     static inline void store_at(oop base, ptrdiff_t offset, T value) {
 437       assert_access_thread_state();
 438       _store_at_func(base, offset, value);
 439     }
 440   };
 441 
 442   template <DecoratorSet decorators, typename T>
 443   struct RuntimeDispatch<decorators, T, BARRIER_LOAD>: AllStatic {
 444     typedef typename AccessFunction<decorators, T, BARRIER_LOAD>::type func_t;
 445     static func_t _load_func;
 446 
 447     static T load_init(void* addr);
 448 
 449     static inline T load(void* addr) {
 450       assert_access_thread_state();
 451       return _load_func(addr);
 452     }
 453   };
 454 
 455   template <DecoratorSet decorators, typename T>
 456   struct RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>: AllStatic {
 457     typedef typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type func_t;
 458     static func_t _load_at_func;
 459 
 460     static T load_at_init(oop base, ptrdiff_t offset);
 461 
 462     static inline T load_at(oop base, ptrdiff_t offset) {
 463       assert_access_thread_state();
 464       return _load_at_func(base, offset);
 465     }
 466   };
 467 
 468   template <DecoratorSet decorators, typename T>
 469   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>: AllStatic {
 470     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type func_t;
 471     static func_t _atomic_cmpxchg_func;
 472 
 473     static T atomic_cmpxchg_init(void* addr, T compare_value, T new_value);
 474 
 475     static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 476       assert_access_thread_state();
 477       return _atomic_cmpxchg_func(addr, compare_value, new_value);
 478     }
 479   };
 480 
 481   template <DecoratorSet decorators, typename T>
 482   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>: AllStatic {
 483     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type func_t;
 484     static func_t _atomic_cmpxchg_at_func;
 485 
 486     static T atomic_cmpxchg_at_init(oop base, ptrdiff_t offset, T compare_value, T new_value);
 487 
 488     static inline T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
 489       assert_access_thread_state();
 490       return _atomic_cmpxchg_at_func(base, offset, compare_value, new_value);
 491     }
 492   };
 493 
 494   template <DecoratorSet decorators, typename T>
 495   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>: AllStatic {
 496     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type func_t;
 497     static func_t _atomic_xchg_func;
 498 
 499     static T atomic_xchg_init(void* addr, T new_value);
 500 
 501     static inline T atomic_xchg(void* addr, T new_value) {
 502       assert_access_thread_state();
 503       return _atomic_xchg_func(addr, new_value);
 504     }
 505   };
 506 
 507   template <DecoratorSet decorators, typename T>
 508   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
 509     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
 510     static func_t _atomic_xchg_at_func;
 511 
 512     static T atomic_xchg_at_init(oop base, ptrdiff_t offset, T new_value);
 513 
 514     static inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 515       assert_access_thread_state();
 516       return _atomic_xchg_at_func(base, offset, new_value);
 517     }
 518   };
 519 
 520   template <DecoratorSet decorators, typename T>
 521   struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
 522     typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
 523     static func_t _arraycopy_func;
 524 
 525     static void arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 526                                arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 527                                size_t length);
 528 
 529     static inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 530                                  arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 531                                  size_t length) {
 532       assert_access_thread_state();
 533       return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
 534                              dst_obj, dst_offset_in_bytes, dst_raw,
 535                              length);
 536     }
 537   };
 538 
 539   template <DecoratorSet decorators, typename T>
 540   struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
 541     typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
 542     static func_t _clone_func;
 543 
 544     static void clone_init(oop src, oop dst, size_t size);
 545 
 546     static inline void clone(oop src, oop dst, size_t size) {
 547       assert_access_thread_state();
 548       _clone_func(src, dst, size);
 549     }
 550   };
 551 
 552   template <DecoratorSet decorators, typename T>
 553   struct RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>: AllStatic {
 554     typedef typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type func_t;
 555     static func_t _value_copy_func;
 556 
 557     static void value_copy_init(void* src, void* dst, InlineKlass* md, LayoutKind lk);
 558 
 559     static inline void value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
 560       _value_copy_func(src, dst, md, lk);
 561     }
 562   };
 563 
 564   // Initialize the function pointers to point to the resolving function.
 565   template <DecoratorSet decorators, typename T>
 566   typename AccessFunction<decorators, T, BARRIER_STORE>::type
 567   RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init;
 568 
 569   template <DecoratorSet decorators, typename T>
 570   typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type
 571   RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init;
 572 
 573   template <DecoratorSet decorators, typename T>
 574   typename AccessFunction<decorators, T, BARRIER_LOAD>::type
 575   RuntimeDispatch<decorators, T, BARRIER_LOAD>::_load_func = &load_init;
 576 
 577   template <DecoratorSet decorators, typename T>
 578   typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type
 579   RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::_load_at_func = &load_at_init;
 580 
 581   template <DecoratorSet decorators, typename T>
 582   typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type
 583   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::_atomic_cmpxchg_func = &atomic_cmpxchg_init;
 584 
 585   template <DecoratorSet decorators, typename T>
 586   typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type
 587   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init;
 588 
 589   template <DecoratorSet decorators, typename T>
 590   typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type
 591   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init;
 592 
 593   template <DecoratorSet decorators, typename T>
 594   typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type
 595   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init;
 596 
 597   template <DecoratorSet decorators, typename T>
 598   typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type
 599   RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init;
 600 
 601   template <DecoratorSet decorators, typename T>
 602   typename AccessFunction<decorators, T, BARRIER_CLONE>::type
 603   RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
 604 
 605   template <DecoratorSet decorators, typename T>
 606   typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type
 607   RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>::_value_copy_func = &value_copy_init;
 608 
 609   // Step 3: Pre-runtime dispatching.
 610   // The PreRuntimeDispatch class is responsible for filtering the barrier strength
 611   // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime
 612   // dispatch point. Otherwise it goes through a runtime check if hardwiring was
 613   // not possible.
 614   struct PreRuntimeDispatch: AllStatic {
 615     template<DecoratorSet decorators>
 616     struct CanHardwireRaw: public std::integral_constant<
 617       bool,
 618       !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access
 619       !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address)
 620       HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address)
 621     {};
 622 
 623     static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP;
 624 
 625     template<DecoratorSet decorators>
 626     static bool is_hardwired_primitive() {
 627       return !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value;
 628     }
 629 
 630     template <DecoratorSet decorators, typename T>
 631     inline static typename EnableIf<
 632       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value>::type
 633     store(void* addr, T value) {
 634       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 635       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 636         Raw::oop_store(addr, value);
 637       } else {
 638         Raw::store(addr, value);
 639       }
 640     }
 641 
 642     template <DecoratorSet decorators, typename T>
 643     inline static typename EnableIf<
 644       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value>::type
 645     store(void* addr, T value) {
 646       if (UseCompressedOops) {
 647         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 648         PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 649       } else {
 650         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 651         PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 652       }
 653     }
 654 
 655     template <DecoratorSet decorators, typename T>
 656     inline static typename EnableIf<
 657       !HasDecorator<decorators, AS_RAW>::value>::type
 658     store(void* addr, T value) {
 659       if (is_hardwired_primitive<decorators>()) {
 660         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 661         PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 662       } else {
 663         RuntimeDispatch<decorators, T, BARRIER_STORE>::store(addr, value);
 664       }
 665     }
 666 
 667     template <DecoratorSet decorators, typename T>
 668     inline static typename EnableIf<
 669       HasDecorator<decorators, AS_RAW>::value>::type
 670     store_at(oop base, ptrdiff_t offset, T value) {
 671       store<decorators>(field_addr(base, offset), value);
 672     }
 673 
 674     template <DecoratorSet decorators, typename T>
 675     inline static typename EnableIf<
 676       !HasDecorator<decorators, AS_RAW>::value>::type
 677     store_at(oop base, ptrdiff_t offset, T value) {
 678       if (is_hardwired_primitive<decorators>()) {
 679         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 680         PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, value);
 681       } else {
 682         RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::store_at(base, offset, value);
 683       }
 684     }
 685 
 686     template <DecoratorSet decorators, typename T>
 687     inline static typename EnableIf<
 688       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type
 689     load(void* addr) {
 690       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 691       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 692         return Raw::template oop_load<T>(addr);
 693       } else {
 694         return Raw::template load<T>(addr);
 695       }
 696     }
 697 
 698     template <DecoratorSet decorators, typename T>
 699     inline static typename EnableIf<
 700       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type
 701     load(void* addr) {
 702       if (UseCompressedOops) {
 703         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 704         return PreRuntimeDispatch::load<expanded_decorators, T>(addr);
 705       } else {
 706         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 707         return PreRuntimeDispatch::load<expanded_decorators, T>(addr);
 708       }
 709     }
 710 
 711     template <DecoratorSet decorators, typename T>
 712     inline static typename EnableIf<
 713       !HasDecorator<decorators, AS_RAW>::value, T>::type
 714     load(void* addr) {
 715       if (is_hardwired_primitive<decorators>()) {
 716         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 717         return PreRuntimeDispatch::load<expanded_decorators, T>(addr);
 718       } else {
 719         return RuntimeDispatch<decorators, T, BARRIER_LOAD>::load(addr);
 720       }
 721     }
 722 
 723     template <DecoratorSet decorators, typename T>
 724     inline static typename EnableIf<
 725       HasDecorator<decorators, AS_RAW>::value, T>::type
 726     load_at(oop base, ptrdiff_t offset) {
 727       return load<decorators, T>(field_addr(base, offset));
 728     }
 729 
 730     template <DecoratorSet decorators, typename T>
 731     inline static typename EnableIf<
 732       !HasDecorator<decorators, AS_RAW>::value, T>::type
 733     load_at(oop base, ptrdiff_t offset) {
 734       if (is_hardwired_primitive<decorators>()) {
 735         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 736         return PreRuntimeDispatch::load_at<expanded_decorators, T>(base, offset);
 737       } else {
 738         return RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::load_at(base, offset);
 739       }
 740     }
 741 
 742     template <DecoratorSet decorators, typename T>
 743     inline static typename EnableIf<
 744       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type
 745     atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 746       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 747       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 748         return Raw::oop_atomic_cmpxchg(addr, compare_value, new_value);
 749       } else {
 750         return Raw::atomic_cmpxchg(addr, compare_value, new_value);
 751       }
 752     }
 753 
 754     template <DecoratorSet decorators, typename T>
 755     inline static typename EnableIf<
 756       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type
 757     atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 758       if (UseCompressedOops) {
 759         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 760         return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
 761       } else {
 762         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 763         return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
 764       }
 765     }
 766 
 767     template <DecoratorSet decorators, typename T>
 768     inline static typename EnableIf<
 769       !HasDecorator<decorators, AS_RAW>::value, T>::type
 770     atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 771       if (is_hardwired_primitive<decorators>()) {
 772         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 773         return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
 774       } else {
 775         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::atomic_cmpxchg(addr, compare_value, new_value);
 776       }
 777     }
 778 
 779     template <DecoratorSet decorators, typename T>
 780     inline static typename EnableIf<
 781       HasDecorator<decorators, AS_RAW>::value, T>::type
 782     atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
 783       return atomic_cmpxchg<decorators>(field_addr(base, offset), compare_value, new_value);
 784     }
 785 
 786     template <DecoratorSet decorators, typename T>
 787     inline static typename EnableIf<
 788       !HasDecorator<decorators, AS_RAW>::value, T>::type
 789     atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
 790       if (is_hardwired_primitive<decorators>()) {
 791         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 792         return PreRuntimeDispatch::atomic_cmpxchg_at<expanded_decorators>(base, offset, compare_value, new_value);
 793       } else {
 794         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::atomic_cmpxchg_at(base, offset, compare_value, new_value);
 795       }
 796     }
 797 
 798     template <DecoratorSet decorators, typename T>
 799     inline static typename EnableIf<
 800       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, T>::type
 801     atomic_xchg(void* addr, T new_value) {
 802       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 803       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 804         return Raw::oop_atomic_xchg(addr, new_value);
 805       } else {
 806         return Raw::atomic_xchg(addr, new_value);
 807       }
 808     }
 809 
 810     template <DecoratorSet decorators, typename T>
 811     inline static typename EnableIf<
 812       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, T>::type
 813     atomic_xchg(void* addr, T new_value) {
 814       if (UseCompressedOops) {
 815         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 816         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
 817       } else {
 818         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 819         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
 820       }
 821     }
 822 
 823     template <DecoratorSet decorators, typename T>
 824     inline static typename EnableIf<
 825       !HasDecorator<decorators, AS_RAW>::value, T>::type
 826     atomic_xchg(void* addr, T new_value) {
 827       if (is_hardwired_primitive<decorators>()) {
 828         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 829         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
 830       } else {
 831         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::atomic_xchg(addr, new_value);
 832       }
 833     }
 834 
 835     template <DecoratorSet decorators, typename T>
 836     inline static typename EnableIf<
 837       HasDecorator<decorators, AS_RAW>::value, T>::type
 838     atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 839       return atomic_xchg<decorators>(field_addr(base, offset), new_value);
 840     }
 841 
 842     template <DecoratorSet decorators, typename T>
 843     inline static typename EnableIf<
 844       !HasDecorator<decorators, AS_RAW>::value, T>::type
 845     atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 846       if (is_hardwired_primitive<decorators>()) {
 847         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 848         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(base, offset, new_value);
 849       } else {
 850         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(base, offset, new_value);
 851       }
 852     }
 853 
 854     template <DecoratorSet decorators, typename T>
 855     inline static typename EnableIf<
 856       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, void>::type
 857     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 858               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 859               size_t length) {
 860       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 861       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 862         Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
 863                            dst_obj, dst_offset_in_bytes, dst_raw,
 864                            length);
 865       } else {
 866         Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 867                        dst_obj, dst_offset_in_bytes, dst_raw,
 868                        length);
 869       }
 870     }
 871 
 872     template <DecoratorSet decorators, typename T>
 873     inline static typename EnableIf<
 874       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, void>::type
 875     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 876               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 877               size_t length) {
 878       if (UseCompressedOops) {
 879         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 880         PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 881                                                            dst_obj, dst_offset_in_bytes, dst_raw,
 882                                                            length);
 883       } else {
 884         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 885         PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 886                                                            dst_obj, dst_offset_in_bytes, dst_raw,
 887                                                            length);
 888       }
 889     }
 890 
 891     template <DecoratorSet decorators, typename T>
 892     inline static typename EnableIf<
 893       !HasDecorator<decorators, AS_RAW>::value, void>::type
 894     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 895               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 896               size_t length) {
 897       if (is_hardwired_primitive<decorators>()) {
 898         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 899         PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 900                                                            dst_obj, dst_offset_in_bytes, dst_raw,
 901                                                            length);
 902       } else {
 903         RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 904                                                                      dst_obj, dst_offset_in_bytes, dst_raw,
 905                                                                      length);
 906       }
 907     }
 908 
 909     template <DecoratorSet decorators>
 910     inline static typename EnableIf<
 911       HasDecorator<decorators, AS_RAW>::value>::type
 912     clone(oop src, oop dst, size_t size) {
 913       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 914       Raw::clone(src, dst, size);
 915     }
 916 
 917     template <DecoratorSet decorators>
 918     inline static typename EnableIf<
 919       !HasDecorator<decorators, AS_RAW>::value>::type
 920     clone(oop src, oop dst, size_t size) {
 921       RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
 922     }
 923 
 924     template <DecoratorSet decorators>
 925     inline static typename EnableIf<
 926       HasDecorator<decorators, AS_RAW>::value>::type
 927     value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
 928       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 929       Raw::value_copy(src, dst, md, lk);
 930     }
 931 
 932     template <DecoratorSet decorators>
 933     inline static typename EnableIf<
 934       !HasDecorator<decorators, AS_RAW>::value>::type
 935       value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
 936       const DecoratorSet expanded_decorators = decorators;
 937       RuntimeDispatch<expanded_decorators, void*, BARRIER_VALUE_COPY>::value_copy(src, dst, md, lk);
 938     }
 939   };
 940 
 941   // Step 2: Reduce types.
 942   // Enforce that for non-oop types, T and P have to be strictly the same.
 943   // P is the type of the address and T is the type of the values.
 944   // As for oop types, it is allow to send T in {narrowOop, oop} and
 945   // P in {narrowOop, oop, HeapWord*}. The following rules apply according to
 946   // the subsequent table. (columns are P, rows are T)
 947   // |           | HeapWord  |   oop   | narrowOop |
 948   // |   oop     |  rt-comp  | hw-none |  hw-comp  |
 949   // | narrowOop |     x     |    x    |  hw-none  |
 950   //
 951   // x means not allowed
 952   // rt-comp means it must be checked at runtime whether the oop is compressed.
 953   // hw-none means it is statically known the oop will not be compressed.
 954   // hw-comp means it is statically known the oop will be compressed.
 955 
 956   template <DecoratorSet decorators, typename T>
 957   inline void store_reduce_types(T* addr, T value) {
 958     PreRuntimeDispatch::store<decorators>(addr, value);
 959   }
 960 
 961   template <DecoratorSet decorators>
 962   inline void store_reduce_types(narrowOop* addr, oop value) {
 963     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
 964                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
 965     PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 966   }
 967 
 968   template <DecoratorSet decorators>
 969   inline void store_reduce_types(narrowOop* addr, narrowOop value) {
 970     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
 971                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
 972     PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 973   }
 974 
 975   template <DecoratorSet decorators>
 976   inline void store_reduce_types(HeapWord* addr, oop value) {
 977     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
 978     PreRuntimeDispatch::store<expanded_decorators>(addr, value);
 979   }
 980 
 981   template <DecoratorSet decorators, typename T>
 982   inline T atomic_cmpxchg_reduce_types(T* addr, T compare_value, T new_value) {
 983     return PreRuntimeDispatch::atomic_cmpxchg<decorators>(addr, compare_value, new_value);
 984   }
 985 
 986   template <DecoratorSet decorators>
 987   inline oop atomic_cmpxchg_reduce_types(narrowOop* addr, oop compare_value, oop new_value) {
 988     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
 989                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
 990     return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
 991   }
 992 
 993   template <DecoratorSet decorators>
 994   inline narrowOop atomic_cmpxchg_reduce_types(narrowOop* addr, narrowOop compare_value, narrowOop new_value) {
 995     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
 996                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
 997     return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
 998   }
 999 
1000   template <DecoratorSet decorators>
1001   inline oop atomic_cmpxchg_reduce_types(HeapWord* addr,
1002                                          oop compare_value,
1003                                          oop new_value) {
1004     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1005     return PreRuntimeDispatch::atomic_cmpxchg<expanded_decorators>(addr, compare_value, new_value);
1006   }
1007 
1008   template <DecoratorSet decorators, typename T>
1009   inline T atomic_xchg_reduce_types(T* addr, T new_value) {
1010     const DecoratorSet expanded_decorators = decorators;
1011     return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
1012   }
1013 
1014   template <DecoratorSet decorators>
1015   inline oop atomic_xchg_reduce_types(narrowOop* addr, oop new_value) {
1016     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1017                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1018     return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
1019   }
1020 
1021   template <DecoratorSet decorators>
1022   inline narrowOop atomic_xchg_reduce_types(narrowOop* addr, narrowOop new_value) {
1023     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1024                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1025     return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
1026   }
1027 
1028   template <DecoratorSet decorators>
1029   inline oop atomic_xchg_reduce_types(HeapWord* addr, oop new_value) {
1030     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1031     return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(addr, new_value);
1032   }
1033 
1034   template <DecoratorSet decorators, typename T>
1035   inline T load_reduce_types(T* addr) {
1036     return PreRuntimeDispatch::load<decorators, T>(addr);
1037   }
1038 
1039   template <DecoratorSet decorators, typename T>
1040   inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1041     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1042                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1043     return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1044   }
1045 
1046   template <DecoratorSet decorators, typename T>
1047   inline oop load_reduce_types(HeapWord* addr) {
1048     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1049     return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1050   }
1051 
1052   template <DecoratorSet decorators, typename T>
1053   inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1054                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1055                                      size_t length) {
1056     PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1057                                               dst_obj, dst_offset_in_bytes, dst_raw,
1058                                               length);
1059   }
1060 
1061   template <DecoratorSet decorators>
1062   inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1063                                      arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1064                                      size_t length) {
1065     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1066     PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1067                                                        dst_obj, dst_offset_in_bytes, dst_raw,
1068                                                        length);
1069   }
1070 
1071   template <DecoratorSet decorators>
1072   inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1073                                      arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1074                                      size_t length) {
1075     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1076                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1077     PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1078                                                        dst_obj, dst_offset_in_bytes, dst_raw,
1079                                                        length);
1080   }
1081 
1082   // Step 1: Set default decorators. This step remembers if a type was volatile
1083   // and then sets the MO_RELAXED decorator by default. Otherwise, a default
1084   // memory ordering is set for the access, and the implied decorator rules
1085   // are applied to select sensible defaults for decorators that have not been
1086   // explicitly set. For example, default object referent strength is set to strong.
1087   // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1088   // and references from the types). This step also perform some type verification
1089   // that the passed in types make sense.
1090 
1091   template <DecoratorSet decorators, typename T>
1092   static void verify_types(){
1093     // If this fails to compile, then you have sent in something that is
1094     // not recognized as a valid primitive type to a primitive Access function.
1095     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1096                    (std::is_pointer<T>::value || std::is_integral<T>::value) ||
1097                     std::is_floating_point<T>::value)); // not allowed primitive type
1098   }
1099 
1100   template <DecoratorSet decorators, typename P, typename T>
1101   inline void store(P* addr, T value) {
1102     verify_types<decorators, T>();
1103     using DecayedP = std::decay_t<P>;
1104     using DecayedT = std::decay_t<T>;
1105     DecayedT decayed_value = value;
1106     // If a volatile address is passed in but no memory ordering decorator,
1107     // set the memory ordering to MO_RELAXED by default.
1108     const DecoratorSet expanded_decorators = DecoratorFixup<
1109       (std::is_volatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1110       (MO_RELAXED | decorators) : decorators>::value;
1111     store_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr), decayed_value);
1112   }
1113 
1114   template <DecoratorSet decorators, typename T>
1115   inline void store_at(oop base, ptrdiff_t offset, T value) {
1116     verify_types<decorators, T>();
1117     using DecayedT = std::decay_t<T>;
1118     DecayedT decayed_value = value;
1119     const DecoratorSet expanded_decorators = DecoratorFixup<decorators |
1120                                              (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1121                                               INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1122     PreRuntimeDispatch::store_at<expanded_decorators>(base, offset, decayed_value);
1123   }
1124 
1125   template <DecoratorSet decorators, typename P, typename T>
1126   inline T load(P* addr) {
1127     verify_types<decorators, T>();
1128     using DecayedP = std::decay_t<P>;
1129     using DecayedT = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
1130                                         typename OopOrNarrowOop<T>::type,
1131                                         std::decay_t<T>>;
1132     // If a volatile address is passed in but no memory ordering decorator,
1133     // set the memory ordering to MO_RELAXED by default.
1134     const DecoratorSet expanded_decorators = DecoratorFixup<
1135       (std::is_volatile<P>::value && !HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1136       (MO_RELAXED | decorators) : decorators>::value;
1137     return load_reduce_types<expanded_decorators, DecayedT>(const_cast<DecayedP*>(addr));
1138   }
1139 
1140   template <DecoratorSet decorators, typename T>
1141   inline T load_at(oop base, ptrdiff_t offset) {
1142     verify_types<decorators, T>();
1143     using DecayedT = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
1144                                         typename OopOrNarrowOop<T>::type,
1145                                         std::decay_t<T>>;
1146     // Expand the decorators (figure out sensible defaults)
1147     // Potentially remember if we need compressed oop awareness
1148     const DecoratorSet expanded_decorators = DecoratorFixup<decorators |
1149                                              (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1150                                               INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1151     return PreRuntimeDispatch::load_at<expanded_decorators, DecayedT>(base, offset);
1152   }
1153 
1154   template <DecoratorSet decorators, typename P, typename T>
1155   inline T atomic_cmpxchg(P* addr, T compare_value, T new_value) {
1156     verify_types<decorators, T>();
1157     using DecayedP = std::decay_t<P>;
1158     using DecayedT = std::decay_t<T>;
1159     DecayedT new_decayed_value = new_value;
1160     DecayedT compare_decayed_value = compare_value;
1161     const DecoratorSet expanded_decorators = DecoratorFixup<
1162       (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1163       (MO_SEQ_CST | decorators) : decorators>::value;
1164     return atomic_cmpxchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr),
1165                                                             compare_decayed_value,
1166                                                             new_decayed_value);
1167   }
1168 
1169   template <DecoratorSet decorators, typename T>
1170   inline T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
1171     verify_types<decorators, T>();
1172     using DecayedT = std::decay_t<T>;
1173     DecayedT new_decayed_value = new_value;
1174     DecayedT compare_decayed_value = compare_value;
1175     // Determine default memory ordering
1176     const DecoratorSet expanded_decorators = DecoratorFixup<
1177       (!HasDecorator<decorators, MO_DECORATOR_MASK>::value) ?
1178       (MO_SEQ_CST | decorators) : decorators>::value;
1179     // Potentially remember that we need compressed oop awareness
1180     const DecoratorSet final_decorators = expanded_decorators |
1181                                           (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1182                                            INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE);
1183     return PreRuntimeDispatch::atomic_cmpxchg_at<final_decorators>(base, offset, compare_decayed_value,
1184                                                                    new_decayed_value);
1185   }
1186 
1187   template <DecoratorSet decorators, typename P, typename T>
1188   inline T atomic_xchg(P* addr, T new_value) {
1189     verify_types<decorators, T>();
1190     using DecayedP = std::decay_t<P>;
1191     using DecayedT = std::decay_t<T>;
1192     DecayedT new_decayed_value = new_value;
1193     // atomic_xchg is only available in SEQ_CST flavour.
1194     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1195     return atomic_xchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr),
1196                                                          new_decayed_value);
1197   }
1198 
1199   template <DecoratorSet decorators, typename T>
1200   inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
1201     verify_types<decorators, T>();
1202     using DecayedT = std::decay_t<T>;
1203     DecayedT new_decayed_value = new_value;
1204     // atomic_xchg is only available in SEQ_CST flavour.
1205     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1206                                              (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1207                                               INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1208     return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(base, offset, new_decayed_value);
1209   }
1210 
1211   template <DecoratorSet decorators, typename T>
1212   inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1213                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1214                         size_t length) {
1215     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1216                    (std::is_same<T, void>::value || std::is_integral<T>::value) ||
1217                     std::is_floating_point<T>::value)); // arraycopy allows type erased void elements
1218     using DecayedT = std::decay_t<T>;
1219     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1220     arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1221                                                 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1222                                                 length);
1223   }
1224 
1225   template <DecoratorSet decorators>
1226   inline void clone(oop src, oop dst, size_t size) {
1227     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1228     PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1229   }
1230 
1231   template <DecoratorSet decorators>
1232   inline void value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
1233     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1234     PreRuntimeDispatch::value_copy<expanded_decorators>(src, dst, md, lk);
1235   }
1236 
1237   // Infer the type that should be returned from an Access::oop_load.
1238   template <typename P, DecoratorSet decorators>
1239   class OopLoadProxy: public StackObj {
1240   private:
1241     P *const _addr;
1242   public:
1243     explicit OopLoadProxy(P* addr) : _addr(addr) {}
1244 
1245     inline operator oop() {
1246       return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr);
1247     }
1248 
1249     inline operator narrowOop() {
1250       return load<decorators | INTERNAL_VALUE_IS_OOP, P, narrowOop>(_addr);
1251     }
1252 
1253     template <typename T>
1254     inline bool operator ==(const T& other) const {
1255       return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) == other;
1256     }
1257 
1258     template <typename T>
1259     inline bool operator !=(const T& other) const {
1260       return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) != other;
1261     }
1262 
1263     inline bool operator ==(std::nullptr_t) const {
1264       return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr) == nullptr;
1265     }
1266 
1267     inline bool operator !=(std::nullptr_t) const {
1268       return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr) != nullptr;
1269     }
1270   };
1271 
1272   // Infer the type that should be returned from an Access::load_at.
1273   template <DecoratorSet decorators>
1274   class LoadAtProxy: public StackObj {
1275   private:
1276     const oop _base;
1277     const ptrdiff_t _offset;
1278   public:
1279     LoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {}
1280 
1281     template <typename T>
1282     inline operator T() const {
1283       return load_at<decorators, T>(_base, _offset);
1284     }
1285 
1286     template <typename T>
1287     inline bool operator ==(const T& other) const { return load_at<decorators, T>(_base, _offset) == other; }
1288 
1289     template <typename T>
1290     inline bool operator !=(const T& other) const { return load_at<decorators, T>(_base, _offset) != other; }
1291   };
1292 
1293   // Infer the type that should be returned from an Access::oop_load_at.
1294   template <DecoratorSet decorators>
1295   class OopLoadAtProxy: public StackObj {
1296   private:
1297     const oop _base;
1298     const ptrdiff_t _offset;
1299   public:
1300     OopLoadAtProxy(oop base, ptrdiff_t offset) : _base(base), _offset(offset) {}
1301 
1302     inline operator oop() const {
1303       return load_at<decorators | INTERNAL_VALUE_IS_OOP, oop>(_base, _offset);
1304     }
1305 
1306     inline operator narrowOop() const {
1307       return load_at<decorators | INTERNAL_VALUE_IS_OOP, narrowOop>(_base, _offset);
1308     }
1309 
1310     template <typename T>
1311     inline bool operator ==(const T& other) const {
1312       return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) == other;
1313     }
1314 
1315     template <typename T>
1316     inline bool operator !=(const T& other) const {
1317       return load_at<decorators | INTERNAL_VALUE_IS_OOP, T>(_base, _offset) != other;
1318     }
1319   };
1320 }
1321 
1322 #endif // SHARE_OOPS_ACCESSBACKEND_HPP
--- EOF ---