< prev index next >

src/hotspot/share/oops/accessBackend.hpp

Print this page

  35 #include "metaprogramming/isIntegral.hpp"
  36 #include "metaprogramming/isPointer.hpp"
  37 #include "metaprogramming/isSame.hpp"
  38 #include "metaprogramming/isVolatile.hpp"
  39 #include "oops/accessDecorators.hpp"
  40 #include "oops/oopsHierarchy.hpp"
  41 #include "runtime/globals.hpp"
  42 #include "utilities/debug.hpp"
  43 #include "utilities/globalDefinitions.hpp"
  44 
  45 
  46 // This metafunction returns either oop or narrowOop depending on whether
  47 // an access needs to use compressed oops or not.
  48 template <DecoratorSet decorators>
  49 struct HeapOopType: AllStatic {
  50   static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
  51                                          HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
  52   typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type;
  53 };
  54 








  55 namespace AccessInternal {
  56   enum BarrierType {
  57     BARRIER_STORE,
  58     BARRIER_STORE_AT,
  59     BARRIER_LOAD,
  60     BARRIER_LOAD_AT,
  61     BARRIER_ATOMIC_CMPXCHG,
  62     BARRIER_ATOMIC_CMPXCHG_AT,
  63     BARRIER_ATOMIC_XCHG,
  64     BARRIER_ATOMIC_XCHG_AT,
  65     BARRIER_ARRAYCOPY,
  66     BARRIER_CLONE

  67   };
  68 
  69   template <DecoratorSet decorators, typename T>
  70   struct MustConvertCompressedOop: public IntegralConstant<bool,
  71     HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
  72     IsSame<typename HeapOopType<decorators>::type, narrowOop>::value &&
  73     IsSame<T, oop>::value> {};
  74 
  75   // This metafunction returns an appropriate oop type if the value is oop-like
  76   // and otherwise returns the same type T.
  77   template <DecoratorSet decorators, typename T>
  78   struct EncodedType: AllStatic {
  79     typedef typename Conditional<
  80       HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
  81       typename HeapOopType<decorators>::type, T>::type type;
  82   };
  83 
  84   template <DecoratorSet decorators>
  85   inline typename HeapOopType<decorators>::type*
  86   oop_field_addr(oop base, ptrdiff_t byte_offset) {

  92   // locking to support wide atomics or not.
  93   template <typename T>
  94 #ifdef SUPPORTS_NATIVE_CX8
  95   struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
  96 #else
  97   struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
  98 #endif
  99 
 100   template <DecoratorSet decorators, typename T>
 101   struct AccessFunctionTypes {
 102     typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
 103     typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
 104     typedef T (*atomic_cmpxchg_at_func_t)(oop base, ptrdiff_t offset, T compare_value, T new_value);
 105     typedef T (*atomic_xchg_at_func_t)(oop base, ptrdiff_t offset, T new_value);
 106 
 107     typedef T (*load_func_t)(void* addr);
 108     typedef void (*store_func_t)(void* addr, T value);
 109     typedef T (*atomic_cmpxchg_func_t)(void* addr, T compare_value, T new_value);
 110     typedef T (*atomic_xchg_func_t)(void* addr, T new_value);
 111 
 112     typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 113                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 114                                      size_t length);
 115     typedef void (*clone_func_t)(oop src, oop dst, size_t size);

 116   };
 117 
 118   template <DecoratorSet decorators>
 119   struct AccessFunctionTypes<decorators, void> {
 120     typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
 121                                      arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
 122                                      size_t length);
 123   };
 124 
 125   template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
 126 
 127 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func)                   \
 128   template <DecoratorSet decorators, typename T>                    \
 129   struct AccessFunction<decorators, T, bt>: AllStatic{              \
 130     typedef typename AccessFunctionTypes<decorators, T>::func type; \
 131   }
 132   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
 133   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
 134   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
 135   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
 136   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
 137   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
 138   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
 139   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
 140   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
 141   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);

 142 #undef ACCESS_GENERATE_ACCESS_FUNCTION
 143 
 144   template <DecoratorSet decorators, typename T, BarrierType barrier_type>
 145   typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier();
 146 
 147   template <DecoratorSet decorators, typename T, BarrierType barrier_type>
 148   typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier();
 149 
 150   class AccessLocker {
 151   public:
 152     AccessLocker();
 153     ~AccessLocker();
 154   };
 155   bool wide_atomic_needs_locking();
 156 
 157   void* field_addr(oop base, ptrdiff_t offset);
 158 
 159   // Forward calls to Copy:: in the cpp file to reduce dependencies and allow
 160   // faster build times, given how frequently included access is.
 161   void arraycopy_arrayof_conjoint_oops(void* src, void* dst, size_t length);

 321   static inline void store(void* addr, T value) {
 322     store_internal<decorators>(addr, value);
 323   }
 324 
 325   template <typename T>
 326   static inline T load(void* addr) {
 327     return load_internal<decorators, T>(addr);
 328   }
 329 
 330   template <typename T>
 331   static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 332     return atomic_cmpxchg_maybe_locked<decorators>(addr, compare_value, new_value);
 333   }
 334 
 335   template <typename T>
 336   static inline T atomic_xchg(void* addr, T new_value) {
 337     return atomic_xchg_maybe_locked<decorators>(addr, new_value);
 338   }
 339 
 340   template <typename T>
 341   static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 342                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 343                         size_t length);
 344 
 345   template <typename T>
 346   static void oop_store(void* addr, T value);
 347   template <typename T>
 348   static void oop_store_at(oop base, ptrdiff_t offset, T value);
 349 
 350   template <typename T>
 351   static T oop_load(void* addr);
 352   template <typename T>
 353   static T oop_load_at(oop base, ptrdiff_t offset);
 354 
 355   template <typename T>
 356   static T oop_atomic_cmpxchg(void* addr, T compare_value, T new_value);
 357   template <typename T>
 358   static T oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value);
 359 
 360   template <typename T>
 361   static T oop_atomic_xchg(void* addr, T new_value);

 366   static void store_at(oop base, ptrdiff_t offset, T value) {
 367     store(field_addr(base, offset), value);
 368   }
 369 
 370   template <typename T>
 371   static T load_at(oop base, ptrdiff_t offset) {
 372     return load<T>(field_addr(base, offset));
 373   }
 374 
 375   template <typename T>
 376   static T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
 377     return atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
 378   }
 379 
 380   template <typename T>
 381   static T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 382     return atomic_xchg(field_addr(base, offset), new_value);
 383   }
 384 
 385   template <typename T>
 386   static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 387                             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 388                             size_t length);
 389 
 390   static void clone(oop src, oop dst, size_t size);


 391 };
 392 
 393 // Below is the implementation of the first 4 steps of the template pipeline:
 394 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
 395 //           and sets default decorators to sensible values.
 396 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
 397 //           multiple types. The P type of the address and T type of the value must
 398 //           match.
 399 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
 400 //           avoided, and in that case avoids it (calling raw accesses or
 401 //           primitive accesses in a build that does not require primitive GC barriers)
 402 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
 403 //           BarrierSet::AccessBarrier accessor that attaches GC-required barriers
 404 //           to the access.
 405 
 406 namespace AccessInternal {
 407   template <typename T>
 408   struct OopOrNarrowOopInternal: AllStatic {
 409     typedef oop type;
 410   };

 527     }
 528   };
 529 
 530   template <DecoratorSet decorators, typename T>
 531   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
 532     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
 533     static func_t _atomic_xchg_at_func;
 534 
 535     static T atomic_xchg_at_init(oop base, ptrdiff_t offset, T new_value);
 536 
 537     static inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 538       return _atomic_xchg_at_func(base, offset, new_value);
 539     }
 540   };
 541 
 542   template <DecoratorSet decorators, typename T>
 543   struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
 544     typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
 545     static func_t _arraycopy_func;
 546 
 547     static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 548                                arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 549                                size_t length);
 550 
 551     static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 552                                  arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 553                                  size_t length) {
 554       return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
 555                              dst_obj, dst_offset_in_bytes, dst_raw,
 556                              length);
 557     }
 558   };
 559 
 560   template <DecoratorSet decorators, typename T>
 561   struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
 562     typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
 563     static func_t _clone_func;
 564 
 565     static void clone_init(oop src, oop dst, size_t size);
 566 
 567     static inline void clone(oop src, oop dst, size_t size) {
 568       _clone_func(src, dst, size);
 569     }
 570   };
 571 












 572   // Initialize the function pointers to point to the resolving function.
 573   template <DecoratorSet decorators, typename T>
 574   typename AccessFunction<decorators, T, BARRIER_STORE>::type
 575   RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init;
 576 
 577   template <DecoratorSet decorators, typename T>
 578   typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type
 579   RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init;
 580 
 581   template <DecoratorSet decorators, typename T>
 582   typename AccessFunction<decorators, T, BARRIER_LOAD>::type
 583   RuntimeDispatch<decorators, T, BARRIER_LOAD>::_load_func = &load_init;
 584 
 585   template <DecoratorSet decorators, typename T>
 586   typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type
 587   RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::_load_at_func = &load_at_init;
 588 
 589   template <DecoratorSet decorators, typename T>
 590   typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type
 591   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::_atomic_cmpxchg_func = &atomic_cmpxchg_init;

 593   template <DecoratorSet decorators, typename T>
 594   typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type
 595   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init;
 596 
 597   template <DecoratorSet decorators, typename T>
 598   typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type
 599   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init;
 600 
 601   template <DecoratorSet decorators, typename T>
 602   typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type
 603   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init;
 604 
 605   template <DecoratorSet decorators, typename T>
 606   typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type
 607   RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init;
 608 
 609   template <DecoratorSet decorators, typename T>
 610   typename AccessFunction<decorators, T, BARRIER_CLONE>::type
 611   RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
 612 




 613   // Step 3: Pre-runtime dispatching.
 614   // The PreRuntimeDispatch class is responsible for filtering the barrier strength
 615   // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime
 616   // dispatch point. Otherwise it goes through a runtime check if hardwiring was
 617   // not possible.
 618   struct PreRuntimeDispatch: AllStatic {
 619     template<DecoratorSet decorators>
 620     struct CanHardwireRaw: public IntegralConstant<
 621       bool,
 622       !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access
 623       !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address)
 624       HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address)
 625     {};
 626 
 627     static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP;
 628 
 629     template<DecoratorSet decorators>
 630     static bool is_hardwired_primitive() {
 631       return !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value;
 632     }

 840     inline static typename EnableIf<
 841       HasDecorator<decorators, AS_RAW>::value, T>::type
 842     atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 843       return atomic_xchg<decorators>(field_addr(base, offset), new_value);
 844     }
 845 
 846     template <DecoratorSet decorators, typename T>
 847     inline static typename EnableIf<
 848       !HasDecorator<decorators, AS_RAW>::value, T>::type
 849     atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 850       if (is_hardwired_primitive<decorators>()) {
 851         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 852         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(base, offset, new_value);
 853       } else {
 854         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(base, offset, new_value);
 855       }
 856     }
 857 
 858     template <DecoratorSet decorators, typename T>
 859     inline static typename EnableIf<
 860       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
 861     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 862               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 863               size_t length) {
 864       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 865       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 866         return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
 867                                   dst_obj, dst_offset_in_bytes, dst_raw,
 868                                   length);
 869       } else {
 870         return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 871                               dst_obj, dst_offset_in_bytes, dst_raw,
 872                               length);
 873       }
 874     }
 875 
 876     template <DecoratorSet decorators, typename T>
 877     inline static typename EnableIf<
 878       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
 879     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 880               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 881               size_t length) {
 882       if (UseCompressedOops) {
 883         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 884         return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 885                                                                   dst_obj, dst_offset_in_bytes, dst_raw,
 886                                                                   length);
 887       } else {
 888         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 889         return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 890                                                                   dst_obj, dst_offset_in_bytes, dst_raw,
 891                                                                   length);
 892       }
 893     }
 894 
 895     template <DecoratorSet decorators, typename T>
 896     inline static typename EnableIf<
 897       !HasDecorator<decorators, AS_RAW>::value, bool>::type
 898     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 899               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 900               size_t length) {
 901       if (is_hardwired_primitive<decorators>()) {
 902         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 903         return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 904                                                                   dst_obj, dst_offset_in_bytes, dst_raw,
 905                                                                   length);
 906       } else {
 907         return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 908                                                                             dst_obj, dst_offset_in_bytes, dst_raw,
 909                                                                             length);
 910       }
 911     }
 912 
 913     template <DecoratorSet decorators>
 914     inline static typename EnableIf<
 915       HasDecorator<decorators, AS_RAW>::value>::type
 916     clone(oop src, oop dst, size_t size) {
 917       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 918       Raw::clone(src, dst, size);
 919     }
 920 
 921     template <DecoratorSet decorators>
 922     inline static typename EnableIf<
 923       !HasDecorator<decorators, AS_RAW>::value>::type
 924     clone(oop src, oop dst, size_t size) {
 925       RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
 926     }
















 927   };
 928 
 929   // Step 2: Reduce types.
 930   // Enforce that for non-oop types, T and P have to be strictly the same.
 931   // P is the type of the address and T is the type of the values.
 932   // As for oop types, it is allow to send T in {narrowOop, oop} and
 933   // P in {narrowOop, oop, HeapWord*}. The following rules apply according to
 934   // the subsequent table. (columns are P, rows are T)
 935   // |           | HeapWord  |   oop   | narrowOop |
 936   // |   oop     |  rt-comp  | hw-none |  hw-comp  |
 937   // | narrowOop |     x     |    x    |  hw-none  |
 938   //
 939   // x means not allowed
 940   // rt-comp means it must be checked at runtime whether the oop is compressed.
 941   // hw-none means it is statically known the oop will not be compressed.
 942   // hw-comp means it is statically known the oop will be compressed.
 943 
 944   template <DecoratorSet decorators, typename T>
 945   inline void store_reduce_types(T* addr, T value) {
 946     PreRuntimeDispatch::store<decorators>(addr, value);

1021 
1022   template <DecoratorSet decorators, typename T>
1023   inline T load_reduce_types(T* addr) {
1024     return PreRuntimeDispatch::load<decorators, T>(addr);
1025   }
1026 
1027   template <DecoratorSet decorators, typename T>
1028   inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1029     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1030                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1031     return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1032   }
1033 
1034   template <DecoratorSet decorators, typename T>
1035   inline oop load_reduce_types(HeapWord* addr) {
1036     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1037     return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1038   }
1039 
1040   template <DecoratorSet decorators, typename T>
1041   inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1042                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1043                                      size_t length) {
1044     return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1045                                                      dst_obj, dst_offset_in_bytes, dst_raw,
1046                                                      length);
1047   }
1048 
1049   template <DecoratorSet decorators>
1050   inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1051                                      arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1052                                      size_t length) {
1053     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1054     return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1055                                                               dst_obj, dst_offset_in_bytes, dst_raw,
1056                                                               length);
1057   }
1058 
1059   template <DecoratorSet decorators>
1060   inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1061                                      arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1062                                      size_t length) {
1063     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1064                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1065     return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1066                                                               dst_obj, dst_offset_in_bytes, dst_raw,
1067                                                               length);
1068   }
1069 
1070   // Step 1: Set default decorators. This step remembers if a type was volatile
1071   // and then sets the MO_RELAXED decorator by default. Otherwise, a default
1072   // memory ordering is set for the access, and the implied decorator rules
1073   // are applied to select sensible defaults for decorators that have not been
1074   // explicitly set. For example, default object referent strength is set to strong.
1075   // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1076   // and references from the types). This step also perform some type verification
1077   // that the passed in types make sense.
1078 
1079   template <DecoratorSet decorators, typename T>
1080   static void verify_types(){
1081     // If this fails to compile, then you have sent in something that is
1082     // not recognized as a valid primitive type to a primitive Access function.
1083     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1084                    (IsPointer<T>::value || IsIntegral<T>::value) ||
1085                     IsFloatingPoint<T>::value)); // not allowed primitive type
1086   }
1087 

1180     DecayedT new_decayed_value = new_value;
1181     // atomic_xchg is only available in SEQ_CST flavour.
1182     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1183     return atomic_xchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr),
1184                                                          new_decayed_value);
1185   }
1186 
1187   template <DecoratorSet decorators, typename T>
1188   inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
1189     verify_types<decorators, T>();
1190     typedef typename Decay<T>::type DecayedT;
1191     DecayedT new_decayed_value = new_value;
1192     // atomic_xchg is only available in SEQ_CST flavour.
1193     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1194                                              (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1195                                               INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1196     return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(base, offset, new_decayed_value);
1197   }
1198 
1199   template <DecoratorSet decorators, typename T>
1200   inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1201                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1202                         size_t length) {
1203     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1204                    (IsSame<T, void>::value || IsIntegral<T>::value) ||
1205                     IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1206     typedef typename Decay<T>::type DecayedT;
1207     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1208     return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1209                                                        dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1210                                                        length);
1211   }
1212 
1213   template <DecoratorSet decorators>
1214   inline void clone(oop src, oop dst, size_t size) {
1215     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1216     PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1217   }
1218 






1219   // Infer the type that should be returned from an Access::oop_load.
1220   template <typename P, DecoratorSet decorators>
1221   class OopLoadProxy: public StackObj {
1222   private:
1223     P *const _addr;
1224   public:
1225     OopLoadProxy(P* addr) : _addr(addr) {}
1226 
1227     inline operator oop() {
1228       return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr);
1229     }
1230 
1231     inline operator narrowOop() {
1232       return load<decorators | INTERNAL_VALUE_IS_OOP, P, narrowOop>(_addr);
1233     }
1234 
1235     template <typename T>
1236     inline bool operator ==(const T& other) const {
1237       return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) == other;
1238     }

  35 #include "metaprogramming/isIntegral.hpp"
  36 #include "metaprogramming/isPointer.hpp"
  37 #include "metaprogramming/isSame.hpp"
  38 #include "metaprogramming/isVolatile.hpp"
  39 #include "oops/accessDecorators.hpp"
  40 #include "oops/oopsHierarchy.hpp"
  41 #include "runtime/globals.hpp"
  42 #include "utilities/debug.hpp"
  43 #include "utilities/globalDefinitions.hpp"
  44 
  45 
  46 // This metafunction returns either oop or narrowOop depending on whether
  47 // an access needs to use compressed oops or not.
  48 template <DecoratorSet decorators>
  49 struct HeapOopType: AllStatic {
  50   static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
  51                                          HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
  52   typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type;
  53 };
  54 
  55 // This meta-function returns either oop or narrowOop depending on whether
  56 // a back-end needs to consider compressed oops types or not.
  57 template <DecoratorSet decorators>
  58 struct ValueOopType: AllStatic {
  59   static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
  60   typedef typename Conditional<needs_oop_compress, narrowOop, oop>::type type;
  61 };
  62 
  63 namespace AccessInternal {
  64   enum BarrierType {
  65     BARRIER_STORE,
  66     BARRIER_STORE_AT,
  67     BARRIER_LOAD,
  68     BARRIER_LOAD_AT,
  69     BARRIER_ATOMIC_CMPXCHG,
  70     BARRIER_ATOMIC_CMPXCHG_AT,
  71     BARRIER_ATOMIC_XCHG,
  72     BARRIER_ATOMIC_XCHG_AT,
  73     BARRIER_ARRAYCOPY,
  74     BARRIER_CLONE,
  75     BARRIER_VALUE_COPY
  76   };
  77 
  78   template <DecoratorSet decorators, typename T>
  79   struct MustConvertCompressedOop: public IntegralConstant<bool,
  80     HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
  81     IsSame<typename HeapOopType<decorators>::type, narrowOop>::value &&
  82     IsSame<T, oop>::value> {};
  83 
  84   // This metafunction returns an appropriate oop type if the value is oop-like
  85   // and otherwise returns the same type T.
  86   template <DecoratorSet decorators, typename T>
  87   struct EncodedType: AllStatic {
  88     typedef typename Conditional<
  89       HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
  90       typename HeapOopType<decorators>::type, T>::type type;
  91   };
  92 
  93   template <DecoratorSet decorators>
  94   inline typename HeapOopType<decorators>::type*
  95   oop_field_addr(oop base, ptrdiff_t byte_offset) {

 101   // locking to support wide atomics or not.
 102   template <typename T>
 103 #ifdef SUPPORTS_NATIVE_CX8
 104   struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
 105 #else
 106   struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
 107 #endif
 108 
 109   template <DecoratorSet decorators, typename T>
 110   struct AccessFunctionTypes {
 111     typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
 112     typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
 113     typedef T (*atomic_cmpxchg_at_func_t)(oop base, ptrdiff_t offset, T compare_value, T new_value);
 114     typedef T (*atomic_xchg_at_func_t)(oop base, ptrdiff_t offset, T new_value);
 115 
 116     typedef T (*load_func_t)(void* addr);
 117     typedef void (*store_func_t)(void* addr, T value);
 118     typedef T (*atomic_cmpxchg_func_t)(void* addr, T compare_value, T new_value);
 119     typedef T (*atomic_xchg_func_t)(void* addr, T new_value);
 120 
 121     typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 122                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 123                                      size_t length);
 124     typedef void (*clone_func_t)(oop src, oop dst, size_t size);
 125     typedef void (*value_copy_func_t)(void* src, void* dst, InlineKlass* md);
 126   };
 127 
 128   template <DecoratorSet decorators>
 129   struct AccessFunctionTypes<decorators, void> {
 130     typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
 131                                      arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
 132                                      size_t length);
 133   };
 134 
 135   template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
 136 
 137 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func)                   \
 138   template <DecoratorSet decorators, typename T>                    \
 139   struct AccessFunction<decorators, T, bt>: AllStatic{              \
 140     typedef typename AccessFunctionTypes<decorators, T>::func type; \
 141   }
 142   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
 143   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
 144   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
 145   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
 146   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
 147   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
 148   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
 149   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
 150   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
 151   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
 152   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_VALUE_COPY, value_copy_func_t);
 153 #undef ACCESS_GENERATE_ACCESS_FUNCTION
 154 
 155   template <DecoratorSet decorators, typename T, BarrierType barrier_type>
 156   typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier();
 157 
 158   template <DecoratorSet decorators, typename T, BarrierType barrier_type>
 159   typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier();
 160 
 161   class AccessLocker {
 162   public:
 163     AccessLocker();
 164     ~AccessLocker();
 165   };
 166   bool wide_atomic_needs_locking();
 167 
 168   void* field_addr(oop base, ptrdiff_t offset);
 169 
 170   // Forward calls to Copy:: in the cpp file to reduce dependencies and allow
 171   // faster build times, given how frequently included access is.
 172   void arraycopy_arrayof_conjoint_oops(void* src, void* dst, size_t length);

 332   static inline void store(void* addr, T value) {
 333     store_internal<decorators>(addr, value);
 334   }
 335 
 336   template <typename T>
 337   static inline T load(void* addr) {
 338     return load_internal<decorators, T>(addr);
 339   }
 340 
 341   template <typename T>
 342   static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) {
 343     return atomic_cmpxchg_maybe_locked<decorators>(addr, compare_value, new_value);
 344   }
 345 
 346   template <typename T>
 347   static inline T atomic_xchg(void* addr, T new_value) {
 348     return atomic_xchg_maybe_locked<decorators>(addr, new_value);
 349   }
 350 
 351   template <typename T>
 352   static void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 353                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 354                         size_t length);
 355 
 356   template <typename T>
 357   static void oop_store(void* addr, T value);
 358   template <typename T>
 359   static void oop_store_at(oop base, ptrdiff_t offset, T value);
 360 
 361   template <typename T>
 362   static T oop_load(void* addr);
 363   template <typename T>
 364   static T oop_load_at(oop base, ptrdiff_t offset);
 365 
 366   template <typename T>
 367   static T oop_atomic_cmpxchg(void* addr, T compare_value, T new_value);
 368   template <typename T>
 369   static T oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value);
 370 
 371   template <typename T>
 372   static T oop_atomic_xchg(void* addr, T new_value);

 377   static void store_at(oop base, ptrdiff_t offset, T value) {
 378     store(field_addr(base, offset), value);
 379   }
 380 
 381   template <typename T>
 382   static T load_at(oop base, ptrdiff_t offset) {
 383     return load<T>(field_addr(base, offset));
 384   }
 385 
 386   template <typename T>
 387   static T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
 388     return atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
 389   }
 390 
 391   template <typename T>
 392   static T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 393     return atomic_xchg(field_addr(base, offset), new_value);
 394   }
 395 
 396   template <typename T>
 397   static void oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 398                             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 399                             size_t length);
 400 
 401   static void clone(oop src, oop dst, size_t size);
 402   static void value_copy(void* src, void* dst, InlineKlass* md);
 403 
 404 };
 405 
 406 // Below is the implementation of the first 4 steps of the template pipeline:
 407 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
 408 //           and sets default decorators to sensible values.
 409 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
 410 //           multiple types. The P type of the address and T type of the value must
 411 //           match.
 412 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
 413 //           avoided, and in that case avoids it (calling raw accesses or
 414 //           primitive accesses in a build that does not require primitive GC barriers)
 415 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
 416 //           BarrierSet::AccessBarrier accessor that attaches GC-required barriers
 417 //           to the access.
 418 
 419 namespace AccessInternal {
 420   template <typename T>
 421   struct OopOrNarrowOopInternal: AllStatic {
 422     typedef oop type;
 423   };

 540     }
 541   };
 542 
 543   template <DecoratorSet decorators, typename T>
 544   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
 545     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
 546     static func_t _atomic_xchg_at_func;
 547 
 548     static T atomic_xchg_at_init(oop base, ptrdiff_t offset, T new_value);
 549 
 550     static inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 551       return _atomic_xchg_at_func(base, offset, new_value);
 552     }
 553   };
 554 
 555   template <DecoratorSet decorators, typename T>
 556   struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
 557     typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
 558     static func_t _arraycopy_func;
 559 
 560     static void arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 561                                arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 562                                size_t length);
 563 
 564     static inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 565                                  arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 566                                  size_t length) {
 567       return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
 568                              dst_obj, dst_offset_in_bytes, dst_raw,
 569                              length);
 570     }
 571   };
 572 
 573   template <DecoratorSet decorators, typename T>
 574   struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
 575     typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
 576     static func_t _clone_func;
 577 
 578     static void clone_init(oop src, oop dst, size_t size);
 579 
 580     static inline void clone(oop src, oop dst, size_t size) {
 581       _clone_func(src, dst, size);
 582     }
 583   };
 584 
 585   template <DecoratorSet decorators, typename T>
 586   struct RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>: AllStatic {
 587     typedef typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type func_t;
 588     static func_t _value_copy_func;
 589 
 590     static void value_copy_init(void* src, void* dst, InlineKlass* md);
 591 
 592     static inline void value_copy(void* src, void* dst, InlineKlass* md) {
 593       _value_copy_func(src, dst, md);
 594     }
 595   };
 596 
 597   // Initialize the function pointers to point to the resolving function.
 598   template <DecoratorSet decorators, typename T>
 599   typename AccessFunction<decorators, T, BARRIER_STORE>::type
 600   RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init;
 601 
 602   template <DecoratorSet decorators, typename T>
 603   typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type
 604   RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init;
 605 
 606   template <DecoratorSet decorators, typename T>
 607   typename AccessFunction<decorators, T, BARRIER_LOAD>::type
 608   RuntimeDispatch<decorators, T, BARRIER_LOAD>::_load_func = &load_init;
 609 
 610   template <DecoratorSet decorators, typename T>
 611   typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type
 612   RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::_load_at_func = &load_at_init;
 613 
 614   template <DecoratorSet decorators, typename T>
 615   typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type
 616   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::_atomic_cmpxchg_func = &atomic_cmpxchg_init;

 618   template <DecoratorSet decorators, typename T>
 619   typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type
 620   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init;
 621 
 622   template <DecoratorSet decorators, typename T>
 623   typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type
 624   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init;
 625 
 626   template <DecoratorSet decorators, typename T>
 627   typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type
 628   RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init;
 629 
 630   template <DecoratorSet decorators, typename T>
 631   typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type
 632   RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init;
 633 
 634   template <DecoratorSet decorators, typename T>
 635   typename AccessFunction<decorators, T, BARRIER_CLONE>::type
 636   RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
 637 
 638   template <DecoratorSet decorators, typename T>
 639   typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type
 640   RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>::_value_copy_func = &value_copy_init;
 641 
 642   // Step 3: Pre-runtime dispatching.
 643   // The PreRuntimeDispatch class is responsible for filtering the barrier strength
 644   // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime
 645   // dispatch point. Otherwise it goes through a runtime check if hardwiring was
 646   // not possible.
 647   struct PreRuntimeDispatch: AllStatic {
 648     template<DecoratorSet decorators>
 649     struct CanHardwireRaw: public IntegralConstant<
 650       bool,
 651       !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access
 652       !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address)
 653       HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address)
 654     {};
 655 
 656     static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP;
 657 
 658     template<DecoratorSet decorators>
 659     static bool is_hardwired_primitive() {
 660       return !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value;
 661     }

 869     inline static typename EnableIf<
 870       HasDecorator<decorators, AS_RAW>::value, T>::type
 871     atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 872       return atomic_xchg<decorators>(field_addr(base, offset), new_value);
 873     }
 874 
 875     template <DecoratorSet decorators, typename T>
 876     inline static typename EnableIf<
 877       !HasDecorator<decorators, AS_RAW>::value, T>::type
 878     atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
 879       if (is_hardwired_primitive<decorators>()) {
 880         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 881         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(base, offset, new_value);
 882       } else {
 883         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(base, offset, new_value);
 884       }
 885     }
 886 
 887     template <DecoratorSet decorators, typename T>
 888     inline static typename EnableIf<
 889       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, void>::type
 890     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 891               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 892               size_t length) {
 893       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 894       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 895         Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
 896                            dst_obj, dst_offset_in_bytes, dst_raw,
 897                            length);
 898       } else {
 899         Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 900                        dst_obj, dst_offset_in_bytes, dst_raw,
 901                        length);
 902       }
 903     }
 904 
 905     template <DecoratorSet decorators, typename T>
 906     inline static typename EnableIf<
 907       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, void>::type
 908     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 909               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 910               size_t length) {
 911       if (UseCompressedOops) {
 912         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 913         PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 914                                                            dst_obj, dst_offset_in_bytes, dst_raw,
 915                                                            length);
 916       } else {
 917         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 918         PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 919                                                            dst_obj, dst_offset_in_bytes, dst_raw,
 920                                                            length);
 921       }
 922     }
 923 
 924     template <DecoratorSet decorators, typename T>
 925     inline static typename EnableIf<
 926       !HasDecorator<decorators, AS_RAW>::value, void>::type
 927     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 928               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 929               size_t length) {
 930       if (is_hardwired_primitive<decorators>()) {
 931         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 932         PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 933                                                            dst_obj, dst_offset_in_bytes, dst_raw,
 934                                                            length);
 935       } else {
 936         RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 937                                                                      dst_obj, dst_offset_in_bytes, dst_raw,
 938                                                                      length);
 939       }
 940     }
 941 
 942     template <DecoratorSet decorators>
 943     inline static typename EnableIf<
 944       HasDecorator<decorators, AS_RAW>::value>::type
 945     clone(oop src, oop dst, size_t size) {
 946       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 947       Raw::clone(src, dst, size);
 948     }
 949 
 950     template <DecoratorSet decorators>
 951     inline static typename EnableIf<
 952       !HasDecorator<decorators, AS_RAW>::value>::type
 953     clone(oop src, oop dst, size_t size) {
 954       RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
 955     }
 956 
 957     template <DecoratorSet decorators>
 958     inline static typename EnableIf<
 959       HasDecorator<decorators, AS_RAW>::value>::type
 960     value_copy(void* src, void* dst, InlineKlass* md) {
 961       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 962       Raw::value_copy(src, dst, md);
 963     }
 964 
 965     template <DecoratorSet decorators>
 966     inline static typename EnableIf<
 967       !HasDecorator<decorators, AS_RAW>::value>::type
 968       value_copy(void* src, void* dst, InlineKlass* md) {
 969       const DecoratorSet expanded_decorators = decorators;
 970       RuntimeDispatch<expanded_decorators, void*, BARRIER_VALUE_COPY>::value_copy(src, dst, md);
 971     }
 972   };
 973 
 974   // Step 2: Reduce types.
 975   // Enforce that for non-oop types, T and P have to be strictly the same.
 976   // P is the type of the address and T is the type of the values.
 977   // As for oop types, it is allow to send T in {narrowOop, oop} and
 978   // P in {narrowOop, oop, HeapWord*}. The following rules apply according to
 979   // the subsequent table. (columns are P, rows are T)
 980   // |           | HeapWord  |   oop   | narrowOop |
 981   // |   oop     |  rt-comp  | hw-none |  hw-comp  |
 982   // | narrowOop |     x     |    x    |  hw-none  |
 983   //
 984   // x means not allowed
 985   // rt-comp means it must be checked at runtime whether the oop is compressed.
 986   // hw-none means it is statically known the oop will not be compressed.
 987   // hw-comp means it is statically known the oop will be compressed.
 988 
 989   template <DecoratorSet decorators, typename T>
 990   inline void store_reduce_types(T* addr, T value) {
 991     PreRuntimeDispatch::store<decorators>(addr, value);

1066 
1067   template <DecoratorSet decorators, typename T>
1068   inline T load_reduce_types(T* addr) {
1069     return PreRuntimeDispatch::load<decorators, T>(addr);
1070   }
1071 
1072   template <DecoratorSet decorators, typename T>
1073   inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1074     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1075                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1076     return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1077   }
1078 
1079   template <DecoratorSet decorators, typename T>
1080   inline oop load_reduce_types(HeapWord* addr) {
1081     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1082     return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1083   }
1084 
1085   template <DecoratorSet decorators, typename T>
1086   inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1087                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1088                                      size_t length) {
1089     PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1090                                               dst_obj, dst_offset_in_bytes, dst_raw,
1091                                               length);
1092   }
1093 
1094   template <DecoratorSet decorators>
1095   inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1096                                      arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1097                                      size_t length) {
1098     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1099     PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1100                                                        dst_obj, dst_offset_in_bytes, dst_raw,
1101                                                        length);
1102   }
1103 
1104   template <DecoratorSet decorators>
1105   inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1106                                      arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1107                                      size_t length) {
1108     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1109                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1110     PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1111                                                        dst_obj, dst_offset_in_bytes, dst_raw,
1112                                                        length);
1113   }
1114 
1115   // Step 1: Set default decorators. This step remembers if a type was volatile
1116   // and then sets the MO_RELAXED decorator by default. Otherwise, a default
1117   // memory ordering is set for the access, and the implied decorator rules
1118   // are applied to select sensible defaults for decorators that have not been
1119   // explicitly set. For example, default object referent strength is set to strong.
1120   // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1121   // and references from the types). This step also perform some type verification
1122   // that the passed in types make sense.
1123 
1124   template <DecoratorSet decorators, typename T>
1125   static void verify_types(){
1126     // If this fails to compile, then you have sent in something that is
1127     // not recognized as a valid primitive type to a primitive Access function.
1128     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1129                    (IsPointer<T>::value || IsIntegral<T>::value) ||
1130                     IsFloatingPoint<T>::value)); // not allowed primitive type
1131   }
1132 

1225     DecayedT new_decayed_value = new_value;
1226     // atomic_xchg is only available in SEQ_CST flavour.
1227     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1228     return atomic_xchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr),
1229                                                          new_decayed_value);
1230   }
1231 
1232   template <DecoratorSet decorators, typename T>
1233   inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
1234     verify_types<decorators, T>();
1235     typedef typename Decay<T>::type DecayedT;
1236     DecayedT new_decayed_value = new_value;
1237     // atomic_xchg is only available in SEQ_CST flavour.
1238     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1239                                              (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1240                                               INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1241     return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(base, offset, new_decayed_value);
1242   }
1243 
1244   template <DecoratorSet decorators, typename T>
1245   inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1246                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1247                         size_t length) {
1248     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1249                    (IsSame<T, void>::value || IsIntegral<T>::value) ||
1250                     IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1251     typedef typename Decay<T>::type DecayedT;
1252     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1253     arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1254                                                 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1255                                                 length);
1256   }
1257 
1258   template <DecoratorSet decorators>
1259   inline void clone(oop src, oop dst, size_t size) {
1260     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1261     PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1262   }
1263 
1264   template <DecoratorSet decorators>
1265   inline void value_copy(void* src, void* dst, InlineKlass* md) {
1266     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1267     PreRuntimeDispatch::value_copy<expanded_decorators>(src, dst, md);
1268   }
1269 
1270   // Infer the type that should be returned from an Access::oop_load.
1271   template <typename P, DecoratorSet decorators>
1272   class OopLoadProxy: public StackObj {
1273   private:
1274     P *const _addr;
1275   public:
1276     OopLoadProxy(P* addr) : _addr(addr) {}
1277 
1278     inline operator oop() {
1279       return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr);
1280     }
1281 
1282     inline operator narrowOop() {
1283       return load<decorators | INTERNAL_VALUE_IS_OOP, P, narrowOop>(_addr);
1284     }
1285 
1286     template <typename T>
1287     inline bool operator ==(const T& other) const {
1288       return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) == other;
1289     }
< prev index next >