< prev index next >

src/hotspot/share/oops/accessBackend.hpp

Print this page




  93   // locking to support wide atomics or not.
  94   template <typename T>
  95 #ifdef SUPPORTS_NATIVE_CX8
  96   struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
  97 #else
  98   struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
  99 #endif
 100 
 101   template <DecoratorSet decorators, typename T>
 102   struct AccessFunctionTypes {
 103     typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
 104     typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
 105     typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
 106     typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
 107 
 108     typedef T (*load_func_t)(void* addr);
 109     typedef void (*store_func_t)(void* addr, T value);
 110     typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
 111     typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
 112 
 113     typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 114                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 115                                      size_t length);
 116     typedef void (*clone_func_t)(oop src, oop dst, size_t size);
 117     typedef oop (*resolve_func_t)(oop obj);
 118     typedef bool (*equals_func_t)(oop o1, oop o2);
 119   };
 120 
 121   template <DecoratorSet decorators>
 122   struct AccessFunctionTypes<decorators, void> {
 123     typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
 124                                      arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
 125                                      size_t length);
 126   };
 127 
 128   template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
 129 
 130 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func)                   \
 131   template <DecoratorSet decorators, typename T>                    \
 132   struct AccessFunction<decorators, T, bt>: AllStatic{              \
 133     typedef typename AccessFunctionTypes<decorators, T>::func type; \
 134   }
 135   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
 136   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
 137   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
 138   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
 139   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
 140   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
 141   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
 142   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
 143   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);


 340   static inline void store(void* addr, T value) {
 341     store_internal<decorators>(addr, value);
 342   }
 343 
 344   template <typename T>
 345   static inline T load(void* addr) {
 346     return load_internal<decorators, T>(addr);
 347   }
 348 
 349   template <typename T>
 350   static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
 351     return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
 352   }
 353 
 354   template <typename T>
 355   static inline T atomic_xchg(T new_value, void* addr) {
 356     return atomic_xchg_maybe_locked<decorators>(new_value, addr);
 357   }
 358 
 359   template <typename T>
 360   static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 361                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 362                         size_t length);
 363 
 364   template <typename T>
 365   static void oop_store(void* addr, T value);
 366   template <typename T>
 367   static void oop_store_at(oop base, ptrdiff_t offset, T value);
 368 
 369   template <typename T>
 370   static T oop_load(void* addr);
 371   template <typename T>
 372   static T oop_load_at(oop base, ptrdiff_t offset);
 373 
 374   template <typename T>
 375   static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
 376   template <typename T>
 377   static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
 378 
 379   template <typename T>
 380   static T oop_atomic_xchg(T new_value, void* addr);


 385   static void store_at(oop base, ptrdiff_t offset, T value) {
 386     store(field_addr(base, offset), value);
 387   }
 388 
 389   template <typename T>
 390   static T load_at(oop base, ptrdiff_t offset) {
 391     return load<T>(field_addr(base, offset));
 392   }
 393 
 394   template <typename T>
 395   static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
 396     return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
 397   }
 398 
 399   template <typename T>
 400   static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
 401     return atomic_xchg(new_value, field_addr(base, offset));
 402   }
 403 
 404   template <typename T>
 405   static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 406                             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 407                             size_t length);
 408 
 409   static void clone(oop src, oop dst, size_t size);
 410 
 411   static oop resolve(oop obj) { return obj; }
 412 
 413   static bool equals(oop o1, oop o2) { return (void*)o1 == (void*)o2; }
 414 };
 415 
 416 // Below is the implementation of the first 4 steps of the template pipeline:
 417 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
 418 //           and sets default decorators to sensible values.
 419 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
 420 //           multiple types. The P type of the address and T type of the value must
 421 //           match.
 422 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
 423 //           avoided, and in that case avoids it (calling raw accesses or
 424 //           primitive accesses in a build that does not require primitive GC barriers)
 425 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding


 550     }
 551   };
 552 
 553   template <DecoratorSet decorators, typename T>
 554   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
 555     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
 556     static func_t _atomic_xchg_at_func;
 557 
 558     static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
 559 
 560     static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
 561       return _atomic_xchg_at_func(new_value, base, offset);
 562     }
 563   };
 564 
 565   template <DecoratorSet decorators, typename T>
 566   struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
 567     typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
 568     static func_t _arraycopy_func;
 569 
 570     static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 571                                arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 572                                size_t length);
 573 
 574     static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 575                                  arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 576                                  size_t length) {
 577       return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
 578                              dst_obj, dst_offset_in_bytes, dst_raw,
 579                              length);
 580     }
 581   };
 582 
 583   template <DecoratorSet decorators, typename T>
 584   struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
 585     typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
 586     static func_t _clone_func;
 587 
 588     static void clone_init(oop src, oop dst, size_t size);
 589 
 590     static inline void clone(oop src, oop dst, size_t size) {
 591       _clone_func(src, dst, size);
 592     }
 593   };
 594 


 896     inline static typename EnableIf<
 897       HasDecorator<decorators, AS_RAW>::value, T>::type
 898     atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
 899       return atomic_xchg<decorators>(new_value, field_addr(base, offset));
 900     }
 901 
 902     template <DecoratorSet decorators, typename T>
 903     inline static typename EnableIf<
 904       !HasDecorator<decorators, AS_RAW>::value, T>::type
 905     atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
 906       if (is_hardwired_primitive<decorators>()) {
 907         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 908         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
 909       } else {
 910         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
 911       }
 912     }
 913 
 914     template <DecoratorSet decorators, typename T>
 915     inline static typename EnableIf<
 916       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
 917     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 918               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 919               size_t length) {
 920       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 921       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 922         return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
 923                                   dst_obj, dst_offset_in_bytes, dst_raw,
 924                                   length);
 925       } else {
 926         return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 927                               dst_obj, dst_offset_in_bytes, dst_raw,
 928                               length);
 929       }
 930     }
 931 
 932     template <DecoratorSet decorators, typename T>
 933     inline static typename EnableIf<
 934       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
 935     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 936               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 937               size_t length) {
 938       if (UseCompressedOops) {
 939         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 940         return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 941                                                                   dst_obj, dst_offset_in_bytes, dst_raw,
 942                                                                   length);
 943       } else {
 944         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 945         return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 946                                                                   dst_obj, dst_offset_in_bytes, dst_raw,
 947                                                                   length);
 948       }
 949     }
 950 
 951     template <DecoratorSet decorators, typename T>
 952     inline static typename EnableIf<
 953       !HasDecorator<decorators, AS_RAW>::value, bool>::type
 954     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 955               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 956               size_t length) {
 957       if (is_hardwired_primitive<decorators>()) {
 958         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 959         return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 960                                                                   dst_obj, dst_offset_in_bytes, dst_raw,
 961                                                                   length);
 962       } else {
 963         return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 964                                                                             dst_obj, dst_offset_in_bytes, dst_raw,
 965                                                                             length);
 966       }
 967     }
 968 
 969     template <DecoratorSet decorators>
 970     inline static typename EnableIf<
 971       HasDecorator<decorators, AS_RAW>::value>::type
 972     clone(oop src, oop dst, size_t size) {
 973       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 974       Raw::clone(src, dst, size);
 975     }
 976 
 977     template <DecoratorSet decorators>
 978     inline static typename EnableIf<
 979       !HasDecorator<decorators, AS_RAW>::value>::type
 980     clone(oop src, oop dst, size_t size) {
 981       RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
 982     }
 983 
 984     template <DecoratorSet decorators>
 985     inline static typename EnableIf<


1107 
1108   template <DecoratorSet decorators, typename T>
1109   inline T load_reduce_types(T* addr) {
1110     return PreRuntimeDispatch::load<decorators, T>(addr);
1111   }
1112 
1113   template <DecoratorSet decorators, typename T>
1114   inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1115     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1116                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1117     return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1118   }
1119 
1120   template <DecoratorSet decorators, typename T>
1121   inline oop load_reduce_types(HeapWord* addr) {
1122     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1123     return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1124   }
1125 
1126   template <DecoratorSet decorators, typename T>
1127   inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1128                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1129                                      size_t length) {
1130     return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1131                                                      dst_obj, dst_offset_in_bytes, dst_raw,
1132                                                      length);
1133   }
1134 
1135   template <DecoratorSet decorators>
1136   inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1137                                      arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1138                                      size_t length) {
1139     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1140     return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1141                                                               dst_obj, dst_offset_in_bytes, dst_raw,
1142                                                               length);
1143   }
1144 
1145   template <DecoratorSet decorators>
1146   inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1147                                      arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1148                                      size_t length) {
1149     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1150                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1151     return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1152                                                               dst_obj, dst_offset_in_bytes, dst_raw,
1153                                                               length);
1154   }
1155 
1156   // Step 1: Set default decorators. This step remembers if a type was volatile
1157   // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1158   // memory ordering is set for the access, and the implied decorator rules
1159   // are applied to select sensible defaults for decorators that have not been
1160   // explicitly set. For example, default object referent strength is set to strong.
1161   // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1162   // and references from the types). This step also perform some type verification
1163   // that the passed in types make sense.
1164 
1165   template <DecoratorSet decorators, typename T>
1166   static void verify_types(){
1167     // If this fails to compile, then you have sent in something that is
1168     // not recognized as a valid primitive type to a primitive Access function.
1169     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1170                    (IsPointer<T>::value || IsIntegral<T>::value) ||
1171                     IsFloatingPoint<T>::value)); // not allowed primitive type
1172   }
1173 


1266     DecayedT new_decayed_value = new_value;
1267     // atomic_xchg is only available in SEQ_CST flavour.
1268     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1269     return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1270                                                          const_cast<DecayedP*>(addr));
1271   }
1272 
1273   template <DecoratorSet decorators, typename T>
1274   inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1275     verify_types<decorators, T>();
1276     typedef typename Decay<T>::type DecayedT;
1277     DecayedT new_decayed_value = new_value;
1278     // atomic_xchg is only available in SEQ_CST flavour.
1279     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1280                                              (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1281                                               INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1282     return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1283   }
1284 
1285   template <DecoratorSet decorators, typename T>
1286   inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1287                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1288                         size_t length) {
1289     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1290                    (IsSame<T, void>::value || IsIntegral<T>::value) ||
1291                     IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1292     typedef typename Decay<T>::type DecayedT;
1293     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1294     return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1295                                                        dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1296                                                        length);
1297   }
1298 
1299   template <DecoratorSet decorators>
1300   inline void clone(oop src, oop dst, size_t size) {
1301     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1302     PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1303   }
1304 
1305   template <DecoratorSet decorators>
1306   inline oop resolve(oop obj) {
1307     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1308     return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1309   }
1310 
1311   template <DecoratorSet decorators>
1312   inline bool equals(oop o1, oop o2) {
1313     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1314     return PreRuntimeDispatch::equals<expanded_decorators>(o1, o2);
1315   }
1316 




  93   // locking to support wide atomics or not.
  94   template <typename T>
  95 #ifdef SUPPORTS_NATIVE_CX8
  96   struct PossiblyLockedAccess: public IntegralConstant<bool, false> {};
  97 #else
  98   struct PossiblyLockedAccess: public IntegralConstant<bool, (sizeof(T) > 4)> {};
  99 #endif
 100 
 101   template <DecoratorSet decorators, typename T>
 102   struct AccessFunctionTypes {
 103     typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
 104     typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
 105     typedef T (*atomic_cmpxchg_at_func_t)(T new_value, oop base, ptrdiff_t offset, T compare_value);
 106     typedef T (*atomic_xchg_at_func_t)(T new_value, oop base, ptrdiff_t offset);
 107 
 108     typedef T (*load_func_t)(void* addr);
 109     typedef void (*store_func_t)(void* addr, T value);
 110     typedef T (*atomic_cmpxchg_func_t)(T new_value, void* addr, T compare_value);
 111     typedef T (*atomic_xchg_func_t)(T new_value, void* addr);
 112 
 113     typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 114                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 115                                      size_t length);
 116     typedef void (*clone_func_t)(oop src, oop dst, size_t size);
 117     typedef oop (*resolve_func_t)(oop obj);
 118     typedef bool (*equals_func_t)(oop o1, oop o2);
 119   };
 120 
 121   template <DecoratorSet decorators>
 122   struct AccessFunctionTypes<decorators, void> {
 123     typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
 124                                      arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
 125                                      size_t length);
 126   };
 127 
 128   template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
 129 
 130 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func)                   \
 131   template <DecoratorSet decorators, typename T>                    \
 132   struct AccessFunction<decorators, T, bt>: AllStatic{              \
 133     typedef typename AccessFunctionTypes<decorators, T>::func type; \
 134   }
 135   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
 136   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
 137   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
 138   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
 139   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
 140   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
 141   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
 142   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
 143   ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);


 340   static inline void store(void* addr, T value) {
 341     store_internal<decorators>(addr, value);
 342   }
 343 
 344   template <typename T>
 345   static inline T load(void* addr) {
 346     return load_internal<decorators, T>(addr);
 347   }
 348 
 349   template <typename T>
 350   static inline T atomic_cmpxchg(T new_value, void* addr, T compare_value) {
 351     return atomic_cmpxchg_maybe_locked<decorators>(new_value, addr, compare_value);
 352   }
 353 
 354   template <typename T>
 355   static inline T atomic_xchg(T new_value, void* addr) {
 356     return atomic_xchg_maybe_locked<decorators>(new_value, addr);
 357   }
 358 
 359   template <typename T>
 360   static void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 361                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 362                         size_t length);
 363 
 364   template <typename T>
 365   static void oop_store(void* addr, T value);
 366   template <typename T>
 367   static void oop_store_at(oop base, ptrdiff_t offset, T value);
 368 
 369   template <typename T>
 370   static T oop_load(void* addr);
 371   template <typename T>
 372   static T oop_load_at(oop base, ptrdiff_t offset);
 373 
 374   template <typename T>
 375   static T oop_atomic_cmpxchg(T new_value, void* addr, T compare_value);
 376   template <typename T>
 377   static T oop_atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value);
 378 
 379   template <typename T>
 380   static T oop_atomic_xchg(T new_value, void* addr);


 385   static void store_at(oop base, ptrdiff_t offset, T value) {
 386     store(field_addr(base, offset), value);
 387   }
 388 
 389   template <typename T>
 390   static T load_at(oop base, ptrdiff_t offset) {
 391     return load<T>(field_addr(base, offset));
 392   }
 393 
 394   template <typename T>
 395   static T atomic_cmpxchg_at(T new_value, oop base, ptrdiff_t offset, T compare_value) {
 396     return atomic_cmpxchg(new_value, field_addr(base, offset), compare_value);
 397   }
 398 
 399   template <typename T>
 400   static T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
 401     return atomic_xchg(new_value, field_addr(base, offset));
 402   }
 403 
 404   template <typename T>
 405   static void oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 406                             arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 407                             size_t length);
 408 
 409   static void clone(oop src, oop dst, size_t size);
 410 
 411   static oop resolve(oop obj) { return obj; }
 412 
 413   static bool equals(oop o1, oop o2) { return (void*)o1 == (void*)o2; }
 414 };
 415 
 416 // Below is the implementation of the first 4 steps of the template pipeline:
 417 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
 418 //           and sets default decorators to sensible values.
 419 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
 420 //           multiple types. The P type of the address and T type of the value must
 421 //           match.
 422 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
 423 //           avoided, and in that case avoids it (calling raw accesses or
 424 //           primitive accesses in a build that does not require primitive GC barriers)
 425 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding


 550     }
 551   };
 552 
 553   template <DecoratorSet decorators, typename T>
 554   struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
 555     typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
 556     static func_t _atomic_xchg_at_func;
 557 
 558     static T atomic_xchg_at_init(T new_value, oop base, ptrdiff_t offset);
 559 
 560     static inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
 561       return _atomic_xchg_at_func(new_value, base, offset);
 562     }
 563   };
 564 
 565   template <DecoratorSet decorators, typename T>
 566   struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
 567     typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
 568     static func_t _arraycopy_func;
 569 
 570     static void arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 571                                arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 572                                size_t length);
 573 
 574     static inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 575                                  arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 576                                  size_t length) {
 577       return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
 578                              dst_obj, dst_offset_in_bytes, dst_raw,
 579                              length);
 580     }
 581   };
 582 
 583   template <DecoratorSet decorators, typename T>
 584   struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
 585     typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
 586     static func_t _clone_func;
 587 
 588     static void clone_init(oop src, oop dst, size_t size);
 589 
 590     static inline void clone(oop src, oop dst, size_t size) {
 591       _clone_func(src, dst, size);
 592     }
 593   };
 594 


 896     inline static typename EnableIf<
 897       HasDecorator<decorators, AS_RAW>::value, T>::type
 898     atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
 899       return atomic_xchg<decorators>(new_value, field_addr(base, offset));
 900     }
 901 
 902     template <DecoratorSet decorators, typename T>
 903     inline static typename EnableIf<
 904       !HasDecorator<decorators, AS_RAW>::value, T>::type
 905     atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
 906       if (is_hardwired_primitive<decorators>()) {
 907         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 908         return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(new_value, base, offset);
 909       } else {
 910         return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(new_value, base, offset);
 911       }
 912     }
 913 
 914     template <DecoratorSet decorators, typename T>
 915     inline static typename EnableIf<
 916       HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, void>::type
 917     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 918               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 919               size_t length) {
 920       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 921       if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
 922         Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
 923                            dst_obj, dst_offset_in_bytes, dst_raw,
 924                            length);
 925       } else {
 926         Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 927                        dst_obj, dst_offset_in_bytes, dst_raw,
 928                        length);
 929       }
 930     }
 931 
 932     template <DecoratorSet decorators, typename T>
 933     inline static typename EnableIf<
 934       HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, void>::type
 935     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 936               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 937               size_t length) {
 938       if (UseCompressedOops) {
 939         const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
 940         PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 941                                                            dst_obj, dst_offset_in_bytes, dst_raw,
 942                                                            length);
 943       } else {
 944         const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
 945         PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 946                                                            dst_obj, dst_offset_in_bytes, dst_raw,
 947                                                            length);
 948       }
 949     }
 950 
 951     template <DecoratorSet decorators, typename T>
 952     inline static typename EnableIf<
 953       !HasDecorator<decorators, AS_RAW>::value, void>::type
 954     arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
 955               arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
 956               size_t length) {
 957       if (is_hardwired_primitive<decorators>()) {
 958         const DecoratorSet expanded_decorators = decorators | AS_RAW;
 959         PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
 960                                                            dst_obj, dst_offset_in_bytes, dst_raw,
 961                                                            length);
 962       } else {
 963         RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
 964                                                                      dst_obj, dst_offset_in_bytes, dst_raw,
 965                                                                      length);
 966       }
 967     }
 968 
 969     template <DecoratorSet decorators>
 970     inline static typename EnableIf<
 971       HasDecorator<decorators, AS_RAW>::value>::type
 972     clone(oop src, oop dst, size_t size) {
 973       typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
 974       Raw::clone(src, dst, size);
 975     }
 976 
 977     template <DecoratorSet decorators>
 978     inline static typename EnableIf<
 979       !HasDecorator<decorators, AS_RAW>::value>::type
 980     clone(oop src, oop dst, size_t size) {
 981       RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
 982     }
 983 
 984     template <DecoratorSet decorators>
 985     inline static typename EnableIf<


1107 
1108   template <DecoratorSet decorators, typename T>
1109   inline T load_reduce_types(T* addr) {
1110     return PreRuntimeDispatch::load<decorators, T>(addr);
1111   }
1112 
1113   template <DecoratorSet decorators, typename T>
1114   inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1115     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1116                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1117     return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1118   }
1119 
1120   template <DecoratorSet decorators, typename T>
1121   inline oop load_reduce_types(HeapWord* addr) {
1122     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1123     return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1124   }
1125 
1126   template <DecoratorSet decorators, typename T>
1127   inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1128                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1129                                      size_t length) {
1130     PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1131                                               dst_obj, dst_offset_in_bytes, dst_raw,
1132                                               length);
1133   }
1134 
1135   template <DecoratorSet decorators>
1136   inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1137                                      arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1138                                      size_t length) {
1139     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1140     PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1141                                                        dst_obj, dst_offset_in_bytes, dst_raw,
1142                                                        length);
1143   }
1144 
1145   template <DecoratorSet decorators>
1146   inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1147                                      arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1148                                      size_t length) {
1149     const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1150                                              INTERNAL_RT_USE_COMPRESSED_OOPS;
1151     PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1152                                                        dst_obj, dst_offset_in_bytes, dst_raw,
1153                                                        length);
1154   }
1155 
1156   // Step 1: Set default decorators. This step remembers if a type was volatile
1157   // and then sets the MO_VOLATILE decorator by default. Otherwise, a default
1158   // memory ordering is set for the access, and the implied decorator rules
1159   // are applied to select sensible defaults for decorators that have not been
1160   // explicitly set. For example, default object referent strength is set to strong.
1161   // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1162   // and references from the types). This step also perform some type verification
1163   // that the passed in types make sense.
1164 
1165   template <DecoratorSet decorators, typename T>
1166   static void verify_types(){
1167     // If this fails to compile, then you have sent in something that is
1168     // not recognized as a valid primitive type to a primitive Access function.
1169     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1170                    (IsPointer<T>::value || IsIntegral<T>::value) ||
1171                     IsFloatingPoint<T>::value)); // not allowed primitive type
1172   }
1173 


1266     DecayedT new_decayed_value = new_value;
1267     // atomic_xchg is only available in SEQ_CST flavour.
1268     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1269     return atomic_xchg_reduce_types<expanded_decorators>(new_decayed_value,
1270                                                          const_cast<DecayedP*>(addr));
1271   }
1272 
1273   template <DecoratorSet decorators, typename T>
1274   inline T atomic_xchg_at(T new_value, oop base, ptrdiff_t offset) {
1275     verify_types<decorators, T>();
1276     typedef typename Decay<T>::type DecayedT;
1277     DecayedT new_decayed_value = new_value;
1278     // atomic_xchg is only available in SEQ_CST flavour.
1279     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1280                                              (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1281                                               INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1282     return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(new_decayed_value, base, offset);
1283   }
1284 
1285   template <DecoratorSet decorators, typename T>
1286   inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1287                         arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1288                         size_t length) {
1289     STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1290                    (IsSame<T, void>::value || IsIntegral<T>::value) ||
1291                     IsFloatingPoint<T>::value)); // arraycopy allows type erased void elements
1292     typedef typename Decay<T>::type DecayedT;
1293     const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1294     arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1295                                                 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1296                                                 length);
1297   }
1298 
1299   template <DecoratorSet decorators>
1300   inline void clone(oop src, oop dst, size_t size) {
1301     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1302     PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1303   }
1304 
1305   template <DecoratorSet decorators>
1306   inline oop resolve(oop obj) {
1307     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1308     return PreRuntimeDispatch::resolve<expanded_decorators>(obj);
1309   }
1310 
1311   template <DecoratorSet decorators>
1312   inline bool equals(oop o1, oop o2) {
1313     const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1314     return PreRuntimeDispatch::equals<expanded_decorators>(o1, o2);
1315   }
1316 


< prev index next >