< prev index next > src/hotspot/share/oops/accessBackend.hpp
Print this page
#include "cppstdlib/type_traits.hpp"
#include "gc/shared/barrierSetConfig.hpp"
#include "memory/allocation.hpp"
#include "metaprogramming/enableIf.hpp"
#include "oops/accessDecorators.hpp"
+ #include "oops/inlineKlass.hpp"
#include "oops/oopsHierarchy.hpp"
#include "runtime/globals.hpp"
#include "utilities/debug.hpp"
#include "utilities/globalDefinitions.hpp"
+ // Result from oop_arraycopy
+ enum class OopCopyResult {
+ ok, // oop array copy sucessful
+ failed_check_class_cast, // oop array copy failed subtype check (ARRAYCOPY_CHECKCAST)
+ failed_check_null // oop array copy failed null check (ARRAYCOPY_NOTNULL)
+ };
+
// This metafunction returns either oop or narrowOop depending on whether
// an access needs to use compressed oops or not.
template <DecoratorSet decorators>
struct HeapOopType: AllStatic {
static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
using type = std::conditional_t<needs_oop_compress, narrowOop, oop>;
};
+ // This meta-function returns either oop or narrowOop depending on whether
+ // a back-end needs to consider compressed oops types or not.
+ template <DecoratorSet decorators>
+ struct ValueOopType: AllStatic {
+ static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
+ using type = std::conditional_t<needs_oop_compress, narrowOop, oop>;
+ };
+
namespace AccessInternal {
enum BarrierType {
BARRIER_STORE,
BARRIER_STORE_AT,
BARRIER_LOAD,
BARRIER_ATOMIC_CMPXCHG,
BARRIER_ATOMIC_CMPXCHG_AT,
BARRIER_ATOMIC_XCHG,
BARRIER_ATOMIC_XCHG_AT,
BARRIER_ARRAYCOPY,
! BARRIER_CLONE
};
template <DecoratorSet decorators, typename T>
struct MustConvertCompressedOop: public std::integral_constant<bool,
HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
BARRIER_ATOMIC_CMPXCHG,
BARRIER_ATOMIC_CMPXCHG_AT,
BARRIER_ATOMIC_XCHG,
BARRIER_ATOMIC_XCHG_AT,
BARRIER_ARRAYCOPY,
! BARRIER_CLONE,
+ BARRIER_VALUE_COPY
};
template <DecoratorSet decorators, typename T>
struct MustConvertCompressedOop: public std::integral_constant<bool,
HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
typedef T (*load_func_t)(void* addr);
typedef void (*store_func_t)(void* addr, T value);
typedef T (*atomic_cmpxchg_func_t)(void* addr, T compare_value, T new_value);
typedef T (*atomic_xchg_func_t)(void* addr, T new_value);
! typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
! size_t length);
typedef void (*clone_func_t)(oop src, oop dst, size_t size);
};
template <DecoratorSet decorators>
struct AccessFunctionTypes<decorators, void> {
! typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
! arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
! size_t length);
};
template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
#define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
typedef T (*load_func_t)(void* addr);
typedef void (*store_func_t)(void* addr, T value);
typedef T (*atomic_cmpxchg_func_t)(void* addr, T compare_value, T new_value);
typedef T (*atomic_xchg_func_t)(void* addr, T new_value);
! typedef OopCopyResult (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
! size_t length);
typedef void (*clone_func_t)(oop src, oop dst, size_t size);
+ typedef void (*value_copy_func_t)(void* src, void* dst, InlineKlass* md, LayoutKind lk);
};
template <DecoratorSet decorators>
struct AccessFunctionTypes<decorators, void> {
! typedef OopCopyResult (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
! arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
! size_t length);
};
template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
#define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
+ ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_VALUE_COPY, value_copy_func_t);
#undef ACCESS_GENERATE_ACCESS_FUNCTION
template <DecoratorSet decorators, typename T, BarrierType barrier_type>
typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier();
void arraycopy_conjoint(T* src, T* dst, size_t length);
template<typename T>
void arraycopy_arrayof_conjoint(T* src, T* dst, size_t length);
template<typename T>
void arraycopy_conjoint_atomic(T* src, T* dst, size_t length);
+
+ void value_copy_internal(void* src, void* dst, size_t length);
}
// This mask specifies what decorators are relevant for raw accesses. When passing
// accesses to the raw layer, irrelevant decorators are removed.
const DecoratorSet RAW_DECORATOR_MASK = INTERNAL_DECORATOR_MASK | MO_DECORATOR_MASK |
static inline T atomic_xchg(void* addr, T new_value) {
return atomic_xchg_internal<decorators>(addr, new_value);
}
template <typename T>
! static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length);
template <typename T>
static void oop_store(void* addr, T value);
static inline T atomic_xchg(void* addr, T new_value) {
return atomic_xchg_internal<decorators>(addr, new_value);
}
template <typename T>
! static void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length);
template <typename T>
static void oop_store(void* addr, T value);
static T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
return atomic_xchg(field_addr(base, offset), new_value);
}
template <typename T>
! static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length);
static void clone(oop src, oop dst, size_t size);
};
namespace AccessInternal {
DEBUG_ONLY(void check_access_thread_state());
#define assert_access_thread_state() DEBUG_ONLY(check_access_thread_state())
static T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
return atomic_xchg(field_addr(base, offset), new_value);
}
template <typename T>
! static void oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length);
static void clone(oop src, oop dst, size_t size);
+ static void value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk);
+
};
namespace AccessInternal {
DEBUG_ONLY(void check_access_thread_state());
#define assert_access_thread_state() DEBUG_ONLY(check_access_thread_state())
template <DecoratorSet decorators, typename T>
struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
static func_t _arraycopy_func;
! static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
! size_t length);
! static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
! size_t length) {
assert_access_thread_state();
return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
dst_obj, dst_offset_in_bytes, dst_raw,
length);
}
template <DecoratorSet decorators, typename T>
struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
static func_t _arraycopy_func;
! static OopCopyResult arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
! size_t length);
! static inline OopCopyResult arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
! size_t length) {
assert_access_thread_state();
return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
dst_obj, dst_offset_in_bytes, dst_raw,
length);
}
assert_access_thread_state();
_clone_func(src, dst, size);
}
};
+ template <DecoratorSet decorators, typename T>
+ struct RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>: AllStatic {
+ typedef typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type func_t;
+ static func_t _value_copy_func;
+
+ static void value_copy_init(void* src, void* dst, InlineKlass* md, LayoutKind lk);
+
+ static inline void value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
+ _value_copy_func(src, dst, md, lk);
+ }
+ };
+
// Initialize the function pointers to point to the resolving function.
template <DecoratorSet decorators, typename T>
typename AccessFunction<decorators, T, BARRIER_STORE>::type
RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init;
template <DecoratorSet decorators, typename T>
typename AccessFunction<decorators, T, BARRIER_CLONE>::type
RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
+ template <DecoratorSet decorators, typename T>
+ typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type
+ RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>::_value_copy_func = &value_copy_init;
+
// Step 3: Pre-runtime dispatching.
// The PreRuntimeDispatch class is responsible for filtering the barrier strength
// decorators. That is, for AS_RAW, it hardwires the accesses without a runtime
// dispatch point. Otherwise it goes through a runtime check if hardwiring was
// not possible.
}
}
template <DecoratorSet decorators, typename T>
inline static typename EnableIf<
! HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length) {
typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
! return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
! dst_obj, dst_offset_in_bytes, dst_raw,
! length);
} else {
! return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
! dst_obj, dst_offset_in_bytes, dst_raw,
! length);
}
}
template <DecoratorSet decorators, typename T>
inline static typename EnableIf<
! HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length) {
if (UseCompressedOops) {
const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
}
}
template <DecoratorSet decorators, typename T>
inline static typename EnableIf<
! HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, OopCopyResult>::type
arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length) {
typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
! Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
! dst_obj, dst_offset_in_bytes, dst_raw,
! length);
} else {
! Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
! dst_obj, dst_offset_in_bytes, dst_raw,
! length);
}
+
+ return OopCopyResult::ok;
}
template <DecoratorSet decorators, typename T>
inline static typename EnableIf<
! HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, OopCopyResult>::type
arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length) {
if (UseCompressedOops) {
const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
}
}
template <DecoratorSet decorators, typename T>
inline static typename EnableIf<
! !HasDecorator<decorators, AS_RAW>::value, bool>::type
arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length) {
if (is_hardwired_primitive<decorators>()) {
const DecoratorSet expanded_decorators = decorators | AS_RAW;
}
}
template <DecoratorSet decorators, typename T>
inline static typename EnableIf<
! !HasDecorator<decorators, AS_RAW>::value, OopCopyResult>::type
arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
size_t length) {
if (is_hardwired_primitive<decorators>()) {
const DecoratorSet expanded_decorators = decorators | AS_RAW;
inline static typename EnableIf<
!HasDecorator<decorators, AS_RAW>::value>::type
clone(oop src, oop dst, size_t size) {
RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
}
+
+ template <DecoratorSet decorators>
+ inline static typename EnableIf<
+ HasDecorator<decorators, AS_RAW>::value>::type
+ value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
+ typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
+ Raw::value_copy(src, dst, md, lk);
+ }
+
+ template <DecoratorSet decorators>
+ inline static typename EnableIf<
+ !HasDecorator<decorators, AS_RAW>::value>::type
+ value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
+ const DecoratorSet expanded_decorators = decorators;
+ RuntimeDispatch<expanded_decorators, void*, BARRIER_VALUE_COPY>::value_copy(src, dst, md, lk);
+ }
};
// Step 2: Reduce types.
// Enforce that for non-oop types, T and P have to be strictly the same.
// P is the type of the address and T is the type of the values.
const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
}
template <DecoratorSet decorators, typename T>
! inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
! size_t length) {
return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
dst_obj, dst_offset_in_bytes, dst_raw,
length);
}
template <DecoratorSet decorators>
! inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
! size_t length) {
const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
dst_obj, dst_offset_in_bytes, dst_raw,
length);
}
template <DecoratorSet decorators>
! inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
! size_t length) {
const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
INTERNAL_RT_USE_COMPRESSED_OOPS;
return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
dst_obj, dst_offset_in_bytes, dst_raw,
length);
const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
}
template <DecoratorSet decorators, typename T>
! inline OopCopyResult arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
! size_t length) {
return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
dst_obj, dst_offset_in_bytes, dst_raw,
length);
}
template <DecoratorSet decorators>
! inline OopCopyResult arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
! size_t length) {
const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
dst_obj, dst_offset_in_bytes, dst_raw,
length);
}
template <DecoratorSet decorators>
! inline OopCopyResult arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
! size_t length) {
const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
INTERNAL_RT_USE_COMPRESSED_OOPS;
return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
dst_obj, dst_offset_in_bytes, dst_raw,
length);
INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(base, offset, new_decayed_value);
}
template <DecoratorSet decorators, typename T>
! inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
! size_t length) {
STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
(std::is_same<T, void>::value || std::is_integral<T>::value) ||
std::is_floating_point<T>::value)); // arraycopy allows type erased void elements
using DecayedT = std::decay_t<T>;
const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(base, offset, new_decayed_value);
}
template <DecoratorSet decorators, typename T>
! inline OopCopyResult arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
! arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
! size_t length) {
STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
(std::is_same<T, void>::value || std::is_integral<T>::value) ||
std::is_floating_point<T>::value)); // arraycopy allows type erased void elements
using DecayedT = std::decay_t<T>;
const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
inline void clone(oop src, oop dst, size_t size) {
const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
}
+ template <DecoratorSet decorators>
+ inline void value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
+ const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
+ PreRuntimeDispatch::value_copy<expanded_decorators>(src, dst, md, lk);
+ }
+
// Infer the type that should be returned from an Access::oop_load.
template <typename P, DecoratorSet decorators>
class OopLoadProxy: public StackObj {
private:
P *const _addr;
< prev index next >