13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_OOPS_ACCESSBACKEND_HPP
26 #define SHARE_OOPS_ACCESSBACKEND_HPP
27
28 #include "cppstdlib/type_traits.hpp"
29 #include "gc/shared/barrierSetConfig.hpp"
30 #include "memory/allocation.hpp"
31 #include "metaprogramming/enableIf.hpp"
32 #include "oops/accessDecorators.hpp"
33 #include "oops/oopsHierarchy.hpp"
34 #include "runtime/globals.hpp"
35 #include "utilities/debug.hpp"
36 #include "utilities/globalDefinitions.hpp"
37
38 // This metafunction returns either oop or narrowOop depending on whether
39 // an access needs to use compressed oops or not.
40 template <DecoratorSet decorators>
41 struct HeapOopType: AllStatic {
42 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
43 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
44 using type = std::conditional_t<needs_oop_compress, narrowOop, oop>;
45 };
46
47 namespace AccessInternal {
48 enum BarrierType {
49 BARRIER_STORE,
50 BARRIER_STORE_AT,
51 BARRIER_LOAD,
52 BARRIER_LOAD_AT,
53 BARRIER_ATOMIC_CMPXCHG,
54 BARRIER_ATOMIC_CMPXCHG_AT,
55 BARRIER_ATOMIC_XCHG,
56 BARRIER_ATOMIC_XCHG_AT,
57 BARRIER_ARRAYCOPY,
58 BARRIER_CLONE
59 };
60
61 template <DecoratorSet decorators, typename T>
62 struct MustConvertCompressedOop: public std::integral_constant<bool,
63 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
64 std::is_same<typename HeapOopType<decorators>::type, narrowOop>::value &&
65 std::is_same<T, oop>::value> {};
66
67 // This metafunction returns an appropriate oop type if the value is oop-like
68 // and otherwise returns the same type T.
69 template <DecoratorSet decorators, typename T>
70 struct EncodedType: AllStatic {
71 using type = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
72 typename HeapOopType<decorators>::type,
73 T>;
74 };
75
76 template <DecoratorSet decorators>
77 inline typename HeapOopType<decorators>::type*
78 oop_field_addr(oop base, ptrdiff_t byte_offset) {
79 return reinterpret_cast<typename HeapOopType<decorators>::type*>(
80 reinterpret_cast<intptr_t>((void*)base) + byte_offset);
81 }
82
83 template <DecoratorSet decorators, typename T>
84 struct AccessFunctionTypes {
85 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
86 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
87 typedef T (*atomic_cmpxchg_at_func_t)(oop base, ptrdiff_t offset, T compare_value, T new_value);
88 typedef T (*atomic_xchg_at_func_t)(oop base, ptrdiff_t offset, T new_value);
89
90 typedef T (*load_func_t)(void* addr);
91 typedef void (*store_func_t)(void* addr, T value);
92 typedef T (*atomic_cmpxchg_func_t)(void* addr, T compare_value, T new_value);
93 typedef T (*atomic_xchg_func_t)(void* addr, T new_value);
94
95 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
96 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
97 size_t length);
98 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
99 };
100
101 template <DecoratorSet decorators>
102 struct AccessFunctionTypes<decorators, void> {
103 typedef bool (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
104 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
105 size_t length);
106 };
107
108 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
109
110 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
111 template <DecoratorSet decorators, typename T> \
112 struct AccessFunction<decorators, T, bt>: AllStatic{ \
113 typedef typename AccessFunctionTypes<decorators, T>::func type; \
114 }
115 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
116 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
117 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
118 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
119 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
120 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
121 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
122 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
123 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
124 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
125 #undef ACCESS_GENERATE_ACCESS_FUNCTION
126
127 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
128 typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier();
129
130 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
131 typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier();
132
133 void* field_addr(oop base, ptrdiff_t offset);
134
135 // Forward calls to Copy:: in the cpp file to reduce dependencies and allow
136 // faster build times, given how frequently included access is.
137 void arraycopy_arrayof_conjoint_oops(void* src, void* dst, size_t length);
138 void arraycopy_conjoint_oops(oop* src, oop* dst, size_t length);
139 void arraycopy_conjoint_oops(narrowOop* src, narrowOop* dst, size_t length);
140
141 void arraycopy_disjoint_words(void* src, void* dst, size_t length);
142 void arraycopy_disjoint_words_atomic(void* src, void* dst, size_t length);
143
144 template<typename T>
145 void arraycopy_conjoint(T* src, T* dst, size_t length);
146 template<typename T>
147 void arraycopy_arrayof_conjoint(T* src, T* dst, size_t length);
148 template<typename T>
149 void arraycopy_conjoint_atomic(T* src, T* dst, size_t length);
150 }
151
152 // This mask specifies what decorators are relevant for raw accesses. When passing
153 // accesses to the raw layer, irrelevant decorators are removed.
154 const DecoratorSet RAW_DECORATOR_MASK = INTERNAL_DECORATOR_MASK | MO_DECORATOR_MASK |
155 ARRAYCOPY_DECORATOR_MASK | IS_NOT_NULL;
156
157 // The RawAccessBarrier performs raw accesses with additional knowledge of
158 // memory ordering, so that OrderAccess/Atomic is called when necessary.
159 // It additionally handles compressed oops, and hence is not completely "raw"
160 // strictly speaking.
161 template <DecoratorSet decorators>
162 class RawAccessBarrier: public AllStatic {
163 protected:
164 static inline void* field_addr(oop base, ptrdiff_t byte_offset) {
165 return AccessInternal::field_addr(base, byte_offset);
166 }
167
168 protected:
169 // Only encode if INTERNAL_VALUE_IS_OOP
269 static inline void store(void* addr, T value) {
270 store_internal<decorators>(addr, value);
271 }
272
273 template <typename T>
274 static inline T load(void* addr) {
275 return load_internal<decorators, T>(addr);
276 }
277
278 template <typename T>
279 static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) {
280 return atomic_cmpxchg_internal<decorators>(addr, compare_value, new_value);
281 }
282
283 template <typename T>
284 static inline T atomic_xchg(void* addr, T new_value) {
285 return atomic_xchg_internal<decorators>(addr, new_value);
286 }
287
288 template <typename T>
289 static bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
290 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
291 size_t length);
292
293 template <typename T>
294 static void oop_store(void* addr, T value);
295 template <typename T>
296 static void oop_store_at(oop base, ptrdiff_t offset, T value);
297
298 template <typename T>
299 static T oop_load(void* addr);
300 template <typename T>
301 static T oop_load_at(oop base, ptrdiff_t offset);
302
303 template <typename T>
304 static T oop_atomic_cmpxchg(void* addr, T compare_value, T new_value);
305 template <typename T>
306 static T oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value);
307
308 template <typename T>
309 static T oop_atomic_xchg(void* addr, T new_value);
314 static void store_at(oop base, ptrdiff_t offset, T value) {
315 store(field_addr(base, offset), value);
316 }
317
318 template <typename T>
319 static T load_at(oop base, ptrdiff_t offset) {
320 return load<T>(field_addr(base, offset));
321 }
322
323 template <typename T>
324 static T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
325 return atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
326 }
327
328 template <typename T>
329 static T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
330 return atomic_xchg(field_addr(base, offset), new_value);
331 }
332
333 template <typename T>
334 static bool oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
335 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
336 size_t length);
337
338 static void clone(oop src, oop dst, size_t size);
339 };
340
341 namespace AccessInternal {
342 DEBUG_ONLY(void check_access_thread_state());
343 #define assert_access_thread_state() DEBUG_ONLY(check_access_thread_state())
344 }
345
346 // Below is the implementation of the first 4 steps of the template pipeline:
347 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
348 // and sets default decorators to sensible values.
349 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
350 // multiple types. The P type of the address and T type of the value must
351 // match.
352 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
353 // avoided, and in that case avoids it (calling raw accesses or
354 // primitive accesses in a build that does not require primitive GC barriers)
355 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
356 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers
357 // to the access.
358
488 };
489
490 template <DecoratorSet decorators, typename T>
491 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
492 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
493 static func_t _atomic_xchg_at_func;
494
495 static T atomic_xchg_at_init(oop base, ptrdiff_t offset, T new_value);
496
497 static inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
498 assert_access_thread_state();
499 return _atomic_xchg_at_func(base, offset, new_value);
500 }
501 };
502
503 template <DecoratorSet decorators, typename T>
504 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
505 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
506 static func_t _arraycopy_func;
507
508 static bool arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
509 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
510 size_t length);
511
512 static inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
513 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
514 size_t length) {
515 assert_access_thread_state();
516 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
517 dst_obj, dst_offset_in_bytes, dst_raw,
518 length);
519 }
520 };
521
522 template <DecoratorSet decorators, typename T>
523 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
524 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
525 static func_t _clone_func;
526
527 static void clone_init(oop src, oop dst, size_t size);
528
529 static inline void clone(oop src, oop dst, size_t size) {
530 assert_access_thread_state();
531 _clone_func(src, dst, size);
532 }
533 };
534
535 // Initialize the function pointers to point to the resolving function.
536 template <DecoratorSet decorators, typename T>
537 typename AccessFunction<decorators, T, BARRIER_STORE>::type
538 RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init;
539
540 template <DecoratorSet decorators, typename T>
541 typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type
542 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init;
543
544 template <DecoratorSet decorators, typename T>
545 typename AccessFunction<decorators, T, BARRIER_LOAD>::type
546 RuntimeDispatch<decorators, T, BARRIER_LOAD>::_load_func = &load_init;
547
548 template <DecoratorSet decorators, typename T>
549 typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type
550 RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::_load_at_func = &load_at_init;
551
552 template <DecoratorSet decorators, typename T>
553 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type
554 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::_atomic_cmpxchg_func = &atomic_cmpxchg_init;
556 template <DecoratorSet decorators, typename T>
557 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type
558 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init;
559
560 template <DecoratorSet decorators, typename T>
561 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type
562 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init;
563
564 template <DecoratorSet decorators, typename T>
565 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type
566 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init;
567
568 template <DecoratorSet decorators, typename T>
569 typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type
570 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init;
571
572 template <DecoratorSet decorators, typename T>
573 typename AccessFunction<decorators, T, BARRIER_CLONE>::type
574 RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
575
576 // Step 3: Pre-runtime dispatching.
577 // The PreRuntimeDispatch class is responsible for filtering the barrier strength
578 // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime
579 // dispatch point. Otherwise it goes through a runtime check if hardwiring was
580 // not possible.
581 struct PreRuntimeDispatch: AllStatic {
582 template<DecoratorSet decorators>
583 struct CanHardwireRaw: public std::integral_constant<
584 bool,
585 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access
586 !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address)
587 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address)
588 {};
589
590 static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP;
591
592 template<DecoratorSet decorators>
593 static bool is_hardwired_primitive() {
594 return !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value;
595 }
803 inline static typename EnableIf<
804 HasDecorator<decorators, AS_RAW>::value, T>::type
805 atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
806 return atomic_xchg<decorators>(field_addr(base, offset), new_value);
807 }
808
809 template <DecoratorSet decorators, typename T>
810 inline static typename EnableIf<
811 !HasDecorator<decorators, AS_RAW>::value, T>::type
812 atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
813 if (is_hardwired_primitive<decorators>()) {
814 const DecoratorSet expanded_decorators = decorators | AS_RAW;
815 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(base, offset, new_value);
816 } else {
817 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(base, offset, new_value);
818 }
819 }
820
821 template <DecoratorSet decorators, typename T>
822 inline static typename EnableIf<
823 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, bool>::type
824 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
825 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
826 size_t length) {
827 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
828 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
829 return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
830 dst_obj, dst_offset_in_bytes, dst_raw,
831 length);
832 } else {
833 return Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
834 dst_obj, dst_offset_in_bytes, dst_raw,
835 length);
836 }
837 }
838
839 template <DecoratorSet decorators, typename T>
840 inline static typename EnableIf<
841 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, bool>::type
842 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
843 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
844 size_t length) {
845 if (UseCompressedOops) {
846 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
847 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
848 dst_obj, dst_offset_in_bytes, dst_raw,
849 length);
850 } else {
851 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
852 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
853 dst_obj, dst_offset_in_bytes, dst_raw,
854 length);
855 }
856 }
857
858 template <DecoratorSet decorators, typename T>
859 inline static typename EnableIf<
860 !HasDecorator<decorators, AS_RAW>::value, bool>::type
861 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
862 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
863 size_t length) {
864 if (is_hardwired_primitive<decorators>()) {
865 const DecoratorSet expanded_decorators = decorators | AS_RAW;
866 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
867 dst_obj, dst_offset_in_bytes, dst_raw,
868 length);
869 } else {
870 return RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
871 dst_obj, dst_offset_in_bytes, dst_raw,
872 length);
873 }
874 }
875
876 template <DecoratorSet decorators>
877 inline static typename EnableIf<
878 HasDecorator<decorators, AS_RAW>::value>::type
879 clone(oop src, oop dst, size_t size) {
880 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
881 Raw::clone(src, dst, size);
882 }
883
884 template <DecoratorSet decorators>
885 inline static typename EnableIf<
886 !HasDecorator<decorators, AS_RAW>::value>::type
887 clone(oop src, oop dst, size_t size) {
888 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
889 }
890 };
891
892 // Step 2: Reduce types.
893 // Enforce that for non-oop types, T and P have to be strictly the same.
894 // P is the type of the address and T is the type of the values.
895 // As for oop types, it is allow to send T in {narrowOop, oop} and
896 // P in {narrowOop, oop, HeapWord*}. The following rules apply according to
897 // the subsequent table. (columns are P, rows are T)
898 // | | HeapWord | oop | narrowOop |
899 // | oop | rt-comp | hw-none | hw-comp |
900 // | narrowOop | x | x | hw-none |
901 //
902 // x means not allowed
903 // rt-comp means it must be checked at runtime whether the oop is compressed.
904 // hw-none means it is statically known the oop will not be compressed.
905 // hw-comp means it is statically known the oop will be compressed.
906
907 template <DecoratorSet decorators, typename T>
908 inline void store_reduce_types(T* addr, T value) {
909 PreRuntimeDispatch::store<decorators>(addr, value);
984
985 template <DecoratorSet decorators, typename T>
986 inline T load_reduce_types(T* addr) {
987 return PreRuntimeDispatch::load<decorators, T>(addr);
988 }
989
990 template <DecoratorSet decorators, typename T>
991 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
992 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
993 INTERNAL_RT_USE_COMPRESSED_OOPS;
994 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
995 }
996
997 template <DecoratorSet decorators, typename T>
998 inline oop load_reduce_types(HeapWord* addr) {
999 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1000 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1001 }
1002
1003 template <DecoratorSet decorators, typename T>
1004 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1005 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1006 size_t length) {
1007 return PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1008 dst_obj, dst_offset_in_bytes, dst_raw,
1009 length);
1010 }
1011
1012 template <DecoratorSet decorators>
1013 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1014 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1015 size_t length) {
1016 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1017 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1018 dst_obj, dst_offset_in_bytes, dst_raw,
1019 length);
1020 }
1021
1022 template <DecoratorSet decorators>
1023 inline bool arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1024 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1025 size_t length) {
1026 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1027 INTERNAL_RT_USE_COMPRESSED_OOPS;
1028 return PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1029 dst_obj, dst_offset_in_bytes, dst_raw,
1030 length);
1031 }
1032
1033 // Step 1: Set default decorators. This step remembers if a type was volatile
1034 // and then sets the MO_RELAXED decorator by default. Otherwise, a default
1035 // memory ordering is set for the access, and the implied decorator rules
1036 // are applied to select sensible defaults for decorators that have not been
1037 // explicitly set. For example, default object referent strength is set to strong.
1038 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1039 // and references from the types). This step also perform some type verification
1040 // that the passed in types make sense.
1041
1042 template <DecoratorSet decorators, typename T>
1043 static void verify_types(){
1044 // If this fails to compile, then you have sent in something that is
1045 // not recognized as a valid primitive type to a primitive Access function.
1046 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1047 (std::is_pointer<T>::value || std::is_integral<T>::value) ||
1048 std::is_floating_point<T>::value)); // not allowed primitive type
1049 }
1050
1143 DecayedT new_decayed_value = new_value;
1144 // atomic_xchg is only available in SEQ_CST flavour.
1145 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1146 return atomic_xchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr),
1147 new_decayed_value);
1148 }
1149
1150 template <DecoratorSet decorators, typename T>
1151 inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
1152 verify_types<decorators, T>();
1153 using DecayedT = std::decay_t<T>;
1154 DecayedT new_decayed_value = new_value;
1155 // atomic_xchg is only available in SEQ_CST flavour.
1156 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1157 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1158 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1159 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(base, offset, new_decayed_value);
1160 }
1161
1162 template <DecoratorSet decorators, typename T>
1163 inline bool arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1164 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1165 size_t length) {
1166 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1167 (std::is_same<T, void>::value || std::is_integral<T>::value) ||
1168 std::is_floating_point<T>::value)); // arraycopy allows type erased void elements
1169 using DecayedT = std::decay_t<T>;
1170 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1171 return arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1172 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1173 length);
1174 }
1175
1176 template <DecoratorSet decorators>
1177 inline void clone(oop src, oop dst, size_t size) {
1178 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1179 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1180 }
1181
1182 // Infer the type that should be returned from an Access::oop_load.
1183 template <typename P, DecoratorSet decorators>
1184 class OopLoadProxy: public StackObj {
1185 private:
1186 P *const _addr;
1187 public:
1188 explicit OopLoadProxy(P* addr) : _addr(addr) {}
1189
1190 inline operator oop() {
1191 return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr);
1192 }
1193
1194 inline operator narrowOop() {
1195 return load<decorators | INTERNAL_VALUE_IS_OOP, P, narrowOop>(_addr);
1196 }
1197
1198 template <typename T>
1199 inline bool operator ==(const T& other) const {
1200 return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) == other;
1201 }
|
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #ifndef SHARE_OOPS_ACCESSBACKEND_HPP
26 #define SHARE_OOPS_ACCESSBACKEND_HPP
27
28 #include "cppstdlib/type_traits.hpp"
29 #include "gc/shared/barrierSetConfig.hpp"
30 #include "memory/allocation.hpp"
31 #include "metaprogramming/enableIf.hpp"
32 #include "oops/accessDecorators.hpp"
33 #include "oops/inlineKlass.hpp"
34 #include "oops/oopsHierarchy.hpp"
35 #include "runtime/globals.hpp"
36 #include "utilities/debug.hpp"
37 #include "utilities/globalDefinitions.hpp"
38
39 // This metafunction returns either oop or narrowOop depending on whether
40 // an access needs to use compressed oops or not.
41 template <DecoratorSet decorators>
42 struct HeapOopType: AllStatic {
43 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value &&
44 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
45 using type = std::conditional_t<needs_oop_compress, narrowOop, oop>;
46 };
47
48 // This meta-function returns either oop or narrowOop depending on whether
49 // a back-end needs to consider compressed oops types or not.
50 template <DecoratorSet decorators>
51 struct ValueOopType: AllStatic {
52 static const bool needs_oop_compress = HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value;
53 using type = std::conditional_t<needs_oop_compress, narrowOop, oop>;
54 };
55
56 namespace AccessInternal {
57 enum BarrierType {
58 BARRIER_STORE,
59 BARRIER_STORE_AT,
60 BARRIER_LOAD,
61 BARRIER_LOAD_AT,
62 BARRIER_ATOMIC_CMPXCHG,
63 BARRIER_ATOMIC_CMPXCHG_AT,
64 BARRIER_ATOMIC_XCHG,
65 BARRIER_ATOMIC_XCHG_AT,
66 BARRIER_ARRAYCOPY,
67 BARRIER_CLONE,
68 BARRIER_VALUE_COPY
69 };
70
71 template <DecoratorSet decorators, typename T>
72 struct MustConvertCompressedOop: public std::integral_constant<bool,
73 HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value &&
74 std::is_same<typename HeapOopType<decorators>::type, narrowOop>::value &&
75 std::is_same<T, oop>::value> {};
76
77 // This metafunction returns an appropriate oop type if the value is oop-like
78 // and otherwise returns the same type T.
79 template <DecoratorSet decorators, typename T>
80 struct EncodedType: AllStatic {
81 using type = std::conditional_t<HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value,
82 typename HeapOopType<decorators>::type,
83 T>;
84 };
85
86 template <DecoratorSet decorators>
87 inline typename HeapOopType<decorators>::type*
88 oop_field_addr(oop base, ptrdiff_t byte_offset) {
89 return reinterpret_cast<typename HeapOopType<decorators>::type*>(
90 reinterpret_cast<intptr_t>((void*)base) + byte_offset);
91 }
92
93 template <DecoratorSet decorators, typename T>
94 struct AccessFunctionTypes {
95 typedef T (*load_at_func_t)(oop base, ptrdiff_t offset);
96 typedef void (*store_at_func_t)(oop base, ptrdiff_t offset, T value);
97 typedef T (*atomic_cmpxchg_at_func_t)(oop base, ptrdiff_t offset, T compare_value, T new_value);
98 typedef T (*atomic_xchg_at_func_t)(oop base, ptrdiff_t offset, T new_value);
99
100 typedef T (*load_func_t)(void* addr);
101 typedef void (*store_func_t)(void* addr, T value);
102 typedef T (*atomic_cmpxchg_func_t)(void* addr, T compare_value, T new_value);
103 typedef T (*atomic_xchg_func_t)(void* addr, T new_value);
104
105 typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
106 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
107 size_t length);
108 typedef void (*clone_func_t)(oop src, oop dst, size_t size);
109 typedef void (*value_copy_func_t)(void* src, void* dst, InlineKlass* md, LayoutKind lk);
110 };
111
112 template <DecoratorSet decorators>
113 struct AccessFunctionTypes<decorators, void> {
114 typedef void (*arraycopy_func_t)(arrayOop src_obj, size_t src_offset_in_bytes, void* src,
115 arrayOop dst_obj, size_t dst_offset_in_bytes, void* dst,
116 size_t length);
117 };
118
119 template <DecoratorSet decorators, typename T, BarrierType barrier> struct AccessFunction {};
120
121 #define ACCESS_GENERATE_ACCESS_FUNCTION(bt, func) \
122 template <DecoratorSet decorators, typename T> \
123 struct AccessFunction<decorators, T, bt>: AllStatic{ \
124 typedef typename AccessFunctionTypes<decorators, T>::func type; \
125 }
126 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE, store_func_t);
127 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_STORE_AT, store_at_func_t);
128 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD, load_func_t);
129 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_LOAD_AT, load_at_func_t);
130 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG, atomic_cmpxchg_func_t);
131 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_CMPXCHG_AT, atomic_cmpxchg_at_func_t);
132 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG, atomic_xchg_func_t);
133 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ATOMIC_XCHG_AT, atomic_xchg_at_func_t);
134 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_ARRAYCOPY, arraycopy_func_t);
135 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_CLONE, clone_func_t);
136 ACCESS_GENERATE_ACCESS_FUNCTION(BARRIER_VALUE_COPY, value_copy_func_t);
137 #undef ACCESS_GENERATE_ACCESS_FUNCTION
138
139 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
140 typename AccessFunction<decorators, T, barrier_type>::type resolve_barrier();
141
142 template <DecoratorSet decorators, typename T, BarrierType barrier_type>
143 typename AccessFunction<decorators, T, barrier_type>::type resolve_oop_barrier();
144
145 void* field_addr(oop base, ptrdiff_t offset);
146
147 // Forward calls to Copy:: in the cpp file to reduce dependencies and allow
148 // faster build times, given how frequently included access is.
149 void arraycopy_arrayof_conjoint_oops(void* src, void* dst, size_t length);
150 void arraycopy_conjoint_oops(oop* src, oop* dst, size_t length);
151 void arraycopy_conjoint_oops(narrowOop* src, narrowOop* dst, size_t length);
152
153 void arraycopy_disjoint_words(void* src, void* dst, size_t length);
154 void arraycopy_disjoint_words_atomic(void* src, void* dst, size_t length);
155
156 template<typename T>
157 void arraycopy_conjoint(T* src, T* dst, size_t length);
158 template<typename T>
159 void arraycopy_arrayof_conjoint(T* src, T* dst, size_t length);
160 template<typename T>
161 void arraycopy_conjoint_atomic(T* src, T* dst, size_t length);
162
163 void value_copy_internal(void* src, void* dst, size_t length);
164 }
165
166 // This mask specifies what decorators are relevant for raw accesses. When passing
167 // accesses to the raw layer, irrelevant decorators are removed.
168 const DecoratorSet RAW_DECORATOR_MASK = INTERNAL_DECORATOR_MASK | MO_DECORATOR_MASK |
169 ARRAYCOPY_DECORATOR_MASK | IS_NOT_NULL;
170
171 // The RawAccessBarrier performs raw accesses with additional knowledge of
172 // memory ordering, so that OrderAccess/Atomic is called when necessary.
173 // It additionally handles compressed oops, and hence is not completely "raw"
174 // strictly speaking.
175 template <DecoratorSet decorators>
176 class RawAccessBarrier: public AllStatic {
177 protected:
178 static inline void* field_addr(oop base, ptrdiff_t byte_offset) {
179 return AccessInternal::field_addr(base, byte_offset);
180 }
181
182 protected:
183 // Only encode if INTERNAL_VALUE_IS_OOP
283 static inline void store(void* addr, T value) {
284 store_internal<decorators>(addr, value);
285 }
286
287 template <typename T>
288 static inline T load(void* addr) {
289 return load_internal<decorators, T>(addr);
290 }
291
292 template <typename T>
293 static inline T atomic_cmpxchg(void* addr, T compare_value, T new_value) {
294 return atomic_cmpxchg_internal<decorators>(addr, compare_value, new_value);
295 }
296
297 template <typename T>
298 static inline T atomic_xchg(void* addr, T new_value) {
299 return atomic_xchg_internal<decorators>(addr, new_value);
300 }
301
302 template <typename T>
303 static void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
304 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
305 size_t length);
306
307 template <typename T>
308 static void oop_store(void* addr, T value);
309 template <typename T>
310 static void oop_store_at(oop base, ptrdiff_t offset, T value);
311
312 template <typename T>
313 static T oop_load(void* addr);
314 template <typename T>
315 static T oop_load_at(oop base, ptrdiff_t offset);
316
317 template <typename T>
318 static T oop_atomic_cmpxchg(void* addr, T compare_value, T new_value);
319 template <typename T>
320 static T oop_atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value);
321
322 template <typename T>
323 static T oop_atomic_xchg(void* addr, T new_value);
328 static void store_at(oop base, ptrdiff_t offset, T value) {
329 store(field_addr(base, offset), value);
330 }
331
332 template <typename T>
333 static T load_at(oop base, ptrdiff_t offset) {
334 return load<T>(field_addr(base, offset));
335 }
336
337 template <typename T>
338 static T atomic_cmpxchg_at(oop base, ptrdiff_t offset, T compare_value, T new_value) {
339 return atomic_cmpxchg(field_addr(base, offset), compare_value, new_value);
340 }
341
342 template <typename T>
343 static T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
344 return atomic_xchg(field_addr(base, offset), new_value);
345 }
346
347 template <typename T>
348 static void oop_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
349 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
350 size_t length);
351
352 static void clone(oop src, oop dst, size_t size);
353 static void value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk);
354
355 };
356
357 namespace AccessInternal {
358 DEBUG_ONLY(void check_access_thread_state());
359 #define assert_access_thread_state() DEBUG_ONLY(check_access_thread_state())
360 }
361
362 // Below is the implementation of the first 4 steps of the template pipeline:
363 // * Step 1: Set default decorators and decay types. This step gets rid of CV qualifiers
364 // and sets default decorators to sensible values.
365 // * Step 2: Reduce types. This step makes sure there is only a single T type and not
366 // multiple types. The P type of the address and T type of the value must
367 // match.
368 // * Step 3: Pre-runtime dispatch. This step checks whether a runtime call can be
369 // avoided, and in that case avoids it (calling raw accesses or
370 // primitive accesses in a build that does not require primitive GC barriers)
371 // * Step 4: Runtime-dispatch. This step performs a runtime dispatch to the corresponding
372 // BarrierSet::AccessBarrier accessor that attaches GC-required barriers
373 // to the access.
374
504 };
505
506 template <DecoratorSet decorators, typename T>
507 struct RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>: AllStatic {
508 typedef typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type func_t;
509 static func_t _atomic_xchg_at_func;
510
511 static T atomic_xchg_at_init(oop base, ptrdiff_t offset, T new_value);
512
513 static inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
514 assert_access_thread_state();
515 return _atomic_xchg_at_func(base, offset, new_value);
516 }
517 };
518
519 template <DecoratorSet decorators, typename T>
520 struct RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>: AllStatic {
521 typedef typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type func_t;
522 static func_t _arraycopy_func;
523
524 static void arraycopy_init(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
525 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
526 size_t length);
527
528 static inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
529 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
530 size_t length) {
531 assert_access_thread_state();
532 return _arraycopy_func(src_obj, src_offset_in_bytes, src_raw,
533 dst_obj, dst_offset_in_bytes, dst_raw,
534 length);
535 }
536 };
537
538 template <DecoratorSet decorators, typename T>
539 struct RuntimeDispatch<decorators, T, BARRIER_CLONE>: AllStatic {
540 typedef typename AccessFunction<decorators, T, BARRIER_CLONE>::type func_t;
541 static func_t _clone_func;
542
543 static void clone_init(oop src, oop dst, size_t size);
544
545 static inline void clone(oop src, oop dst, size_t size) {
546 assert_access_thread_state();
547 _clone_func(src, dst, size);
548 }
549 };
550
551 template <DecoratorSet decorators, typename T>
552 struct RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>: AllStatic {
553 typedef typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type func_t;
554 static func_t _value_copy_func;
555
556 static void value_copy_init(void* src, void* dst, InlineKlass* md, LayoutKind lk);
557
558 static inline void value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
559 _value_copy_func(src, dst, md, lk);
560 }
561 };
562
563 // Initialize the function pointers to point to the resolving function.
564 template <DecoratorSet decorators, typename T>
565 typename AccessFunction<decorators, T, BARRIER_STORE>::type
566 RuntimeDispatch<decorators, T, BARRIER_STORE>::_store_func = &store_init;
567
568 template <DecoratorSet decorators, typename T>
569 typename AccessFunction<decorators, T, BARRIER_STORE_AT>::type
570 RuntimeDispatch<decorators, T, BARRIER_STORE_AT>::_store_at_func = &store_at_init;
571
572 template <DecoratorSet decorators, typename T>
573 typename AccessFunction<decorators, T, BARRIER_LOAD>::type
574 RuntimeDispatch<decorators, T, BARRIER_LOAD>::_load_func = &load_init;
575
576 template <DecoratorSet decorators, typename T>
577 typename AccessFunction<decorators, T, BARRIER_LOAD_AT>::type
578 RuntimeDispatch<decorators, T, BARRIER_LOAD_AT>::_load_at_func = &load_at_init;
579
580 template <DecoratorSet decorators, typename T>
581 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG>::type
582 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG>::_atomic_cmpxchg_func = &atomic_cmpxchg_init;
584 template <DecoratorSet decorators, typename T>
585 typename AccessFunction<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::type
586 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_CMPXCHG_AT>::_atomic_cmpxchg_at_func = &atomic_cmpxchg_at_init;
587
588 template <DecoratorSet decorators, typename T>
589 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG>::type
590 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG>::_atomic_xchg_func = &atomic_xchg_init;
591
592 template <DecoratorSet decorators, typename T>
593 typename AccessFunction<decorators, T, BARRIER_ATOMIC_XCHG_AT>::type
594 RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::_atomic_xchg_at_func = &atomic_xchg_at_init;
595
596 template <DecoratorSet decorators, typename T>
597 typename AccessFunction<decorators, T, BARRIER_ARRAYCOPY>::type
598 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::_arraycopy_func = &arraycopy_init;
599
600 template <DecoratorSet decorators, typename T>
601 typename AccessFunction<decorators, T, BARRIER_CLONE>::type
602 RuntimeDispatch<decorators, T, BARRIER_CLONE>::_clone_func = &clone_init;
603
604 template <DecoratorSet decorators, typename T>
605 typename AccessFunction<decorators, T, BARRIER_VALUE_COPY>::type
606 RuntimeDispatch<decorators, T, BARRIER_VALUE_COPY>::_value_copy_func = &value_copy_init;
607
608 // Step 3: Pre-runtime dispatching.
609 // The PreRuntimeDispatch class is responsible for filtering the barrier strength
610 // decorators. That is, for AS_RAW, it hardwires the accesses without a runtime
611 // dispatch point. Otherwise it goes through a runtime check if hardwiring was
612 // not possible.
613 struct PreRuntimeDispatch: AllStatic {
614 template<DecoratorSet decorators>
615 struct CanHardwireRaw: public std::integral_constant<
616 bool,
617 !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // primitive access
618 !HasDecorator<decorators, INTERNAL_CONVERT_COMPRESSED_OOP>::value || // don't care about compressed oops (oop* address)
619 HasDecorator<decorators, INTERNAL_RT_USE_COMPRESSED_OOPS>::value> // we can infer we use compressed oops (narrowOop* address)
620 {};
621
622 static const DecoratorSet convert_compressed_oops = INTERNAL_RT_USE_COMPRESSED_OOPS | INTERNAL_CONVERT_COMPRESSED_OOP;
623
624 template<DecoratorSet decorators>
625 static bool is_hardwired_primitive() {
626 return !HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value;
627 }
835 inline static typename EnableIf<
836 HasDecorator<decorators, AS_RAW>::value, T>::type
837 atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
838 return atomic_xchg<decorators>(field_addr(base, offset), new_value);
839 }
840
841 template <DecoratorSet decorators, typename T>
842 inline static typename EnableIf<
843 !HasDecorator<decorators, AS_RAW>::value, T>::type
844 atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
845 if (is_hardwired_primitive<decorators>()) {
846 const DecoratorSet expanded_decorators = decorators | AS_RAW;
847 return PreRuntimeDispatch::atomic_xchg<expanded_decorators>(base, offset, new_value);
848 } else {
849 return RuntimeDispatch<decorators, T, BARRIER_ATOMIC_XCHG_AT>::atomic_xchg_at(base, offset, new_value);
850 }
851 }
852
853 template <DecoratorSet decorators, typename T>
854 inline static typename EnableIf<
855 HasDecorator<decorators, AS_RAW>::value && CanHardwireRaw<decorators>::value, void>::type
856 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
857 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
858 size_t length) {
859 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
860 if (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value) {
861 Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw,
862 dst_obj, dst_offset_in_bytes, dst_raw,
863 length);
864 } else {
865 Raw::arraycopy(src_obj, src_offset_in_bytes, src_raw,
866 dst_obj, dst_offset_in_bytes, dst_raw,
867 length);
868 }
869 }
870
871 template <DecoratorSet decorators, typename T>
872 inline static typename EnableIf<
873 HasDecorator<decorators, AS_RAW>::value && !CanHardwireRaw<decorators>::value, void>::type
874 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
875 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
876 size_t length) {
877 if (UseCompressedOops) {
878 const DecoratorSet expanded_decorators = decorators | convert_compressed_oops;
879 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
880 dst_obj, dst_offset_in_bytes, dst_raw,
881 length);
882 } else {
883 const DecoratorSet expanded_decorators = decorators & ~convert_compressed_oops;
884 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
885 dst_obj, dst_offset_in_bytes, dst_raw,
886 length);
887 }
888 }
889
890 template <DecoratorSet decorators, typename T>
891 inline static typename EnableIf<
892 !HasDecorator<decorators, AS_RAW>::value, void>::type
893 arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
894 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
895 size_t length) {
896 if (is_hardwired_primitive<decorators>()) {
897 const DecoratorSet expanded_decorators = decorators | AS_RAW;
898 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
899 dst_obj, dst_offset_in_bytes, dst_raw,
900 length);
901 } else {
902 RuntimeDispatch<decorators, T, BARRIER_ARRAYCOPY>::arraycopy(src_obj, src_offset_in_bytes, src_raw,
903 dst_obj, dst_offset_in_bytes, dst_raw,
904 length);
905 }
906 }
907
908 template <DecoratorSet decorators>
909 inline static typename EnableIf<
910 HasDecorator<decorators, AS_RAW>::value>::type
911 clone(oop src, oop dst, size_t size) {
912 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
913 Raw::clone(src, dst, size);
914 }
915
916 template <DecoratorSet decorators>
917 inline static typename EnableIf<
918 !HasDecorator<decorators, AS_RAW>::value>::type
919 clone(oop src, oop dst, size_t size) {
920 RuntimeDispatch<decorators, oop, BARRIER_CLONE>::clone(src, dst, size);
921 }
922
923 template <DecoratorSet decorators>
924 inline static typename EnableIf<
925 HasDecorator<decorators, AS_RAW>::value>::type
926 value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
927 typedef RawAccessBarrier<decorators & RAW_DECORATOR_MASK> Raw;
928 Raw::value_copy(src, dst, md, lk);
929 }
930
931 template <DecoratorSet decorators>
932 inline static typename EnableIf<
933 !HasDecorator<decorators, AS_RAW>::value>::type
934 value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
935 const DecoratorSet expanded_decorators = decorators;
936 RuntimeDispatch<expanded_decorators, void*, BARRIER_VALUE_COPY>::value_copy(src, dst, md, lk);
937 }
938 };
939
940 // Step 2: Reduce types.
941 // Enforce that for non-oop types, T and P have to be strictly the same.
942 // P is the type of the address and T is the type of the values.
943 // As for oop types, it is allow to send T in {narrowOop, oop} and
944 // P in {narrowOop, oop, HeapWord*}. The following rules apply according to
945 // the subsequent table. (columns are P, rows are T)
946 // | | HeapWord | oop | narrowOop |
947 // | oop | rt-comp | hw-none | hw-comp |
948 // | narrowOop | x | x | hw-none |
949 //
950 // x means not allowed
951 // rt-comp means it must be checked at runtime whether the oop is compressed.
952 // hw-none means it is statically known the oop will not be compressed.
953 // hw-comp means it is statically known the oop will be compressed.
954
955 template <DecoratorSet decorators, typename T>
956 inline void store_reduce_types(T* addr, T value) {
957 PreRuntimeDispatch::store<decorators>(addr, value);
1032
1033 template <DecoratorSet decorators, typename T>
1034 inline T load_reduce_types(T* addr) {
1035 return PreRuntimeDispatch::load<decorators, T>(addr);
1036 }
1037
1038 template <DecoratorSet decorators, typename T>
1039 inline typename OopOrNarrowOop<T>::type load_reduce_types(narrowOop* addr) {
1040 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1041 INTERNAL_RT_USE_COMPRESSED_OOPS;
1042 return PreRuntimeDispatch::load<expanded_decorators, typename OopOrNarrowOop<T>::type>(addr);
1043 }
1044
1045 template <DecoratorSet decorators, typename T>
1046 inline oop load_reduce_types(HeapWord* addr) {
1047 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1048 return PreRuntimeDispatch::load<expanded_decorators, oop>(addr);
1049 }
1050
1051 template <DecoratorSet decorators, typename T>
1052 inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
1053 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1054 size_t length) {
1055 PreRuntimeDispatch::arraycopy<decorators>(src_obj, src_offset_in_bytes, src_raw,
1056 dst_obj, dst_offset_in_bytes, dst_raw,
1057 length);
1058 }
1059
1060 template <DecoratorSet decorators>
1061 inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, HeapWord* src_raw,
1062 arrayOop dst_obj, size_t dst_offset_in_bytes, HeapWord* dst_raw,
1063 size_t length) {
1064 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP;
1065 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1066 dst_obj, dst_offset_in_bytes, dst_raw,
1067 length);
1068 }
1069
1070 template <DecoratorSet decorators>
1071 inline void arraycopy_reduce_types(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
1072 arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
1073 size_t length) {
1074 const DecoratorSet expanded_decorators = decorators | INTERNAL_CONVERT_COMPRESSED_OOP |
1075 INTERNAL_RT_USE_COMPRESSED_OOPS;
1076 PreRuntimeDispatch::arraycopy<expanded_decorators>(src_obj, src_offset_in_bytes, src_raw,
1077 dst_obj, dst_offset_in_bytes, dst_raw,
1078 length);
1079 }
1080
1081 // Step 1: Set default decorators. This step remembers if a type was volatile
1082 // and then sets the MO_RELAXED decorator by default. Otherwise, a default
1083 // memory ordering is set for the access, and the implied decorator rules
1084 // are applied to select sensible defaults for decorators that have not been
1085 // explicitly set. For example, default object referent strength is set to strong.
1086 // This step also decays the types passed in (e.g. getting rid of CV qualifiers
1087 // and references from the types). This step also perform some type verification
1088 // that the passed in types make sense.
1089
1090 template <DecoratorSet decorators, typename T>
1091 static void verify_types(){
1092 // If this fails to compile, then you have sent in something that is
1093 // not recognized as a valid primitive type to a primitive Access function.
1094 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value || // oops have already been validated
1095 (std::is_pointer<T>::value || std::is_integral<T>::value) ||
1096 std::is_floating_point<T>::value)); // not allowed primitive type
1097 }
1098
1191 DecayedT new_decayed_value = new_value;
1192 // atomic_xchg is only available in SEQ_CST flavour.
1193 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST>::value;
1194 return atomic_xchg_reduce_types<expanded_decorators>(const_cast<DecayedP*>(addr),
1195 new_decayed_value);
1196 }
1197
1198 template <DecoratorSet decorators, typename T>
1199 inline T atomic_xchg_at(oop base, ptrdiff_t offset, T new_value) {
1200 verify_types<decorators, T>();
1201 using DecayedT = std::decay_t<T>;
1202 DecayedT new_decayed_value = new_value;
1203 // atomic_xchg is only available in SEQ_CST flavour.
1204 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | MO_SEQ_CST |
1205 (HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ?
1206 INTERNAL_CONVERT_COMPRESSED_OOP : DECORATORS_NONE)>::value;
1207 return PreRuntimeDispatch::atomic_xchg_at<expanded_decorators>(base, offset, new_decayed_value);
1208 }
1209
1210 template <DecoratorSet decorators, typename T>
1211 inline void arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, const T* src_raw,
1212 arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
1213 size_t length) {
1214 STATIC_ASSERT((HasDecorator<decorators, INTERNAL_VALUE_IS_OOP>::value ||
1215 (std::is_same<T, void>::value || std::is_integral<T>::value) ||
1216 std::is_floating_point<T>::value)); // arraycopy allows type erased void elements
1217 using DecayedT = std::decay_t<T>;
1218 const DecoratorSet expanded_decorators = DecoratorFixup<decorators | IS_ARRAY | IN_HEAP>::value;
1219 arraycopy_reduce_types<expanded_decorators>(src_obj, src_offset_in_bytes, const_cast<DecayedT*>(src_raw),
1220 dst_obj, dst_offset_in_bytes, const_cast<DecayedT*>(dst_raw),
1221 length);
1222 }
1223
1224 template <DecoratorSet decorators>
1225 inline void clone(oop src, oop dst, size_t size) {
1226 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1227 PreRuntimeDispatch::clone<expanded_decorators>(src, dst, size);
1228 }
1229
1230 template <DecoratorSet decorators>
1231 inline void value_copy(void* src, void* dst, InlineKlass* md, LayoutKind lk) {
1232 const DecoratorSet expanded_decorators = DecoratorFixup<decorators>::value;
1233 PreRuntimeDispatch::value_copy<expanded_decorators>(src, dst, md, lk);
1234 }
1235
1236 // Infer the type that should be returned from an Access::oop_load.
1237 template <typename P, DecoratorSet decorators>
1238 class OopLoadProxy: public StackObj {
1239 private:
1240 P *const _addr;
1241 public:
1242 explicit OopLoadProxy(P* addr) : _addr(addr) {}
1243
1244 inline operator oop() {
1245 return load<decorators | INTERNAL_VALUE_IS_OOP, P, oop>(_addr);
1246 }
1247
1248 inline operator narrowOop() {
1249 return load<decorators | INTERNAL_VALUE_IS_OOP, P, narrowOop>(_addr);
1250 }
1251
1252 template <typename T>
1253 inline bool operator ==(const T& other) const {
1254 return load<decorators | INTERNAL_VALUE_IS_OOP, P, T>(_addr) == other;
1255 }
|