1 /*
2 * Copyright (c) 2006, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "runtime/sharedRuntime.hpp"
26 #include "utilities/align.hpp"
27 #include "utilities/byteswap.hpp"
28 #include "utilities/copy.hpp"
29
30
31 // Copy bytes; larger units are filled atomically if everything is aligned.
32 void Copy::conjoint_memory_atomic(const void* from, void* to, size_t size) {
33 uintptr_t bits = (uintptr_t) from | (uintptr_t) to | (uintptr_t) size;
34
35 // (Note: We could improve performance by ignoring the low bits of size,
36 // and putting a short cleanup loop after each bulk copy loop.
37 // There are plenty of other ways to make this faster also,
38 // and it's a slippery slope. For now, let's keep this code simple
39 // since the simplicity helps clarify the atomicity semantics of
40 // this operation. There are also CPU-specific assembly versions
41 // which may or may not want to include such optimizations.)
42
43 if (bits % sizeof(jlong) == 0) {
44 Copy::conjoint_jlongs_atomic((const jlong*) from, (jlong*) to, size / sizeof(jlong));
45 } else if (bits % sizeof(jint) == 0) {
46 Copy::conjoint_jints_atomic((const jint*) from, (jint*) to, size / sizeof(jint));
47 } else if (bits % sizeof(jshort) == 0) {
48 Copy::conjoint_jshorts_atomic((const jshort*) from, (jshort*) to, size / sizeof(jshort));
49 } else {
50 // Not aligned, so no need to be atomic.
51 Copy::conjoint_jbytes((const void*) from, (void*) to, size);
52 }
53 }
54
55 #define COPY_ALIGNED_SEGMENT(t) \
56 if (bits % sizeof(t) == 0) { \
57 size_t segment = remain / sizeof(t); \
58 if (segment > 0) { \
59 Copy::conjoint_##t##s_atomic((const t*) cursor_from, (t*) cursor_to, segment); \
60 remain -= segment * sizeof(t); \
61 cursor_from = (void*)(((char*)cursor_from) + segment * sizeof(t)); \
62 cursor_to = (void*)(((char*)cursor_to) + segment * sizeof(t)); \
63 } \
64 } \
65
66 void Copy::copy_value_content(const void* from, void* to, size_t size) {
67 // Simple cases first
68 uintptr_t bits = (uintptr_t) from | (uintptr_t) to | (uintptr_t) size;
69 if (bits % sizeof(jlong) == 0) {
70 Copy::conjoint_jlongs_atomic((const jlong*) from, (jlong*) to, size / sizeof(jlong));
71 return;
72 } else if (bits % sizeof(jint) == 0) {
73 Copy::conjoint_jints_atomic((const jint*) from, (jint*) to, size / sizeof(jint));
74 return;
75 } else if (bits % sizeof(jshort) == 0) {
76 Copy::conjoint_jshorts_atomic((const jshort*) from, (jshort*) to, size / sizeof(jshort));
77 return;
78 }
79
80 // Complex cases
81 bits = (uintptr_t) from | (uintptr_t) to;
82 const void* cursor_from = from;
83 void* cursor_to = to;
84 size_t remain = size;
85 COPY_ALIGNED_SEGMENT(jlong)
86 COPY_ALIGNED_SEGMENT(jint)
87 COPY_ALIGNED_SEGMENT(jshort)
88 if (remain > 0) {
89 Copy::conjoint_jbytes((const void*) cursor_from, (void*) cursor_to, remain);
90 }
91 }
92
93 #undef COPY_ALIGNED_SEGMENT
94
95 class CopySwap : AllStatic {
96 public:
97 /**
98 * Copy and optionally byte swap elements
99 *
100 * <swap> - true if elements should be byte swapped
101 *
102 * @param src address of source
103 * @param dst address of destination
104 * @param byte_count number of bytes to copy
105 * @param elem_size size of the elements to copy-swap
106 */
107 template<bool swap>
108 static void conjoint_swap_if_needed(const void* src, void* dst, size_t byte_count, size_t elem_size) {
109 assert(src != nullptr, "address must not be null");
110 assert(dst != nullptr, "address must not be null");
111 assert(elem_size == 2 || elem_size == 4 || elem_size == 8,
112 "incorrect element size: %zu", elem_size);
113 assert(is_aligned(byte_count, elem_size),
114 "byte_count %zu must be multiple of element size %zu", byte_count, elem_size);
115
116 address src_end = (address)src + byte_count;
117
118 if (dst <= src || dst >= src_end) {
119 do_conjoint_swap<RIGHT,swap>(src, dst, byte_count, elem_size);
120 } else {
121 do_conjoint_swap<LEFT,swap>(src, dst, byte_count, elem_size);
122 }
123 }
124
125 private:
126 enum CopyDirection {
127 RIGHT, // lower -> higher address
128 LEFT // higher -> lower address
129 };
130
131 /**
132 * Copy and byte swap elements
133 *
134 * <T> - type of element to copy
135 * <D> - copy direction
136 * <is_src_aligned> - true if src argument is aligned to element size
137 * <is_dst_aligned> - true if dst argument is aligned to element size
138 *
139 * @param src address of source
140 * @param dst address of destination
141 * @param byte_count number of bytes to copy
142 */
143 template <typename T, CopyDirection D, bool swap, bool is_src_aligned, bool is_dst_aligned>
144 static void do_conjoint_swap(const void* src, void* dst, size_t byte_count) {
145 const char* cur_src;
146 char* cur_dst;
147
148 switch (D) {
149 case RIGHT:
150 cur_src = (const char*)src;
151 cur_dst = (char*)dst;
152 break;
153 case LEFT:
154 cur_src = (const char*)src + byte_count - sizeof(T);
155 cur_dst = (char*)dst + byte_count - sizeof(T);
156 break;
157 }
158
159 for (size_t i = 0; i < byte_count / sizeof(T); i++) {
160 T tmp;
161
162 if (is_src_aligned) {
163 tmp = *(T*)cur_src;
164 } else {
165 memcpy(&tmp, cur_src, sizeof(T));
166 }
167
168 if (swap) {
169 tmp = byteswap(tmp);
170 }
171
172 if (is_dst_aligned) {
173 *(T*)cur_dst = tmp;
174 } else {
175 memcpy(cur_dst, &tmp, sizeof(T));
176 }
177
178 switch (D) {
179 case RIGHT:
180 cur_src += sizeof(T);
181 cur_dst += sizeof(T);
182 break;
183 case LEFT:
184 cur_src -= sizeof(T);
185 cur_dst -= sizeof(T);
186 break;
187 }
188 }
189 }
190
191 /**
192 * Copy and byte swap elements
193 *
194 * <T> - type of element to copy
195 * <D> - copy direction
196 * <swap> - true if elements should be byte swapped
197 *
198 * @param src address of source
199 * @param dst address of destination
200 * @param byte_count number of bytes to copy
201 */
202 template <typename T, CopyDirection direction, bool swap>
203 static void do_conjoint_swap(const void* src, void* dst, size_t byte_count) {
204 if (is_aligned(src, sizeof(T))) {
205 if (is_aligned(dst, sizeof(T))) {
206 do_conjoint_swap<T,direction,swap,true,true>(src, dst, byte_count);
207 } else {
208 do_conjoint_swap<T,direction,swap,true,false>(src, dst, byte_count);
209 }
210 } else {
211 if (is_aligned(dst, sizeof(T))) {
212 do_conjoint_swap<T,direction,swap,false,true>(src, dst, byte_count);
213 } else {
214 do_conjoint_swap<T,direction,swap,false,false>(src, dst, byte_count);
215 }
216 }
217 }
218
219
220 /**
221 * Copy and byte swap elements
222 *
223 * <D> - copy direction
224 * <swap> - true if elements should be byte swapped
225 *
226 * @param src address of source
227 * @param dst address of destination
228 * @param byte_count number of bytes to copy
229 * @param elem_size size of the elements to copy-swap
230 */
231 template <CopyDirection D, bool swap>
232 static void do_conjoint_swap(const void* src, void* dst, size_t byte_count, size_t elem_size) {
233 switch (elem_size) {
234 case 2: do_conjoint_swap<uint16_t,D,swap>(src, dst, byte_count); break;
235 case 4: do_conjoint_swap<uint32_t,D,swap>(src, dst, byte_count); break;
236 case 8: do_conjoint_swap<uint64_t,D,swap>(src, dst, byte_count); break;
237 default: guarantee(false, "do_conjoint_swap: Invalid elem_size %zu\n", elem_size);
238 }
239 }
240 };
241
242 void Copy::conjoint_copy(const void* src, void* dst, size_t byte_count, size_t elem_size) {
243 CopySwap::conjoint_swap_if_needed<false>(src, dst, byte_count, elem_size);
244 }
245
246 void Copy::conjoint_swap(const void* src, void* dst, size_t byte_count, size_t elem_size) {
247 CopySwap::conjoint_swap_if_needed<true>(src, dst, byte_count, elem_size);
248 }
249
250 // Fill bytes; larger units are filled atomically if everything is aligned.
251 void Copy::fill_to_memory_atomic(void* to, size_t size, jubyte value) {
252 address dst = (address)to;
253 uintptr_t bits = (uintptr_t)to | (uintptr_t)size;
254 if (bits % sizeof(jlong) == 0) {
255 jlong fill = (julong)((jubyte)value); // zero-extend
256 if (fill != 0) {
257 fill += fill << 8;
258 fill += fill << 16;
259 fill += fill << 32;
260 }
261 // Copy::fill_to_jlongs_atomic((jlong*) dst, size / sizeof(jlong));
262 for (uintptr_t off = 0; off < size; off += sizeof(jlong)) {
263 *(jlong*)(dst + off) = fill;
264 }
265 } else if (bits % sizeof(jint) == 0) {
266 jint fill = (juint)((jubyte)value); // zero-extend
267 if (fill != 0) {
268 fill += fill << 8;
269 fill += fill << 16;
270 }
271 // Copy::fill_to_jints_atomic((jint*) dst, size / sizeof(jint));
272 for (uintptr_t off = 0; off < size; off += sizeof(jint)) {
273 *(jint*)(dst + off) = fill;
274 }
275 } else if (bits % sizeof(jshort) == 0) {
276 jshort fill = (jushort)((jubyte)value); // zero-extend
277 fill += (jshort)(fill << 8);
278 // Copy::fill_to_jshorts_atomic((jshort*) dst, size / sizeof(jshort));
279 for (uintptr_t off = 0; off < size; off += sizeof(jshort)) {
280 *(jshort*)(dst + off) = fill;
281 }
282 } else {
283 // Not aligned, so no need to be atomic.
284 #ifdef MUSL_LIBC
285 // This code is used by Unsafe and may hit the next page after truncation
286 // of mapped memory. Therefore, we use volatile to prevent compilers from
287 // replacing the loop by memset which may not trigger SIGBUS as needed
288 // (observed on Alpine Linux x86_64)
289 jbyte fill = value;
290 for (uintptr_t off = 0; off < size; off += sizeof(jbyte)) {
291 *(volatile jbyte*)(dst + off) = fill;
292 }
293 #else
294 Copy::fill_to_bytes(dst, size, value);
295 #endif
296 }
297 }