1 /*
  2  * Copyright (c) 2006, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "runtime/sharedRuntime.hpp"
 26 #include "utilities/align.hpp"
 27 #include "utilities/byteswap.hpp"
 28 #include "utilities/copy.hpp"
 29 
 30 
 31 // Copy bytes; larger units are filled atomically if everything is aligned.
 32 void Copy::conjoint_memory_atomic(const void* from, void* to, size_t size) {
 33   uintptr_t bits = (uintptr_t) from | (uintptr_t) to | (uintptr_t) size;
 34 
 35   // (Note:  We could improve performance by ignoring the low bits of size,
 36   // and putting a short cleanup loop after each bulk copy loop.
 37   // There are plenty of other ways to make this faster also,
 38   // and it's a slippery slope.  For now, let's keep this code simple
 39   // since the simplicity helps clarify the atomicity semantics of
 40   // this operation.  There are also CPU-specific assembly versions
 41   // which may or may not want to include such optimizations.)
 42 
 43   if (bits % sizeof(jlong) == 0) {
 44     Copy::conjoint_jlongs_atomic((const jlong*) from, (jlong*) to, size / sizeof(jlong));
 45   } else if (bits % sizeof(jint) == 0) {
 46     Copy::conjoint_jints_atomic((const jint*) from, (jint*) to, size / sizeof(jint));
 47   } else if (bits % sizeof(jshort) == 0) {
 48     Copy::conjoint_jshorts_atomic((const jshort*) from, (jshort*) to, size / sizeof(jshort));
 49   } else {
 50     // Not aligned, so no need to be atomic.
 51     Copy::conjoint_jbytes((const void*) from, (void*) to, size);
 52   }
 53 }
 54 
 55 #define COPY_ALIGNED_SEGMENT(t) \
 56   if (bits % sizeof(t) == 0) { \
 57     size_t segment = remain / sizeof(t); \
 58     if (segment > 0) { \
 59       Copy::conjoint_##t##s_atomic((const t*) cursor_from, (t*) cursor_to, segment); \
 60       remain -= segment * sizeof(t); \
 61       cursor_from = (void*)(((char*)cursor_from) + segment * sizeof(t)); \
 62       cursor_to = (void*)(((char*)cursor_to) + segment * sizeof(t)); \
 63     } \
 64   } \
 65 
 66 void Copy::copy_value_content(const void* from, void* to, size_t size) {
 67   // Simple cases first
 68   uintptr_t bits = (uintptr_t) from | (uintptr_t) to | (uintptr_t) size;
 69   if (bits % sizeof(jlong) == 0) {
 70     Copy::conjoint_jlongs_atomic((const jlong*) from, (jlong*) to, size / sizeof(jlong));
 71     return;
 72   } else if (bits % sizeof(jint) == 0) {
 73     Copy::conjoint_jints_atomic((const jint*) from, (jint*) to, size / sizeof(jint));
 74     return;
 75   } else if (bits % sizeof(jshort) == 0) {
 76     Copy::conjoint_jshorts_atomic((const jshort*) from, (jshort*) to, size / sizeof(jshort));
 77     return;
 78   }
 79 
 80   // Complex cases
 81   bits = (uintptr_t) from | (uintptr_t) to;
 82   const void* cursor_from = from;
 83   void* cursor_to = to;
 84   size_t remain = size;
 85   COPY_ALIGNED_SEGMENT(jlong)
 86   COPY_ALIGNED_SEGMENT(jint)
 87   COPY_ALIGNED_SEGMENT(jshort)
 88   if (remain > 0) {
 89     Copy::conjoint_jbytes((const void*) cursor_from, (void*) cursor_to, remain);
 90   }
 91 }
 92 
 93 #undef COPY_ALIGNED_SEGMENT
 94 
 95 class CopySwap : AllStatic {
 96 public:
 97   /**
 98    * Copy and optionally byte swap elements
 99    *
100    * <swap> - true if elements should be byte swapped
101    *
102    * @param src address of source
103    * @param dst address of destination
104    * @param byte_count number of bytes to copy
105    * @param elem_size size of the elements to copy-swap
106    */
107   template<bool swap>
108   static void conjoint_swap_if_needed(const void* src, void* dst, size_t byte_count, size_t elem_size) {
109     assert(src != nullptr, "address must not be null");
110     assert(dst != nullptr, "address must not be null");
111     assert(elem_size == 2 || elem_size == 4 || elem_size == 8,
112            "incorrect element size: %zu", elem_size);
113     assert(is_aligned(byte_count, elem_size),
114            "byte_count %zu must be multiple of element size %zu", byte_count, elem_size);
115 
116     address src_end = (address)src + byte_count;
117 
118     if (dst <= src || dst >= src_end) {
119       do_conjoint_swap<RIGHT,swap>(src, dst, byte_count, elem_size);
120     } else {
121       do_conjoint_swap<LEFT,swap>(src, dst, byte_count, elem_size);
122     }
123   }
124 
125 private:
126   enum CopyDirection {
127     RIGHT, // lower -> higher address
128     LEFT   // higher -> lower address
129   };
130 
131   /**
132    * Copy and byte swap elements
133    *
134    * <T> - type of element to copy
135    * <D> - copy direction
136    * <is_src_aligned> - true if src argument is aligned to element size
137    * <is_dst_aligned> - true if dst argument is aligned to element size
138    *
139    * @param src address of source
140    * @param dst address of destination
141    * @param byte_count number of bytes to copy
142    */
143   template <typename T, CopyDirection D, bool swap, bool is_src_aligned, bool is_dst_aligned>
144   static void do_conjoint_swap(const void* src, void* dst, size_t byte_count) {
145     const char* cur_src;
146     char* cur_dst;
147 
148     switch (D) {
149     case RIGHT:
150       cur_src = (const char*)src;
151       cur_dst = (char*)dst;
152       break;
153     case LEFT:
154       cur_src = (const char*)src + byte_count - sizeof(T);
155       cur_dst = (char*)dst + byte_count - sizeof(T);
156       break;
157     }
158 
159     for (size_t i = 0; i < byte_count / sizeof(T); i++) {
160       T tmp;
161 
162       if (is_src_aligned) {
163         tmp = *(T*)cur_src;
164       } else {
165         memcpy(&tmp, cur_src, sizeof(T));
166       }
167 
168       if (swap) {
169         tmp = byteswap(tmp);
170       }
171 
172       if (is_dst_aligned) {
173         *(T*)cur_dst = tmp;
174       } else {
175         memcpy(cur_dst, &tmp, sizeof(T));
176       }
177 
178       switch (D) {
179       case RIGHT:
180         cur_src += sizeof(T);
181         cur_dst += sizeof(T);
182         break;
183       case LEFT:
184         cur_src -= sizeof(T);
185         cur_dst -= sizeof(T);
186         break;
187       }
188     }
189   }
190 
191   /**
192    * Copy and byte swap elements
193    *
194    * <T>    - type of element to copy
195    * <D>    - copy direction
196    * <swap> - true if elements should be byte swapped
197    *
198    * @param src address of source
199    * @param dst address of destination
200    * @param byte_count number of bytes to copy
201    */
202   template <typename T, CopyDirection direction, bool swap>
203   static void do_conjoint_swap(const void* src, void* dst, size_t byte_count) {
204     if (is_aligned(src, sizeof(T))) {
205       if (is_aligned(dst, sizeof(T))) {
206         do_conjoint_swap<T,direction,swap,true,true>(src, dst, byte_count);
207       } else {
208         do_conjoint_swap<T,direction,swap,true,false>(src, dst, byte_count);
209       }
210     } else {
211       if (is_aligned(dst, sizeof(T))) {
212         do_conjoint_swap<T,direction,swap,false,true>(src, dst, byte_count);
213       } else {
214         do_conjoint_swap<T,direction,swap,false,false>(src, dst, byte_count);
215       }
216     }
217   }
218 
219 
220   /**
221    * Copy and byte swap elements
222    *
223    * <D>    - copy direction
224    * <swap> - true if elements should be byte swapped
225    *
226    * @param src address of source
227    * @param dst address of destination
228    * @param byte_count number of bytes to copy
229    * @param elem_size size of the elements to copy-swap
230    */
231   template <CopyDirection D, bool swap>
232   static void do_conjoint_swap(const void* src, void* dst, size_t byte_count, size_t elem_size) {
233     switch (elem_size) {
234     case 2: do_conjoint_swap<uint16_t,D,swap>(src, dst, byte_count); break;
235     case 4: do_conjoint_swap<uint32_t,D,swap>(src, dst, byte_count); break;
236     case 8: do_conjoint_swap<uint64_t,D,swap>(src, dst, byte_count); break;
237     default: guarantee(false, "do_conjoint_swap: Invalid elem_size %zu\n", elem_size);
238     }
239   }
240 };
241 
242 void Copy::conjoint_copy(const void* src, void* dst, size_t byte_count, size_t elem_size) {
243   CopySwap::conjoint_swap_if_needed<false>(src, dst, byte_count, elem_size);
244 }
245 
246 void Copy::conjoint_swap(const void* src, void* dst, size_t byte_count, size_t elem_size) {
247   CopySwap::conjoint_swap_if_needed<true>(src, dst, byte_count, elem_size);
248 }
249 
250 // Fill bytes; larger units are filled atomically if everything is aligned.
251 void Copy::fill_to_memory_atomic(void* to, size_t size, jubyte value) {
252   address dst = (address)to;
253   uintptr_t bits = (uintptr_t)to | (uintptr_t)size;
254   if (bits % sizeof(jlong) == 0) {
255     jlong fill = (julong)((jubyte)value);  // zero-extend
256     if (fill != 0) {
257       fill += fill << 8;
258       fill += fill << 16;
259       fill += fill << 32;
260     }
261     // Copy::fill_to_jlongs_atomic((jlong*) dst, size / sizeof(jlong));
262     for (uintptr_t off = 0; off < size; off += sizeof(jlong)) {
263       *(jlong*)(dst + off) = fill;
264     }
265   } else if (bits % sizeof(jint) == 0) {
266     jint fill = (juint)((jubyte)value);  // zero-extend
267     if (fill != 0) {
268       fill += fill << 8;
269       fill += fill << 16;
270     }
271     // Copy::fill_to_jints_atomic((jint*) dst, size / sizeof(jint));
272     for (uintptr_t off = 0; off < size; off += sizeof(jint)) {
273       *(jint*)(dst + off) = fill;
274     }
275   } else if (bits % sizeof(jshort) == 0) {
276     jshort fill = (jushort)((jubyte)value);  // zero-extend
277     fill += (jshort)(fill << 8);
278     // Copy::fill_to_jshorts_atomic((jshort*) dst, size / sizeof(jshort));
279     for (uintptr_t off = 0; off < size; off += sizeof(jshort)) {
280       *(jshort*)(dst + off) = fill;
281     }
282   } else {
283     // Not aligned, so no need to be atomic.
284 #ifdef MUSL_LIBC
285     // This code is used by Unsafe and may hit the next page after truncation
286     // of mapped memory. Therefore, we use volatile to prevent compilers from
287     // replacing the loop by memset which may not trigger SIGBUS as needed
288     // (observed on Alpine Linux x86_64)
289     jbyte fill = value;
290     for (uintptr_t off = 0; off < size; off += sizeof(jbyte)) {
291       *(volatile jbyte*)(dst + off) = fill;
292     }
293 #else
294     Copy::fill_to_bytes(dst, size, value);
295 #endif
296   }
297 }