1 /* 2 * Copyright (c) 2006, 2025, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "runtime/sharedRuntime.hpp" 26 #include "utilities/align.hpp" 27 #include "utilities/byteswap.hpp" 28 #include "utilities/copy.hpp" 29 30 31 // Copy bytes; larger units are filled atomically if everything is aligned. 32 void Copy::conjoint_memory_atomic(const void* from, void* to, size_t size) { 33 uintptr_t bits = (uintptr_t) from | (uintptr_t) to | (uintptr_t) size; 34 35 // (Note: We could improve performance by ignoring the low bits of size, 36 // and putting a short cleanup loop after each bulk copy loop. 37 // There are plenty of other ways to make this faster also, 38 // and it's a slippery slope. For now, let's keep this code simple 39 // since the simplicity helps clarify the atomicity semantics of 40 // this operation. There are also CPU-specific assembly versions 41 // which may or may not want to include such optimizations.) 42 43 if (bits % sizeof(jlong) == 0) { 44 Copy::conjoint_jlongs_atomic((const jlong*) from, (jlong*) to, size / sizeof(jlong)); 45 } else if (bits % sizeof(jint) == 0) { 46 Copy::conjoint_jints_atomic((const jint*) from, (jint*) to, size / sizeof(jint)); 47 } else if (bits % sizeof(jshort) == 0) { 48 Copy::conjoint_jshorts_atomic((const jshort*) from, (jshort*) to, size / sizeof(jshort)); 49 } else { 50 // Not aligned, so no need to be atomic. 51 Copy::conjoint_jbytes((const void*) from, (void*) to, size); 52 } 53 } 54 55 #define COPY_ALIGNED_SEGMENT(t) \ 56 if (bits % sizeof(t) == 0) { \ 57 size_t segment = remain / sizeof(t); \ 58 if (segment > 0) { \ 59 Copy::conjoint_##t##s_atomic((const t*) cursor_from, (t*) cursor_to, segment); \ 60 remain -= segment * sizeof(t); \ 61 cursor_from = (void*)(((char*)cursor_from) + segment * sizeof(t)); \ 62 cursor_to = (void*)(((char*)cursor_to) + segment * sizeof(t)); \ 63 } \ 64 } \ 65 66 void Copy::copy_value_content(const void* from, void* to, size_t size) { 67 // Simple cases first 68 uintptr_t bits = (uintptr_t) from | (uintptr_t) to | (uintptr_t) size; 69 if (bits % sizeof(jlong) == 0) { 70 Copy::conjoint_jlongs_atomic((const jlong*) from, (jlong*) to, size / sizeof(jlong)); 71 return; 72 } else if (bits % sizeof(jint) == 0) { 73 Copy::conjoint_jints_atomic((const jint*) from, (jint*) to, size / sizeof(jint)); 74 return; 75 } else if (bits % sizeof(jshort) == 0) { 76 Copy::conjoint_jshorts_atomic((const jshort*) from, (jshort*) to, size / sizeof(jshort)); 77 return; 78 } 79 80 // Complex cases 81 bits = (uintptr_t) from | (uintptr_t) to; 82 const void* cursor_from = from; 83 void* cursor_to = to; 84 size_t remain = size; 85 COPY_ALIGNED_SEGMENT(jlong) 86 COPY_ALIGNED_SEGMENT(jint) 87 COPY_ALIGNED_SEGMENT(jshort) 88 if (remain > 0) { 89 Copy::conjoint_jbytes((const void*) cursor_from, (void*) cursor_to, remain); 90 } 91 } 92 93 #undef COPY_ALIGNED_SEGMENT 94 95 class CopySwap : AllStatic { 96 public: 97 /** 98 * Copy and optionally byte swap elements 99 * 100 * <swap> - true if elements should be byte swapped 101 * 102 * @param src address of source 103 * @param dst address of destination 104 * @param byte_count number of bytes to copy 105 * @param elem_size size of the elements to copy-swap 106 */ 107 template<bool swap> 108 static void conjoint_swap_if_needed(const void* src, void* dst, size_t byte_count, size_t elem_size) { 109 assert(src != nullptr, "address must not be null"); 110 assert(dst != nullptr, "address must not be null"); 111 assert(elem_size == 2 || elem_size == 4 || elem_size == 8, 112 "incorrect element size: %zu", elem_size); 113 assert(is_aligned(byte_count, elem_size), 114 "byte_count %zu must be multiple of element size %zu", byte_count, elem_size); 115 116 address src_end = (address)src + byte_count; 117 118 if (dst <= src || dst >= src_end) { 119 do_conjoint_swap<RIGHT,swap>(src, dst, byte_count, elem_size); 120 } else { 121 do_conjoint_swap<LEFT,swap>(src, dst, byte_count, elem_size); 122 } 123 } 124 125 private: 126 enum CopyDirection { 127 RIGHT, // lower -> higher address 128 LEFT // higher -> lower address 129 }; 130 131 /** 132 * Copy and byte swap elements 133 * 134 * <T> - type of element to copy 135 * <D> - copy direction 136 * <is_src_aligned> - true if src argument is aligned to element size 137 * <is_dst_aligned> - true if dst argument is aligned to element size 138 * 139 * @param src address of source 140 * @param dst address of destination 141 * @param byte_count number of bytes to copy 142 */ 143 template <typename T, CopyDirection D, bool swap, bool is_src_aligned, bool is_dst_aligned> 144 static void do_conjoint_swap(const void* src, void* dst, size_t byte_count) { 145 const char* cur_src; 146 char* cur_dst; 147 148 switch (D) { 149 case RIGHT: 150 cur_src = (const char*)src; 151 cur_dst = (char*)dst; 152 break; 153 case LEFT: 154 cur_src = (const char*)src + byte_count - sizeof(T); 155 cur_dst = (char*)dst + byte_count - sizeof(T); 156 break; 157 } 158 159 for (size_t i = 0; i < byte_count / sizeof(T); i++) { 160 T tmp; 161 162 if (is_src_aligned) { 163 tmp = *(T*)cur_src; 164 } else { 165 memcpy(&tmp, cur_src, sizeof(T)); 166 } 167 168 if (swap) { 169 tmp = byteswap(tmp); 170 } 171 172 if (is_dst_aligned) { 173 *(T*)cur_dst = tmp; 174 } else { 175 memcpy(cur_dst, &tmp, sizeof(T)); 176 } 177 178 switch (D) { 179 case RIGHT: 180 cur_src += sizeof(T); 181 cur_dst += sizeof(T); 182 break; 183 case LEFT: 184 cur_src -= sizeof(T); 185 cur_dst -= sizeof(T); 186 break; 187 } 188 } 189 } 190 191 /** 192 * Copy and byte swap elements 193 * 194 * <T> - type of element to copy 195 * <D> - copy direction 196 * <swap> - true if elements should be byte swapped 197 * 198 * @param src address of source 199 * @param dst address of destination 200 * @param byte_count number of bytes to copy 201 */ 202 template <typename T, CopyDirection direction, bool swap> 203 static void do_conjoint_swap(const void* src, void* dst, size_t byte_count) { 204 if (is_aligned(src, sizeof(T))) { 205 if (is_aligned(dst, sizeof(T))) { 206 do_conjoint_swap<T,direction,swap,true,true>(src, dst, byte_count); 207 } else { 208 do_conjoint_swap<T,direction,swap,true,false>(src, dst, byte_count); 209 } 210 } else { 211 if (is_aligned(dst, sizeof(T))) { 212 do_conjoint_swap<T,direction,swap,false,true>(src, dst, byte_count); 213 } else { 214 do_conjoint_swap<T,direction,swap,false,false>(src, dst, byte_count); 215 } 216 } 217 } 218 219 220 /** 221 * Copy and byte swap elements 222 * 223 * <D> - copy direction 224 * <swap> - true if elements should be byte swapped 225 * 226 * @param src address of source 227 * @param dst address of destination 228 * @param byte_count number of bytes to copy 229 * @param elem_size size of the elements to copy-swap 230 */ 231 template <CopyDirection D, bool swap> 232 static void do_conjoint_swap(const void* src, void* dst, size_t byte_count, size_t elem_size) { 233 switch (elem_size) { 234 case 2: do_conjoint_swap<uint16_t,D,swap>(src, dst, byte_count); break; 235 case 4: do_conjoint_swap<uint32_t,D,swap>(src, dst, byte_count); break; 236 case 8: do_conjoint_swap<uint64_t,D,swap>(src, dst, byte_count); break; 237 default: guarantee(false, "do_conjoint_swap: Invalid elem_size %zu\n", elem_size); 238 } 239 } 240 }; 241 242 void Copy::conjoint_copy(const void* src, void* dst, size_t byte_count, size_t elem_size) { 243 CopySwap::conjoint_swap_if_needed<false>(src, dst, byte_count, elem_size); 244 } 245 246 void Copy::conjoint_swap(const void* src, void* dst, size_t byte_count, size_t elem_size) { 247 CopySwap::conjoint_swap_if_needed<true>(src, dst, byte_count, elem_size); 248 } 249 250 // Fill bytes; larger units are filled atomically if everything is aligned. 251 void Copy::fill_to_memory_atomic(void* to, size_t size, jubyte value) { 252 address dst = (address)to; 253 uintptr_t bits = (uintptr_t)to | (uintptr_t)size; 254 if (bits % sizeof(jlong) == 0) { 255 jlong fill = (julong)((jubyte)value); // zero-extend 256 if (fill != 0) { 257 fill += fill << 8; 258 fill += fill << 16; 259 fill += fill << 32; 260 } 261 // Copy::fill_to_jlongs_atomic((jlong*) dst, size / sizeof(jlong)); 262 for (uintptr_t off = 0; off < size; off += sizeof(jlong)) { 263 *(jlong*)(dst + off) = fill; 264 } 265 } else if (bits % sizeof(jint) == 0) { 266 jint fill = (juint)((jubyte)value); // zero-extend 267 if (fill != 0) { 268 fill += fill << 8; 269 fill += fill << 16; 270 } 271 // Copy::fill_to_jints_atomic((jint*) dst, size / sizeof(jint)); 272 for (uintptr_t off = 0; off < size; off += sizeof(jint)) { 273 *(jint*)(dst + off) = fill; 274 } 275 } else if (bits % sizeof(jshort) == 0) { 276 jshort fill = (jushort)((jubyte)value); // zero-extend 277 fill += (jshort)(fill << 8); 278 // Copy::fill_to_jshorts_atomic((jshort*) dst, size / sizeof(jshort)); 279 for (uintptr_t off = 0; off < size; off += sizeof(jshort)) { 280 *(jshort*)(dst + off) = fill; 281 } 282 } else { 283 // Not aligned, so no need to be atomic. 284 #ifdef MUSL_LIBC 285 // This code is used by Unsafe and may hit the next page after truncation 286 // of mapped memory. Therefore, we use volatile to prevent compilers from 287 // replacing the loop by memset which may not trigger SIGBUS as needed 288 // (observed on Alpine Linux x86_64) 289 jbyte fill = value; 290 for (uintptr_t off = 0; off < size; off += sizeof(jbyte)) { 291 *(volatile jbyte*)(dst + off) = fill; 292 } 293 #else 294 Copy::fill_to_bytes(dst, size, value); 295 #endif 296 } 297 }