1 /*
  2  * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  4  * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
  5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  6  *
  7  * This code is free software; you can redistribute it and/or modify it
  8  * under the terms of the GNU General Public License version 2 only, as
  9  * published by the Free Software Foundation.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  *
 25  */
 26 
 27 // no precompiled headers
 28 #include "asm/macroAssembler.hpp"
 29 #include "classfile/classLoader.hpp"
 30 #include "classfile/vmSymbols.hpp"
 31 #include "code/codeCache.hpp"
 32 #include "code/vtableStubs.hpp"
 33 #include "interpreter/interpreter.hpp"
 34 #include "jvm.h"
 35 #include "logging/log.hpp"
 36 #include "memory/allocation.inline.hpp"
 37 #include "os_bsd.hpp"
 38 #include "os_posix.hpp"
 39 #include "prims/jniFastGetField.hpp"
 40 #include "prims/jvm_misc.hpp"
 41 #include "runtime/arguments.hpp"
 42 #include "runtime/frame.inline.hpp"
 43 #include "runtime/interfaceSupport.inline.hpp"
 44 #include "runtime/java.hpp"
 45 #include "runtime/javaCalls.hpp"
 46 #include "runtime/javaThread.hpp"
 47 #include "runtime/mutexLocker.hpp"
 48 #include "runtime/osThread.hpp"
 49 #include "runtime/safepointMechanism.hpp"
 50 #include "runtime/sharedRuntime.hpp"
 51 #include "runtime/stubRoutines.hpp"
 52 #include "runtime/timer.hpp"
 53 #include "signals_posix.hpp"
 54 #include "utilities/align.hpp"
 55 #include "utilities/events.hpp"
 56 #include "utilities/vmError.hpp"
 57 
 58 // put OS-includes here
 59 # include <sys/types.h>
 60 # include <sys/mman.h>
 61 # include <pthread.h>
 62 # include <signal.h>
 63 # include <errno.h>
 64 # include <dlfcn.h>
 65 # include <stdlib.h>
 66 # include <stdio.h>
 67 # include <unistd.h>
 68 # include <sys/resource.h>
 69 # include <sys/stat.h>
 70 # include <sys/time.h>
 71 # include <sys/utsname.h>
 72 # include <sys/socket.h>
 73 # include <sys/wait.h>
 74 # include <pwd.h>
 75 # include <poll.h>
 76 #ifndef __OpenBSD__
 77 # include <ucontext.h>
 78 #endif
 79 
 80 #if !defined(__APPLE__) && !defined(__NetBSD__)
 81 # include <pthread_np.h>
 82 #endif
 83 
 84 #define SPELL_REG_SP "sp"
 85 #define SPELL_REG_FP "fp"
 86 
 87 #ifdef __APPLE__
 88 // see darwin-xnu/osfmk/mach/arm/_structs.h
 89 
 90 // 10.5 UNIX03 member name prefixes
 91 #define DU3_PREFIX(s, m) __ ## s.__ ## m
 92 #endif
 93 
 94 #define context_x    uc_mcontext->DU3_PREFIX(ss,x)
 95 #define context_fp   uc_mcontext->DU3_PREFIX(ss,fp)
 96 #define context_lr   uc_mcontext->DU3_PREFIX(ss,lr)
 97 #define context_sp   uc_mcontext->DU3_PREFIX(ss,sp)
 98 #define context_pc   uc_mcontext->DU3_PREFIX(ss,pc)
 99 #define context_cpsr uc_mcontext->DU3_PREFIX(ss,cpsr)
100 #define context_esr  uc_mcontext->DU3_PREFIX(es,esr)
101 
102 address os::current_stack_pointer() {
103 #if defined(__clang__) || defined(__llvm__)
104   void *sp;
105   __asm__("mov %0, " SPELL_REG_SP : "=r"(sp));
106   return (address) sp;
107 #else
108   register void *sp __asm__ (SPELL_REG_SP);
109   return (address) sp;
110 #endif
111 }
112 
113 char* os::non_memory_address_word() {
114   // Must never look like an address returned by reserve_memory,
115   // even in its subfields (as defined by the CPU immediate fields,
116   // if the CPU splits constants across multiple instructions).
117 
118   // the return value used in computation of Universe::non_oop_word(), which
119   // is loaded by cpu/aarch64 by MacroAssembler::movptr(Register, uintptr_t)
120   return (char*) 0xffffffffffff;
121 }
122 
123 address os::Posix::ucontext_get_pc(const ucontext_t * uc) {
124   return (address)uc->context_pc;
125 }
126 
127 void os::Posix::ucontext_set_pc(ucontext_t * uc, address pc) {
128   uc->context_pc = (intptr_t)pc ;
129 }
130 
131 intptr_t* os::Bsd::ucontext_get_sp(const ucontext_t * uc) {
132   return (intptr_t*)uc->context_sp;
133 }
134 
135 intptr_t* os::Bsd::ucontext_get_fp(const ucontext_t * uc) {
136   return (intptr_t*)uc->context_fp;
137 }
138 
139 address os::fetch_frame_from_context(const void* ucVoid,
140                     intptr_t** ret_sp, intptr_t** ret_fp) {
141 
142   address epc;
143   const ucontext_t* uc = (const ucontext_t*)ucVoid;
144 
145   if (uc != nullptr) {
146     epc = os::Posix::ucontext_get_pc(uc);
147     if (ret_sp) *ret_sp = os::Bsd::ucontext_get_sp(uc);
148     if (ret_fp) *ret_fp = os::Bsd::ucontext_get_fp(uc);
149   } else {
150     epc = nullptr;
151     if (ret_sp) *ret_sp = (intptr_t *)nullptr;
152     if (ret_fp) *ret_fp = (intptr_t *)nullptr;
153   }
154 
155   return epc;
156 }
157 
158 frame os::fetch_frame_from_context(const void* ucVoid) {
159   intptr_t* sp;
160   intptr_t* fp;
161   address epc = fetch_frame_from_context(ucVoid, &sp, &fp);
162   return frame(sp, fp, epc);
163 }
164 
165 frame os::fetch_compiled_frame_from_context(const void* ucVoid) {
166   const ucontext_t* uc = (const ucontext_t*)ucVoid;
167   // In compiled code, the stack banging is performed before LR
168   // has been saved in the frame.  LR is live, and SP and FP
169   // belong to the caller.
170   intptr_t* fp = os::Bsd::ucontext_get_fp(uc);
171   intptr_t* sp = os::Bsd::ucontext_get_sp(uc);
172   address pc = (address)(uc->context_lr
173                          - NativeInstruction::instruction_size);
174   return frame(sp, fp, pc);
175 }
176 
177 // JVM compiled with -fno-omit-frame-pointer, so RFP is saved on the stack.
178 frame os::get_sender_for_C_frame(frame* fr) {
179   return frame(fr->sender_sp(), fr->link(), fr->sender_pc());
180 }
181 
182 NOINLINE frame os::current_frame() {
183   intptr_t *fp = *(intptr_t **)__builtin_frame_address(0);
184   frame myframe((intptr_t*)os::current_stack_pointer(),
185                 (intptr_t*)fp,
186                 CAST_FROM_FN_PTR(address, os::current_frame));
187   if (os::is_first_C_frame(&myframe)) {
188     // stack is not walkable
189     return frame();
190   } else {
191     return os::get_sender_for_C_frame(&myframe);
192   }
193 }
194 
195 bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
196                                              ucontext_t* uc, JavaThread* thread) {
197   // Enable WXWrite: this function is called by the signal handler at arbitrary
198   // point of execution.
199   ThreadWXEnable wx(WXWrite, thread);
200 
201   // decide if this trap can be handled by a stub
202   address stub = nullptr;
203 
204   address pc          = nullptr;
205 
206   //%note os_trap_1
207   if (info != nullptr && uc != nullptr && thread != nullptr) {
208     pc = (address) os::Posix::ucontext_get_pc(uc);
209 
210     // Handle ALL stack overflow variations here
211     if (sig == SIGSEGV || sig == SIGBUS) {
212       address addr = (address) info->si_addr;
213 
214       // Make sure the high order byte is sign extended, as it may be masked away by the hardware.
215       if ((uintptr_t(addr) & (uintptr_t(1) << 55)) != 0) {
216         addr = address(uintptr_t(addr) | (uintptr_t(0xFF) << 56));
217       }
218 
219       // check if fault address is within thread stack
220       if (thread->is_in_full_stack(addr)) {
221         // stack overflow
222         if (os::Posix::handle_stack_overflow(thread, addr, pc, uc, &stub)) {
223           return true; // continue
224         }
225       }
226     }
227 
228     // We test if stub is already set (by the stack overflow code
229     // above) so it is not overwritten by the code that follows. This
230     // check is not required on other platforms, because on other
231     // platforms we check for SIGSEGV only or SIGBUS only, where here
232     // we have to check for both SIGSEGV and SIGBUS.
233     if (thread->thread_state() == _thread_in_Java && stub == nullptr) {
234       // Java thread running in Java code => find exception handler if any
235       // a fault inside compiled code, the interpreter, or a stub
236 
237       // Handle signal from NativeJump::patch_verified_entry().
238       if ((sig == SIGILL)
239           && nativeInstruction_at(pc)->is_sigill_not_entrant()) {
240         if (TraceTraps) {
241           tty->print_cr("trap: not_entrant");
242         }
243         stub = SharedRuntime::get_handle_wrong_method_stub();
244       } else if ((sig == SIGSEGV || sig == SIGBUS) && SafepointMechanism::is_poll_address((address)info->si_addr)) {
245         stub = SharedRuntime::get_poll_stub(pc);
246 #if defined(__APPLE__)
247       // 32-bit Darwin reports a SIGBUS for nearly all memory access exceptions.
248       // 64-bit Darwin may also use a SIGBUS (seen with compressed oops).
249       // Catching SIGBUS here prevents the implicit SIGBUS null check below from
250       // being called, so only do so if the implicit null check is not necessary.
251       } else if (sig == SIGBUS && !MacroAssembler::uses_implicit_null_check(info->si_addr)) {
252 #else
253       } else if (sig == SIGBUS /* && info->si_code == BUS_OBJERR */) {
254 #endif
255         // BugId 4454115: A read from a MappedByteBuffer can fault
256         // here if the underlying file has been truncated.
257         // Do not crash the VM in such a case.
258         CodeBlob* cb = CodeCache::find_blob(pc);
259         nmethod* nm = (cb != nullptr) ? cb->as_nmethod_or_null() : nullptr;
260         bool is_unsafe_arraycopy = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc));
261         if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_arraycopy) {
262           address next_pc = pc + NativeCall::instruction_size;
263           if (is_unsafe_arraycopy) {
264             next_pc = UnsafeCopyMemory::page_error_continue_pc(pc);
265           }
266           stub = SharedRuntime::handle_unsafe_access(thread, next_pc);
267         }
268       } else if (sig == SIGILL && nativeInstruction_at(pc)->is_stop()) {
269         // Pull a pointer to the error message out of the instruction
270         // stream.
271         const uint64_t *detail_msg_ptr
272           = (uint64_t*)(pc + NativeInstruction::instruction_size);
273         const char *detail_msg = (const char *)*detail_msg_ptr;
274         const char *msg = "stop";
275         if (TraceTraps) {
276           tty->print_cr("trap: %s: (SIGILL)", msg);
277         }
278 
279         // End life with a fatal error, message and detail message and the context.
280         // Note: no need to do any post-processing here (e.g. signal chaining)
281         VMError::report_and_die(thread, uc, nullptr, 0, msg, "%s", detail_msg);
282         ShouldNotReachHere();
283 
284       } else if (sig == SIGFPE &&
285           (info->si_code == FPE_INTDIV || info->si_code == FPE_FLTDIV)) {
286         stub =
287           SharedRuntime::
288           continuation_for_implicit_exception(thread,
289                                               pc,
290                                               SharedRuntime::
291                                               IMPLICIT_DIVIDE_BY_ZERO);
292       } else if ((sig == SIGSEGV || sig == SIGBUS) &&
293                  MacroAssembler::uses_implicit_null_check(info->si_addr)) {
294           // Determination of interpreter/vtable stub/compiled code null exception
295           stub = SharedRuntime::continuation_for_implicit_exception(thread, pc, SharedRuntime::IMPLICIT_NULL);
296       }
297     } else if ((thread->thread_state() == _thread_in_vm ||
298                  thread->thread_state() == _thread_in_native) &&
299                sig == SIGBUS && /* info->si_code == BUS_OBJERR && */
300                thread->doing_unsafe_access()) {
301       address next_pc = pc + NativeCall::instruction_size;
302       if (UnsafeCopyMemory::contains_pc(pc)) {
303         next_pc = UnsafeCopyMemory::page_error_continue_pc(pc);
304       }
305       stub = SharedRuntime::handle_unsafe_access(thread, next_pc);
306     }
307 
308     // jni_fast_Get<Primitive>Field can trap at certain pc's if a GC kicks in
309     // and the heap gets shrunk before the field access.
310     if ((sig == SIGSEGV) || (sig == SIGBUS)) {
311       address addr = JNI_FastGetField::find_slowcase_pc(pc);
312       if (addr != (address)-1) {
313         stub = addr;
314       }
315     }
316   }
317 
318   if (stub != nullptr) {
319     // save all thread context in case we need to restore it
320     if (thread != nullptr) thread->set_saved_exception_pc(pc);
321 
322     os::Posix::ucontext_set_pc(uc, stub);
323     return true;
324   }
325 
326   return false; // Mute compiler
327 }
328 
329 void os::Bsd::init_thread_fpu_state(void) {
330 }
331 
332 ////////////////////////////////////////////////////////////////////////////////
333 // thread stack
334 
335 // Minimum usable stack sizes required to get to user code. Space for
336 // HotSpot guard pages is added later.
337 size_t os::_compiler_thread_min_stack_allowed = 72 * K;
338 size_t os::_java_thread_min_stack_allowed = 72 * K;
339 size_t os::_vm_internal_thread_min_stack_allowed = 72 * K;
340 
341 // return default stack size for thr_type
342 size_t os::Posix::default_stack_size(os::ThreadType thr_type) {
343   // default stack size (compiler thread needs larger stack)
344   size_t s = (thr_type == os::compiler_thread ? 4 * M : 1 * M);
345   return s;
346 }
347 void os::current_stack_base_and_size(address* base, size_t* size) {
348   address bottom;
349 #ifdef __APPLE__
350   pthread_t self = pthread_self();
351   *base = (address) pthread_get_stackaddr_np(self);
352   *size = pthread_get_stacksize_np(self);
353   bottom = *base - *size;
354 #elif defined(__OpenBSD__)
355   stack_t ss;
356   int rslt = pthread_stackseg_np(pthread_self(), &ss);
357 
358   if (rslt != 0)
359     fatal("pthread_stackseg_np failed with error = %d", rslt);
360 
361   *base = (address) ss.ss_sp;
362   *size = ss.ss_size;
363   bottom = *base - *size;
364 #else
365   pthread_attr_t attr;
366 
367   int rslt = pthread_attr_init(&attr);
368 
369   // JVM needs to know exact stack location, abort if it fails
370   if (rslt != 0)
371     fatal("pthread_attr_init failed with error = %d", rslt);
372 
373   rslt = pthread_attr_get_np(pthread_self(), &attr);
374 
375   if (rslt != 0)
376     fatal("pthread_attr_get_np failed with error = %d", rslt);
377 
378   if (pthread_attr_getstackaddr(&attr, (void **)&bottom) != 0 ||
379       pthread_attr_getstacksize(&attr, size) != 0) {
380     fatal("Can not locate current stack attributes!");
381   }
382 
383   *base = bottom + *size;
384 
385   pthread_attr_destroy(&attr);
386 #endif
387   assert(os::current_stack_pointer() >= bottom &&
388          os::current_stack_pointer() < *base, "just checking");
389 }
390 
391 /////////////////////////////////////////////////////////////////////////////
392 // helper functions for fatal error handler
393 
394 void os::print_context(outputStream *st, const void *context) {
395   if (context == nullptr) return;
396 
397   const ucontext_t *uc = (const ucontext_t*)context;
398 
399   st->print_cr("Registers:");
400   st->print( " x0=" INTPTR_FORMAT, (intptr_t)uc->context_x[ 0]);
401   st->print("  x1=" INTPTR_FORMAT, (intptr_t)uc->context_x[ 1]);
402   st->print("  x2=" INTPTR_FORMAT, (intptr_t)uc->context_x[ 2]);
403   st->print("  x3=" INTPTR_FORMAT, (intptr_t)uc->context_x[ 3]);
404   st->cr();
405   st->print( " x4=" INTPTR_FORMAT, (intptr_t)uc->context_x[ 4]);
406   st->print("  x5=" INTPTR_FORMAT, (intptr_t)uc->context_x[ 5]);
407   st->print("  x6=" INTPTR_FORMAT, (intptr_t)uc->context_x[ 6]);
408   st->print("  x7=" INTPTR_FORMAT, (intptr_t)uc->context_x[ 7]);
409   st->cr();
410   st->print( " x8=" INTPTR_FORMAT, (intptr_t)uc->context_x[ 8]);
411   st->print("  x9=" INTPTR_FORMAT, (intptr_t)uc->context_x[ 9]);
412   st->print(" x10=" INTPTR_FORMAT, (intptr_t)uc->context_x[10]);
413   st->print(" x11=" INTPTR_FORMAT, (intptr_t)uc->context_x[11]);
414   st->cr();
415   st->print( "x12=" INTPTR_FORMAT, (intptr_t)uc->context_x[12]);
416   st->print(" x13=" INTPTR_FORMAT, (intptr_t)uc->context_x[13]);
417   st->print(" x14=" INTPTR_FORMAT, (intptr_t)uc->context_x[14]);
418   st->print(" x15=" INTPTR_FORMAT, (intptr_t)uc->context_x[15]);
419   st->cr();
420   st->print( "x16=" INTPTR_FORMAT, (intptr_t)uc->context_x[16]);
421   st->print(" x17=" INTPTR_FORMAT, (intptr_t)uc->context_x[17]);
422   st->print(" x18=" INTPTR_FORMAT, (intptr_t)uc->context_x[18]);
423   st->print(" x19=" INTPTR_FORMAT, (intptr_t)uc->context_x[19]);
424   st->cr();
425   st->print( "x20=" INTPTR_FORMAT, (intptr_t)uc->context_x[20]);
426   st->print(" x21=" INTPTR_FORMAT, (intptr_t)uc->context_x[21]);
427   st->print(" x22=" INTPTR_FORMAT, (intptr_t)uc->context_x[22]);
428   st->print(" x23=" INTPTR_FORMAT, (intptr_t)uc->context_x[23]);
429   st->cr();
430   st->print( "x24=" INTPTR_FORMAT, (intptr_t)uc->context_x[24]);
431   st->print(" x25=" INTPTR_FORMAT, (intptr_t)uc->context_x[25]);
432   st->print(" x26=" INTPTR_FORMAT, (intptr_t)uc->context_x[26]);
433   st->print(" x27=" INTPTR_FORMAT, (intptr_t)uc->context_x[27]);
434   st->cr();
435   st->print( "x28=" INTPTR_FORMAT, (intptr_t)uc->context_x[28]);
436   st->print("  fp=" INTPTR_FORMAT, (intptr_t)uc->context_fp);
437   st->print("  lr=" INTPTR_FORMAT, (intptr_t)uc->context_lr);
438   st->print("  sp=" INTPTR_FORMAT, (intptr_t)uc->context_sp);
439   st->cr();
440   st->print(  "pc=" INTPTR_FORMAT,  (intptr_t)uc->context_pc);
441   st->print(" cpsr=" INTPTR_FORMAT, (intptr_t)uc->context_cpsr);
442   st->cr();
443 }
444 
445 void os::print_tos_pc(outputStream *st, const void *context) {
446   if (context == nullptr) return;
447 
448   const ucontext_t* uc = (const ucontext_t*)context;
449 
450   address sp = (address)os::Bsd::ucontext_get_sp(uc);
451   print_tos(st, sp);
452   st->cr();
453 
454   // Note: it may be unsafe to inspect memory near pc. For example, pc may
455   // point to garbage if entry point in an nmethod is corrupted. Leave
456   // this at the end, and hope for the best.
457   address pc = os::Posix::ucontext_get_pc(uc);
458   print_instructions(st, pc);
459   st->cr();
460 }
461 
462 void os::print_register_info(outputStream *st, const void *context, int& continuation) {
463   const int register_count = 29 /* x0-x28 */ + 3 /* fp, lr, sp */;
464   int n = continuation;
465   assert(n >= 0 && n <= register_count, "Invalid continuation value");
466   if (context == nullptr || n == register_count) {
467     return;
468   }
469 
470   const ucontext_t *uc = (const ucontext_t*)context;
471   while (n < register_count) {
472     // Update continuation with next index before printing location
473     continuation = n + 1;
474     switch (n) {
475     case 29:
476       st->print(" fp="); print_location(st, uc->context_fp);
477       break;
478     case 30:
479       st->print(" lr="); print_location(st, uc->context_lr);
480       break;
481     case 31:
482       st->print(" sp="); print_location(st, uc->context_sp);
483       break;
484     default:
485       st->print("x%-2d=",n); print_location(st, uc->context_x[n]);
486       break;
487     }
488     ++n;
489   }
490 }
491 
492 void os::setup_fpu() {
493 }
494 
495 #ifndef PRODUCT
496 void os::verify_stack_alignment() {
497   assert(((intptr_t)os::current_stack_pointer() & (StackAlignmentInBytes-1)) == 0, "incorrect stack alignment");
498 }
499 #endif
500 
501 int os::extra_bang_size_in_bytes() {
502   // AArch64 does not require the additional stack bang.
503   return 0;
504 }
505 
506 void os::current_thread_enable_wx(WXMode mode) {
507   pthread_jit_write_protect_np(mode == WXExec);
508 }
509 
510 static inline void atomic_copy64(const volatile void *src, volatile void *dst) {
511   *(jlong *) dst = *(const jlong *) src;
512 }
513 
514 extern "C" {
515   int SpinPause() {
516     // We don't use StubRoutines::aarch64::spin_wait stub in order to
517     // avoid a costly call to os::current_thread_enable_wx() on MacOS.
518     // We should return 1 if SpinPause is implemented, and since there
519     // will be a sequence of 11 instructions for NONE and YIELD and 12
520     // instructions for NOP and ISB, SpinPause will always return 1.
521     uint64_t br_dst;
522     const int instructions_per_case = 2;
523     int64_t off = VM_Version::spin_wait_desc().inst() * instructions_per_case * Assembler::instruction_size;
524 
525     assert(VM_Version::spin_wait_desc().inst() >= SpinWait::NONE &&
526            VM_Version::spin_wait_desc().inst() <= SpinWait::YIELD, "must be");
527     assert(-1 == SpinWait::NONE,  "must be");
528     assert( 0 == SpinWait::NOP,   "must be");
529     assert( 1 == SpinWait::ISB,   "must be");
530     assert( 2 == SpinWait::YIELD, "must be");
531 
532     asm volatile(
533         "  adr  %[d], 20          \n" // 20 == PC here + 5 instructions => address
534                                       // to entry for case SpinWait::NOP
535         "  add  %[d], %[d], %[o]  \n"
536         "  br   %[d]              \n"
537         "  b    SpinPause_return  \n" // case SpinWait::NONE  (-1)
538         "  nop                    \n" // padding
539         "  nop                    \n" // case SpinWait::NOP   ( 0)
540         "  b    SpinPause_return  \n"
541         "  isb                    \n" // case SpinWait::ISB   ( 1)
542         "  b    SpinPause_return  \n"
543         "  yield                  \n" // case SpinWait::YIELD ( 2)
544         "SpinPause_return:        \n"
545         : [d]"=&r"(br_dst)
546         : [o]"r"(off)
547         : "memory");
548     return 1;
549   }
550 
551   void _Copy_conjoint_jshorts_atomic(const jshort* from, jshort* to, size_t count) {
552     if (from > to) {
553       const jshort *end = from + count;
554       while (from < end)
555         *(to++) = *(from++);
556     }
557     else if (from < to) {
558       const jshort *end = from;
559       from += count - 1;
560       to   += count - 1;
561       while (from >= end)
562         *(to--) = *(from--);
563     }
564   }
565   void _Copy_conjoint_jints_atomic(const jint* from, jint* to, size_t count) {
566     if (from > to) {
567       const jint *end = from + count;
568       while (from < end)
569         *(to++) = *(from++);
570     }
571     else if (from < to) {
572       const jint *end = from;
573       from += count - 1;
574       to   += count - 1;
575       while (from >= end)
576         *(to--) = *(from--);
577     }
578   }
579 
580   void _Copy_conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) {
581     if (from > to) {
582       const jlong *end = from + count;
583       while (from < end)
584         atomic_copy64(from++, to++);
585     }
586     else if (from < to) {
587       const jlong *end = from;
588       from += count - 1;
589       to   += count - 1;
590       while (from >= end)
591         atomic_copy64(from--, to--);
592     }
593   }
594 
595   void _Copy_arrayof_conjoint_bytes(const HeapWord* from,
596                                     HeapWord* to,
597                                     size_t    count) {
598     memmove(to, from, count);
599   }
600   void _Copy_arrayof_conjoint_jshorts(const HeapWord* from,
601                                       HeapWord* to,
602                                       size_t    count) {
603     memmove(to, from, count * 2);
604   }
605   void _Copy_arrayof_conjoint_jints(const HeapWord* from,
606                                     HeapWord* to,
607                                     size_t    count) {
608     memmove(to, from, count * 4);
609   }
610   void _Copy_arrayof_conjoint_jlongs(const HeapWord* from,
611                                      HeapWord* to,
612                                      size_t    count) {
613     memmove(to, from, count * 8);
614   }
615 };