1 /*
   2  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef OS_CPU_LINUX_AARCH64_VM_COPY_LINUX_AARCH64_INLINE_HPP
  26 #define OS_CPU_LINUX_AARCH64_VM_COPY_LINUX_AARCH64_INLINE_HPP
  27 
  28 #define COPY_SMALL(from, to, count)                                     \
  29 {                                                                       \
  30         long tmp0, tmp1, tmp2, tmp3;                                    \
  31         long tmp4, tmp5, tmp6, tmp7;                                    \
  32   __asm volatile(                                                       \
  33 "       adr     %[t0], 0f;"                                             \
  34 "       add     %[t0], %[t0], %[cnt], lsl #6;"                          \
  35 "       br      %[t0];"                                                 \
  36 "       .align  6;"                                                     \
  37 "0:"                                                                    \
  38 "       b       1f;"                                                    \
  39 "8:"                                                                    \
  40 "       ldr     %[t0], [%[s], #0];"                                     \
  41 "       ldp     %[t1], %[t2], [%[s], #8];"                              \
  42 "       ldp     %[t3], %[t4], [%[s], #24];"                             \
  43 "       ldr     %[t5], [%[s], #40];"                                    \
  44 "       tbz     %[d], #3, 10f;"                                         \
  45 "9:"                                                                    \
  46 "       str     %[t0], [%[d], #0];"                                     \
  47 "       stp     %[t1], %[t2], [%[d], #8];"                              \
  48 "       stp     %[t3], %[t4], [%[d], #24];"                             \
  49 "       str     %[t5], [%[d], #40];"                                    \
  50 "       b       1f;"                                                    \
  51 "       .align  6;"                                                     \
  52 "       ldr     %[t0], [%[s], #0];"                                     \
  53 "       str     %[t0], [%[d], #0];"                                     \
  54 "       b       1f;"                                                    \
  55 "2:"                                                                    \
  56 "       ldr     %[t0], [%[s], #0];"                                     \
  57 "       ldp     %[t1], %[t2], [%[s], #8];"                              \
  58 "       ldp     %[t3], %[t4], [%[s], #24];"                             \
  59 "       ldp     %[t5], %[t6], [%[s], #40];"                             \
  60 "       ldr     %[t7], [%[s], #56];"                                    \
  61 "       tbz     %[d], #3, 4f;"                                          \
  62 "3:"                                                                    \
  63 "       str     %[t0], [%[d], #0];"                                     \
  64 "       stp     %[t1], %[t2], [%[d], #8];"                              \
  65 "       stp     %[t3], %[t4], [%[d], #24];"                             \
  66 "       stp     %[t5], %[t6], [%[d], #40];"                             \
  67 "       str     %[t7], [%[d], #56];"                                    \
  68 "       b       1f;"                                                    \
  69 "       .align  6;"                                                     \
  70 "       ldr     %[t0], [%[s], #0];"                                     \
  71 "       ldr     %[t1], [%[s], #8];"                                     \
  72 "       str     %[t0], [%[d], #0];"                                     \
  73 "       str     %[t1], [%[d], #8];"                                     \
  74 "       b       1f;"                                                    \
  75 "5:"                                                                    \
  76 "       ldr     %[t0], [%[s], #0];"                                     \
  77 "       ldp     %[t1], %[t2], [%[s], #8];"                              \
  78 "       ldp     %[t3], %[t4], [%[s], #24];"                             \
  79 "       ldp     %[t5], %[t6], [%[s], #40];"                             \
  80 "       tbz     %[d], #3, 7f;"                                          \
  81 "6:"                                                                    \
  82 "       str     %[t0], [%[d], #0];"                                     \
  83 "       stp     %[t1], %[t2], [%[d], #8];"                              \
  84 "       stp     %[t3], %[t4], [%[d], #24];"                             \
  85 "       stp     %[t5], %[t6], [%[d], #40];"                             \
  86 "       b       1f;"                                                    \
  87 "       .align  6;"                                                     \
  88 "       ldr     %[t0], [%[s], #0];"                                     \
  89 "       ldr     %[t1], [%[s], #8];"                                     \
  90 "       ldr     %[t2], [%[s], #16];"                                    \
  91 "       str     %[t0], [%[d], #0];"                                     \
  92 "       str     %[t1], [%[d], #8];"                                     \
  93 "       str     %[t2], [%[d], #16];"                                    \
  94 "       b       1f;"                                                    \
  95 "       .align  6;"                                                     \
  96 "       ldr     %[t0], [%[s], #0];"                                     \
  97 "       ldr     %[t1], [%[s], #8];"                                     \
  98 "       ldr     %[t2], [%[s], #16];"                                    \
  99 "       ldr     %[t3], [%[s], #24];"                                    \
 100 "       str     %[t0], [%[d], #0];"                                     \
 101 "       str     %[t1], [%[d], #8];"                                     \
 102 "       str     %[t2], [%[d], #16];"                                    \
 103 "       str     %[t3], [%[d], #24];"                                    \
 104 "       b       1f;"                                                    \
 105 "       .align  6;"                                                     \
 106 "       ldr     %[t0], [%[s], #0];"                                     \
 107 "       ldr     %[t1], [%[s], #8];"                                     \
 108 "       ldr     %[t2], [%[s], #16];"                                    \
 109 "       ldr     %[t3], [%[s], #24];"                                    \
 110 "       ldr     %[t4], [%[s], #32];"                                    \
 111 "       str     %[t0], [%[d], #0];"                                     \
 112 "       str     %[t1], [%[d], #8];"                                     \
 113 "       str     %[t2], [%[d], #16];"                                    \
 114 "       str     %[t3], [%[d], #24];"                                    \
 115 "       str     %[t4], [%[d], #32];"                                    \
 116 "       b       1f;"                                                    \
 117 "       .align  6;"                                                     \
 118 "       tbnz    %[s], #3, 8b;"                                          \
 119 "       ldp     %[t0], %[t1], [%[s], #0];"                              \
 120 "       ldp     %[t2], %[t3], [%[s], #16];"                             \
 121 "       ldp     %[t4], %[t5], [%[s], #32];"                             \
 122 "       tbnz    %[d], #3, 9b;"                                          \
 123 "10:"                                                                   \
 124 "       stp     %[t0], %[t1], [%[d], #0];"                              \
 125 "       stp     %[t2], %[t3], [%[d], #16];"                             \
 126 "       stp     %[t4], %[t5], [%[d], #32];"                             \
 127 "       b       1f;"                                                    \
 128 "       .align  6;"                                                     \
 129 "       tbnz    %[s], #3, 5b;"                                          \
 130 "       ldp     %[t0], %[t1], [%[s], #0];"                              \
 131 "       ldp     %[t2], %[t3], [%[s], #16];"                             \
 132 "       ldp     %[t4], %[t5], [%[s], #32];"                             \
 133 "       ldr     %[t6], [%[s], #48];"                                    \
 134 "       tbnz    %[d], #3, 6b;"                                          \
 135 "7:"                                                                    \
 136 "       stp     %[t0], %[t1], [%[d], #0];"                              \
 137 "       stp     %[t2], %[t3], [%[d], #16];"                             \
 138 "       stp     %[t4], %[t5], [%[d], #32];"                             \
 139 "       str     %[t6], [%[d], #48];"                                    \
 140 "       b       1f;"                                                    \
 141 "       .align  6;"                                                     \
 142 "       tbnz    %[s], #3, 2b;"                                          \
 143 "       ldp     %[t0], %[t1], [%[s], #0];"                              \
 144 "       ldp     %[t2], %[t3], [%[s], #16];"                             \
 145 "       ldp     %[t4], %[t5], [%[s], #32];"                             \
 146 "       ldp     %[t6], %[t7], [%[s], #48];"                             \
 147 "       tbnz    %[d], #3, 3b;"                                          \
 148 "4:"                                                                    \
 149 "       stp     %[t0], %[t1], [%[d], #0];"                              \
 150 "       stp     %[t2], %[t3], [%[d], #16];"                             \
 151 "       stp     %[t4], %[t5], [%[d], #32];"                             \
 152 "       stp     %[t6], %[t7], [%[d], #48];"                             \
 153 "1:"                                                                    \
 154   : [s]"+r"(from), [d]"+r"(to), [cnt]"+r"(count),                       \
 155     [t0]"=&r"(tmp0), [t1]"=&r"(tmp1), [t2]"=&r"(tmp2), [t3]"=&r"(tmp3), \
 156     [t4]"=&r"(tmp4), [t5]"=&r"(tmp5), [t6]"=&r"(tmp6), [t7]"=&r"(tmp7)  \
 157   :                                                                     \
 158   : "memory", "cc");                                                    \
 159 }
 160 
 161 static void pd_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
 162   __asm volatile( "prfm pldl1strm, [%[s], #0];" :: [s]"r"(from) : "memory");
 163   if (__builtin_expect(count <= 8, 1)) {
 164     COPY_SMALL(from, to, count);
 165     return;
 166   }
 167   _Copy_conjoint_words(from, to, count);
 168 }
 169 
 170 static void pd_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
 171   if (__builtin_constant_p(count)) {
 172     memcpy(to, from, count * sizeof(HeapWord));
 173     return;
 174   }
 175   __asm volatile( "prfm pldl1strm, [%[s], #0];" :: [s]"r"(from) : "memory");
 176   if (__builtin_expect(count <= 8, 1)) {
 177     COPY_SMALL(from, to, count);
 178     return;
 179   }
 180   _Copy_disjoint_words(from, to, count);
 181 }
 182 
 183 static void pd_disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count) {
 184   __asm volatile( "prfm pldl1strm, [%[s], #0];" :: [s]"r"(from) : "memory");
 185   if (__builtin_expect(count <= 8, 1)) {
 186     COPY_SMALL(from, to, count);
 187     return;
 188   }
 189   _Copy_disjoint_words(from, to, count);
 190 }
 191 
 192 static void pd_aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) {
 193   pd_conjoint_words(from, to, count);
 194 }
 195 
 196 static void pd_aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) {
 197   pd_disjoint_words(from, to, count);
 198 }
 199 
 200 static void pd_conjoint_bytes(void* from, void* to, size_t count) {
 201   (void)memmove(to, from, count);
 202 }
 203 
 204 static void pd_conjoint_bytes_atomic(void* from, void* to, size_t count) {
 205   pd_conjoint_bytes(from, to, count);
 206 }
 207 
 208 static void pd_conjoint_jshorts_atomic(jshort* from, jshort* to, size_t count) {
 209   _Copy_conjoint_jshorts_atomic(from, to, count);
 210 }
 211 
 212 static void pd_conjoint_jints_atomic(jint* from, jint* to, size_t count) {
 213   _Copy_conjoint_jints_atomic(from, to, count);
 214 }
 215 
 216 static void pd_conjoint_jlongs_atomic(jlong* from, jlong* to, size_t count) {
 217   _Copy_conjoint_jlongs_atomic(from, to, count);
 218 }
 219 
 220 static void pd_conjoint_oops_atomic(oop* from, oop* to, size_t count) {
 221   assert(!UseCompressedOops, "foo!");
 222   assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
 223   _Copy_conjoint_jlongs_atomic((jlong*)from, (jlong*)to, count);
 224 }
 225 
 226 static void pd_arrayof_conjoint_bytes(HeapWord* from, HeapWord* to, size_t count) {
 227   _Copy_arrayof_conjoint_bytes(from, to, count);
 228 }
 229 
 230 static void pd_arrayof_conjoint_jshorts(HeapWord* from, HeapWord* to, size_t count) {
 231   _Copy_arrayof_conjoint_jshorts(from, to, count);
 232 }
 233 
 234 static void pd_arrayof_conjoint_jints(HeapWord* from, HeapWord* to, size_t count) {
 235    _Copy_arrayof_conjoint_jints(from, to, count);
 236 }
 237 
 238 static void pd_arrayof_conjoint_jlongs(HeapWord* from, HeapWord* to, size_t count) {
 239   _Copy_arrayof_conjoint_jlongs(from, to, count);
 240 }
 241 
 242 static void pd_arrayof_conjoint_oops(HeapWord* from, HeapWord* to, size_t count) {
 243   assert(!UseCompressedOops, "foo!");
 244   assert(BytesPerLong == BytesPerOop, "jlongs and oops must be the same size");
 245   _Copy_arrayof_conjoint_jlongs(from, to, count);
 246 }
 247 
 248 #endif // OS_CPU_LINUX_AARCH64_VM_COPY_LINUX_AARCH64_INLINE_HPP