1 /*
2 * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2025, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/register.hpp"
29 #include "atomic_aarch64.hpp"
30 #include "compiler/oopMap.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/barrierSetAssembler.hpp"
33 #include "gc/shared/gc_globals.hpp"
34 #include "gc/shared/tlab_globals.hpp"
35 #include "interpreter/interpreter.hpp"
36 #include "memory/universe.hpp"
37 #include "nativeInst_aarch64.hpp"
38 #include "oops/instanceOop.hpp"
39 #include "oops/method.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "prims/methodHandles.hpp"
43 #include "prims/upcallLinker.hpp"
44 #include "runtime/arguments.hpp"
45 #include "runtime/atomicAccess.hpp"
46 #include "runtime/continuation.hpp"
47 #include "runtime/continuationEntry.inline.hpp"
48 #include "runtime/frame.inline.hpp"
49 #include "runtime/handles.inline.hpp"
50 #include "runtime/javaThread.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubCodeGenerator.hpp"
53 #include "runtime/stubRoutines.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/checkedCast.hpp"
56 #include "utilities/debug.hpp"
57 #include "utilities/globalDefinitions.hpp"
58 #include "utilities/intpow.hpp"
59 #include "utilities/powerOfTwo.hpp"
60 #ifdef COMPILER2
61 #include "opto/runtime.hpp"
62 #endif
63 #if INCLUDE_ZGC
64 #include "gc/z/zThreadLocalData.hpp"
65 #endif
66
67 // Declaration and definition of StubGenerator (no .hpp file).
68 // For a more detailed description of the stub routine structure
69 // see the comment in stubRoutines.hpp
70
71 #undef __
72 #define __ _masm->
73
74 #ifdef PRODUCT
75 #define BLOCK_COMMENT(str) /* nothing */
76 #else
77 #define BLOCK_COMMENT(str) __ block_comment(str)
78 #endif
79
80 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
81
82 // Constant data definitions
83
84 static const uint32_t _sha256_round_consts[64] = {
85 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
86 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
87 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
88 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
89 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
90 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
91 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
92 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
93 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
94 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
95 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
96 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
97 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
98 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
99 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
100 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
101 };
102
103 static const uint64_t _sha512_round_consts[80] = {
104 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL,
105 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L,
106 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L,
107 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L,
108 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L,
109 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L,
110 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L,
111 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L,
112 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL,
113 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L,
114 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL,
115 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL,
116 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L,
117 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L,
118 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L,
119 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L,
120 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L,
121 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL,
122 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL,
123 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL,
124 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L,
125 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L,
126 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL,
127 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL,
128 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL,
129 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL,
130 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L
131 };
132
133 static const uint64_t _sha3_round_consts[24] = {
134 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
135 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
136 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
137 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
138 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
139 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
140 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
141 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
142 };
143
144 static const uint64_t _double_keccak_round_consts[24] = {
145 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
146 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
147 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
148 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
149 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
150 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
151 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
152 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
153 };
154
155 //Omit 3rd limb of modulus since it is 0
156 static const int64_t _modulus_P256[5] = {
157 0x000fffffffffffffL, 0x00000fffffffffffL,
158 0x0000001000000000L, 0x0000ffffffff0000L
159 };
160
161 static const char _encodeBlock_toBase64[64] = {
162 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
163 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
164 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
165 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
166 '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
167 };
168
169 static const char _encodeBlock_toBase64URL[64] = {
170 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
171 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
172 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
173 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
174 '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'
175 };
176
177 // Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64,
178 // except the trailing character '=' is also treated illegal value in this intrinsic. That
179 // is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here.
180 static const uint8_t _decodeBlock_fromBase64ForNoSIMD[256] = {
181 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
182 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
183 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
184 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
185 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
186 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u,
187 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
188 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
189 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
190 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
191 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
192 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
193 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
194 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
195 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
196 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
197 };
198
199 static const uint8_t _decodeBlock_fromBase64URLForNoSIMD[256] = {
200 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
201 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
202 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
203 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
204 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
205 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u,
206 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
207 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
208 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
209 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
210 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
211 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
212 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
213 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
214 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
215 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
216 };
217
218 // A legal value of base64 code is in range [0, 127]. We need two lookups
219 // with tbl/tbx and combine them to get the decode data. The 1st table vector
220 // lookup use tbl, out of range indices are set to 0 in destination. The 2nd
221 // table vector lookup use tbx, out of range indices are unchanged in
222 // destination. Input [64..126] is mapped to index [65, 127] in second lookup.
223 // The value of index 64 is set to 0, so that we know that we already get the
224 // decoded data with the 1st lookup.
225 static const uint8_t _decodeBlock_fromBase64ForSIMD[128] = {
226 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
227 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
228 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
229 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
230 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
231 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
232 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
233 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
234 };
235
236 static const uint8_t _decodeBlock_fromBase64URLForSIMD[128] = {
237 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
238 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
239 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
240 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
241 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
242 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
243 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
244 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
245 };
246
247
248 // Stub Code definitions
249
250 class StubGenerator: public StubCodeGenerator {
251 private:
252
253 #ifdef PRODUCT
254 #define inc_counter_np(counter) ((void)0)
255 #else
256 void inc_counter_np_(uint& counter) {
257 __ incrementw(ExternalAddress((address)&counter));
258 }
259 #define inc_counter_np(counter) \
260 BLOCK_COMMENT("inc_counter " #counter); \
261 inc_counter_np_(counter);
262 #endif
263
264 // Call stubs are used to call Java from C
265 //
266 // Arguments:
267 // c_rarg0: call wrapper address address
268 // c_rarg1: result address
269 // c_rarg2: result type BasicType
270 // c_rarg3: method Method*
271 // c_rarg4: (interpreter) entry point address
272 // c_rarg5: parameters intptr_t*
273 // c_rarg6: parameter size (in words) int
274 // c_rarg7: thread Thread*
275 //
276 // There is no return from the stub itself as any Java result
277 // is written to result
278 //
279 // we save r30 (lr) as the return PC at the base of the frame and
280 // link r29 (fp) below it as the frame pointer installing sp (r31)
281 // into fp.
282 //
283 // we save r0-r7, which accounts for all the c arguments.
284 //
285 // TODO: strictly do we need to save them all? they are treated as
286 // volatile by C so could we omit saving the ones we are going to
287 // place in global registers (thread? method?) or those we only use
288 // during setup of the Java call?
289 //
290 // we don't need to save r8 which C uses as an indirect result location
291 // return register.
292 //
293 // we don't need to save r9-r15 which both C and Java treat as
294 // volatile
295 //
296 // we don't need to save r16-18 because Java does not use them
297 //
298 // we save r19-r28 which Java uses as scratch registers and C
299 // expects to be callee-save
300 //
301 // we save the bottom 64 bits of each value stored in v8-v15; it is
302 // the responsibility of the caller to preserve larger values.
303 //
304 // so the stub frame looks like this when we enter Java code
305 //
306 // [ return_from_Java ] <--- sp
307 // [ argument word n ]
308 // ...
309 // -29 [ argument word 1 ]
310 // -28 [ saved Floating-point Control Register ]
311 // -26 [ saved v15 ] <--- sp_after_call
312 // -25 [ saved v14 ]
313 // -24 [ saved v13 ]
314 // -23 [ saved v12 ]
315 // -22 [ saved v11 ]
316 // -21 [ saved v10 ]
317 // -20 [ saved v9 ]
318 // -19 [ saved v8 ]
319 // -18 [ saved r28 ]
320 // -17 [ saved r27 ]
321 // -16 [ saved r26 ]
322 // -15 [ saved r25 ]
323 // -14 [ saved r24 ]
324 // -13 [ saved r23 ]
325 // -12 [ saved r22 ]
326 // -11 [ saved r21 ]
327 // -10 [ saved r20 ]
328 // -9 [ saved r19 ]
329 // -8 [ call wrapper (r0) ]
330 // -7 [ result (r1) ]
331 // -6 [ result type (r2) ]
332 // -5 [ method (r3) ]
333 // -4 [ entry point (r4) ]
334 // -3 [ parameters (r5) ]
335 // -2 [ parameter size (r6) ]
336 // -1 [ thread (r7) ]
337 // 0 [ saved fp (r29) ] <--- fp == saved sp (r31)
338 // 1 [ saved lr (r30) ]
339
340 // Call stub stack layout word offsets from fp
341 enum call_stub_layout {
342 sp_after_call_off = -28,
343
344 fpcr_off = sp_after_call_off,
345 d15_off = -26,
346 d13_off = -24,
347 d11_off = -22,
348 d9_off = -20,
349
350 r28_off = -18,
351 r26_off = -16,
352 r24_off = -14,
353 r22_off = -12,
354 r20_off = -10,
355 call_wrapper_off = -8,
356 result_off = -7,
357 result_type_off = -6,
358 method_off = -5,
359 entry_point_off = -4,
360 parameter_size_off = -2,
361 thread_off = -1,
362 fp_f = 0,
363 retaddr_off = 1,
364 };
365
366 address generate_call_stub(address& return_address) {
367 assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 &&
368 (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
369 "adjust this code");
370
371 StubId stub_id = StubId::stubgen_call_stub_id;
372 GrowableArray<address> entries;
373 int entry_count = StubInfo::entry_count(stub_id);
374 assert(entry_count == 2, "sanity check");
375 address start = load_archive_data(stub_id, &entries);
376 if (start != nullptr) {
377 assert(entries.length() == 1, "expected 1 extra entry");
378 return_address = entries.at(0);
379 return start;
380 }
381 StubCodeMark mark(this, stub_id);
382 start = __ pc();
383
384 const Address sp_after_call (rfp, sp_after_call_off * wordSize);
385
386 const Address fpcr_save (rfp, fpcr_off * wordSize);
387 const Address call_wrapper (rfp, call_wrapper_off * wordSize);
388 const Address result (rfp, result_off * wordSize);
389 const Address result_type (rfp, result_type_off * wordSize);
390 const Address method (rfp, method_off * wordSize);
391 const Address entry_point (rfp, entry_point_off * wordSize);
392 const Address parameter_size(rfp, parameter_size_off * wordSize);
393
394 const Address thread (rfp, thread_off * wordSize);
395
396 const Address d15_save (rfp, d15_off * wordSize);
397 const Address d13_save (rfp, d13_off * wordSize);
398 const Address d11_save (rfp, d11_off * wordSize);
399 const Address d9_save (rfp, d9_off * wordSize);
400
401 const Address r28_save (rfp, r28_off * wordSize);
402 const Address r26_save (rfp, r26_off * wordSize);
403 const Address r24_save (rfp, r24_off * wordSize);
404 const Address r22_save (rfp, r22_off * wordSize);
405 const Address r20_save (rfp, r20_off * wordSize);
406
407 // stub code
408
409 address aarch64_entry = __ pc();
410
411 // set up frame and move sp to end of save area
412 __ enter();
413 __ sub(sp, rfp, -sp_after_call_off * wordSize);
414
415 // save register parameters and Java scratch/global registers
416 // n.b. we save thread even though it gets installed in
417 // rthread because we want to sanity check rthread later
418 __ str(c_rarg7, thread);
419 __ strw(c_rarg6, parameter_size);
420 __ stp(c_rarg4, c_rarg5, entry_point);
421 __ stp(c_rarg2, c_rarg3, result_type);
422 __ stp(c_rarg0, c_rarg1, call_wrapper);
423
424 __ stp(r20, r19, r20_save);
425 __ stp(r22, r21, r22_save);
426 __ stp(r24, r23, r24_save);
427 __ stp(r26, r25, r26_save);
428 __ stp(r28, r27, r28_save);
429
430 __ stpd(v9, v8, d9_save);
431 __ stpd(v11, v10, d11_save);
432 __ stpd(v13, v12, d13_save);
433 __ stpd(v15, v14, d15_save);
434
435 __ get_fpcr(rscratch1);
436 __ str(rscratch1, fpcr_save);
437 // Set FPCR to the state we need. We do want Round to Nearest. We
438 // don't want non-IEEE rounding modes or floating-point traps.
439 __ bfi(rscratch1, zr, 22, 4); // Clear DN, FZ, and Rmode
440 __ bfi(rscratch1, zr, 8, 5); // Clear exception-control bits (8-12)
441 __ set_fpcr(rscratch1);
442
443 // install Java thread in global register now we have saved
444 // whatever value it held
445 __ mov(rthread, c_rarg7);
446 // And method
447 __ mov(rmethod, c_rarg3);
448
449 // set up the heapbase register
450 __ reinit_heapbase();
451
452 #ifdef ASSERT
453 // make sure we have no pending exceptions
454 {
455 Label L;
456 __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
457 __ cmp(rscratch1, (u1)NULL_WORD);
458 __ br(Assembler::EQ, L);
459 __ stop("StubRoutines::call_stub: entered with pending exception");
460 __ BIND(L);
461 }
462 #endif
463 // pass parameters if any
464 __ mov(esp, sp);
465 __ sub(rscratch1, sp, c_rarg6, ext::uxtw, LogBytesPerWord); // Move SP out of the way
466 __ andr(sp, rscratch1, -2 * wordSize);
467
468 BLOCK_COMMENT("pass parameters if any");
469 Label parameters_done;
470 // parameter count is still in c_rarg6
471 // and parameter pointer identifying param 1 is in c_rarg5
472 __ cbzw(c_rarg6, parameters_done);
473
474 address loop = __ pc();
475 __ ldr(rscratch1, Address(__ post(c_rarg5, wordSize)));
476 __ subsw(c_rarg6, c_rarg6, 1);
477 __ push(rscratch1);
478 __ br(Assembler::GT, loop);
479
480 __ BIND(parameters_done);
481
482 // call Java entry -- passing methdoOop, and current sp
483 // rmethod: Method*
484 // r19_sender_sp: sender sp
485 BLOCK_COMMENT("call Java function");
486 __ mov(r19_sender_sp, sp);
487 __ blr(c_rarg4);
488
489 // we do this here because the notify will already have been done
490 // if we get to the next instruction via an exception
491 //
492 // n.b. adding this instruction here affects the calculation of
493 // whether or not a routine returns to the call stub (used when
494 // doing stack walks) since the normal test is to check the return
495 // pc against the address saved below. so we may need to allow for
496 // this extra instruction in the check.
497
498 // save current address for use by exception handling code
499
500 return_address = __ pc();
501 entries.append(return_address);
502
503 // store result depending on type (everything that is not
504 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
505 // n.b. this assumes Java returns an integral result in r0
506 // and a floating result in j_farg0
507 __ ldr(j_rarg2, result);
508 Label is_long, is_float, is_double, exit;
509 __ ldr(j_rarg1, result_type);
510 __ cmp(j_rarg1, (u1)T_OBJECT);
511 __ br(Assembler::EQ, is_long);
512 __ cmp(j_rarg1, (u1)T_LONG);
513 __ br(Assembler::EQ, is_long);
514 __ cmp(j_rarg1, (u1)T_FLOAT);
515 __ br(Assembler::EQ, is_float);
516 __ cmp(j_rarg1, (u1)T_DOUBLE);
517 __ br(Assembler::EQ, is_double);
518
519 // handle T_INT case
520 __ strw(r0, Address(j_rarg2));
521
522 __ BIND(exit);
523
524 // pop parameters
525 __ sub(esp, rfp, -sp_after_call_off * wordSize);
526
527 #ifdef ASSERT
528 // verify that threads correspond
529 {
530 Label L, S;
531 __ ldr(rscratch1, thread);
532 __ cmp(rthread, rscratch1);
533 __ br(Assembler::NE, S);
534 __ get_thread(rscratch1);
535 __ cmp(rthread, rscratch1);
536 __ br(Assembler::EQ, L);
537 __ BIND(S);
538 __ stop("StubRoutines::call_stub: threads must correspond");
539 __ BIND(L);
540 }
541 #endif
542
543 __ pop_cont_fastpath(rthread);
544
545 // restore callee-save registers
546 __ ldpd(v15, v14, d15_save);
547 __ ldpd(v13, v12, d13_save);
548 __ ldpd(v11, v10, d11_save);
549 __ ldpd(v9, v8, d9_save);
550
551 __ ldp(r28, r27, r28_save);
552 __ ldp(r26, r25, r26_save);
553 __ ldp(r24, r23, r24_save);
554 __ ldp(r22, r21, r22_save);
555 __ ldp(r20, r19, r20_save);
556
557 // restore fpcr
558 __ ldr(rscratch1, fpcr_save);
559 __ set_fpcr(rscratch1);
560
561 __ ldp(c_rarg0, c_rarg1, call_wrapper);
562 __ ldrw(c_rarg2, result_type);
563 __ ldr(c_rarg3, method);
564 __ ldp(c_rarg4, c_rarg5, entry_point);
565 __ ldp(c_rarg6, c_rarg7, parameter_size);
566
567 // leave frame and return to caller
568 __ leave();
569 __ ret(lr);
570
571 // handle return types different from T_INT
572
573 __ BIND(is_long);
574 __ str(r0, Address(j_rarg2, 0));
575 __ br(Assembler::AL, exit);
576
577 __ BIND(is_float);
578 __ strs(j_farg0, Address(j_rarg2, 0));
579 __ br(Assembler::AL, exit);
580
581 __ BIND(is_double);
582 __ strd(j_farg0, Address(j_rarg2, 0));
583 __ br(Assembler::AL, exit);
584
585 // record the stub entry and end plus the auxiliary entry
586 store_archive_data(stub_id, start, __ pc(), &entries);
587
588 return start;
589 }
590
591 // Return point for a Java call if there's an exception thrown in
592 // Java code. The exception is caught and transformed into a
593 // pending exception stored in JavaThread that can be tested from
594 // within the VM.
595 //
596 // Note: Usually the parameters are removed by the callee. In case
597 // of an exception crossing an activation frame boundary, that is
598 // not the case if the callee is compiled code => need to setup the
599 // rsp.
600 //
601 // r0: exception oop
602
603 address generate_catch_exception() {
604 StubId stub_id = StubId::stubgen_catch_exception_id;
605 int entry_count = StubInfo::entry_count(stub_id);
606 assert(entry_count == 1, "sanity check");
607 address start = load_archive_data(stub_id);
608 if (start != nullptr) {
609 return start;
610 }
611 StubCodeMark mark(this, stub_id);
612 start = __ pc();
613
614 // same as in generate_call_stub():
615 const Address sp_after_call(rfp, sp_after_call_off * wordSize);
616 const Address thread (rfp, thread_off * wordSize);
617
618 #ifdef ASSERT
619 // verify that threads correspond
620 {
621 Label L, S;
622 __ ldr(rscratch1, thread);
623 __ cmp(rthread, rscratch1);
624 __ br(Assembler::NE, S);
625 __ get_thread(rscratch1);
626 __ cmp(rthread, rscratch1);
627 __ br(Assembler::EQ, L);
628 __ bind(S);
629 __ stop("StubRoutines::catch_exception: threads must correspond");
630 __ bind(L);
631 }
632 #endif
633
634 // set pending exception
635 __ verify_oop(r0);
636
637 __ str(r0, Address(rthread, Thread::pending_exception_offset()));
638 // special case -- add file name string to AOT address table
639 address file = (address)AOTCodeCache::add_C_string(__FILE__);
640 __ lea(rscratch1, ExternalAddress(file));
641 __ str(rscratch1, Address(rthread, Thread::exception_file_offset()));
642 __ movw(rscratch1, (int)__LINE__);
643 __ strw(rscratch1, Address(rthread, Thread::exception_line_offset()));
644
645 // complete return to VM
646 assert(StubRoutines::_call_stub_return_address != nullptr,
647 "_call_stub_return_address must have been generated before");
648 __ b(RuntimeAddress(StubRoutines::_call_stub_return_address));
649
650 // record the stub entry and end
651 store_archive_data(stub_id, start, __ pc());
652
653 return start;
654 }
655
656 // Continuation point for runtime calls returning with a pending
657 // exception. The pending exception check happened in the runtime
658 // or native call stub. The pending exception in Thread is
659 // converted into a Java-level exception.
660 //
661 // Contract with Java-level exception handlers:
662 // r0: exception
663 // r3: throwing pc
664 //
665 // NOTE: At entry of this stub, exception-pc must be in LR !!
666
667 // NOTE: this is always used as a jump target within generated code
668 // so it just needs to be generated code with no x86 prolog
669
670 address generate_forward_exception() {
671 StubId stub_id = StubId::stubgen_forward_exception_id;
672 int entry_count = StubInfo::entry_count(stub_id);
673 assert(entry_count == 1, "sanity check");
674 address start = load_archive_data(stub_id);
675 if (start != nullptr) {
676 return start;
677 }
678 StubCodeMark mark(this, stub_id);
679 start = __ pc();
680
681 // Upon entry, LR points to the return address returning into
682 // Java (interpreted or compiled) code; i.e., the return address
683 // becomes the throwing pc.
684 //
685 // Arguments pushed before the runtime call are still on the stack
686 // but the exception handler will reset the stack pointer ->
687 // ignore them. A potential result in registers can be ignored as
688 // well.
689
690 #ifdef ASSERT
691 // make sure this code is only executed if there is a pending exception
692 {
693 Label L;
694 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
695 __ cbnz(rscratch1, L);
696 __ stop("StubRoutines::forward exception: no pending exception (1)");
697 __ bind(L);
698 }
699 #endif
700
701 // compute exception handler into r19
702
703 // call the VM to find the handler address associated with the
704 // caller address. pass thread in r0 and caller pc (ret address)
705 // in r1. n.b. the caller pc is in lr, unlike x86 where it is on
706 // the stack.
707 __ mov(c_rarg1, lr);
708 // lr will be trashed by the VM call so we move it to R19
709 // (callee-saved) because we also need to pass it to the handler
710 // returned by this call.
711 __ mov(r19, lr);
712 BLOCK_COMMENT("call exception_handler_for_return_address");
713 __ call_VM_leaf(CAST_FROM_FN_PTR(address,
714 SharedRuntime::exception_handler_for_return_address),
715 rthread, c_rarg1);
716 // Reinitialize the ptrue predicate register, in case the external runtime
717 // call clobbers ptrue reg, as we may return to SVE compiled code.
718 __ reinitialize_ptrue();
719
720 // we should not really care that lr is no longer the callee
721 // address. we saved the value the handler needs in r19 so we can
722 // just copy it to r3. however, the C2 handler will push its own
723 // frame and then calls into the VM and the VM code asserts that
724 // the PC for the frame above the handler belongs to a compiled
725 // Java method. So, we restore lr here to satisfy that assert.
726 __ mov(lr, r19);
727 // setup r0 & r3 & clear pending exception
728 __ mov(r3, r19);
729 __ mov(r19, r0);
730 __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
731 __ str(zr, Address(rthread, Thread::pending_exception_offset()));
732
733 #ifdef ASSERT
734 // make sure exception is set
735 {
736 Label L;
737 __ cbnz(r0, L);
738 __ stop("StubRoutines::forward exception: no pending exception (2)");
739 __ bind(L);
740 }
741 #endif
742
743 // continue at exception handler
744 // r0: exception
745 // r3: throwing pc
746 // r19: exception handler
747 __ verify_oop(r0);
748 __ br(r19);
749
750 // record the stub entry and end
751 store_archive_data(stub_id, start, __ pc());
752
753 return start;
754 }
755
756 // Non-destructive plausibility checks for oops
757 //
758 // Arguments:
759 // r0: oop to verify
760 // rscratch1: error message
761 //
762 // Stack after saving c_rarg3:
763 // [tos + 0]: saved c_rarg3
764 // [tos + 1]: saved c_rarg2
765 // [tos + 2]: saved lr
766 // [tos + 3]: saved rscratch2
767 // [tos + 4]: saved r0
768 // [tos + 5]: saved rscratch1
769 address generate_verify_oop() {
770 StubId stub_id = StubId::stubgen_verify_oop_id;
771 int entry_count = StubInfo::entry_count(stub_id);
772 assert(entry_count == 1, "sanity check");
773 address start = load_archive_data(stub_id);
774 if (start != nullptr) {
775 return start;
776 }
777 StubCodeMark mark(this, stub_id);
778 start = __ pc();
779
780 Label exit, error;
781
782 // save c_rarg2 and c_rarg3
783 __ stp(c_rarg3, c_rarg2, Address(__ pre(sp, -16)));
784
785 // __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
786 __ lea(c_rarg2, ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
787 __ ldr(c_rarg3, Address(c_rarg2));
788 __ add(c_rarg3, c_rarg3, 1);
789 __ str(c_rarg3, Address(c_rarg2));
790
791 // object is in r0
792 // make sure object is 'reasonable'
793 __ cbz(r0, exit); // if obj is null it is OK
794
795 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
796 bs_asm->check_oop(_masm, r0, c_rarg2, c_rarg3, error);
797
798 // return if everything seems ok
799 __ bind(exit);
800
801 __ ldp(c_rarg3, c_rarg2, Address(__ post(sp, 16)));
802 __ ret(lr);
803
804 // handle errors
805 __ bind(error);
806 __ ldp(c_rarg3, c_rarg2, Address(__ post(sp, 16)));
807
808 __ push(RegSet::range(r0, r29), sp);
809 // debug(char* msg, int64_t pc, int64_t regs[])
810 __ mov(c_rarg0, rscratch1); // pass address of error message
811 __ mov(c_rarg1, lr); // pass return address
812 __ mov(c_rarg2, sp); // pass address of regs on stack
813 #ifndef PRODUCT
814 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
815 #endif
816 BLOCK_COMMENT("call MacroAssembler::debug");
817 __ mov(rscratch1, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
818 __ blr(rscratch1);
819 __ hlt(0);
820
821 // record the stub entry and end
822 store_archive_data(stub_id, start, __ pc());
823
824 return start;
825 }
826
827 // Generate indices for iota vector.
828 void generate_iota_indices(StubId stub_id) {
829 GrowableArray<address> entries;
830 int entry_count = StubInfo::entry_count(stub_id);
831 assert(entry_count == VECTOR_IOTA_COUNT, "sanity check");
832 address start = load_archive_data(stub_id, &entries);
833 if (start != nullptr) {
834 assert(entries.length() == entry_count - 1,
835 "unexpected entries count %d", entries.length());
836 StubRoutines::aarch64::_vector_iota_indices[0] = start;
837 for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
838 StubRoutines::aarch64::_vector_iota_indices[i] = entries.at(i - 1);
839 }
840 return;
841 }
842 __ align(CodeEntryAlignment);
843 StubCodeMark mark(this, stub_id);
844 start = __ pc();
845 // B
846 __ emit_data64(0x0706050403020100, relocInfo::none);
847 __ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none);
848 entries.append(__ pc());
849 // H
850 __ emit_data64(0x0003000200010000, relocInfo::none);
851 __ emit_data64(0x0007000600050004, relocInfo::none);
852 entries.append(__ pc());
853 // S
854 __ emit_data64(0x0000000100000000, relocInfo::none);
855 __ emit_data64(0x0000000300000002, relocInfo::none);
856 entries.append(__ pc());
857 // D
858 __ emit_data64(0x0000000000000000, relocInfo::none);
859 __ emit_data64(0x0000000000000001, relocInfo::none);
860 entries.append(__ pc());
861 // S - FP
862 __ emit_data64(0x3F80000000000000, relocInfo::none); // 0.0f, 1.0f
863 __ emit_data64(0x4040000040000000, relocInfo::none); // 2.0f, 3.0f
864 entries.append(__ pc());
865 // D - FP
866 __ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d
867 __ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d
868
869 // record the stub entry and end
870 store_archive_data(stub_id, start, __ pc(), &entries);
871
872 // install the entry addresses in the entry array
873 assert(entries.length() == entry_count - 1,
874 "unexpected entries count %d", entries.length());
875 StubRoutines::aarch64::_vector_iota_indices[0] = start;
876 for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
877 StubRoutines::aarch64::_vector_iota_indices[i] = entries.at(i - 1);
878 }
879 }
880
881 // The inner part of zero_words(). This is the bulk operation,
882 // zeroing words in blocks, possibly using DC ZVA to do it. The
883 // caller is responsible for zeroing the last few words.
884 //
885 // Inputs:
886 // r10: the HeapWord-aligned base address of an array to zero.
887 // r11: the count in HeapWords, r11 > 0.
888 //
889 // Returns r10 and r11, adjusted for the caller to clear.
890 // r10: the base address of the tail of words left to clear.
891 // r11: the number of words in the tail.
892 // r11 < MacroAssembler::zero_words_block_size.
893
894 address generate_zero_blocks() {
895 StubId stub_id = StubId::stubgen_zero_blocks_id;
896 int entry_count = StubInfo::entry_count(stub_id);
897 assert(entry_count == 1, "sanity check");
898 address start = load_archive_data(stub_id);
899 if (start != nullptr) {
900 return start;
901 }
902 __ align(CodeEntryAlignment);
903 StubCodeMark mark(this, stub_id);
904 Label done;
905 Label base_aligned;
906
907 Register base = r10, cnt = r11;
908
909 start = __ pc();
910
911 if (UseBlockZeroing) {
912 int zva_length = VM_Version::zva_length();
913
914 // Ensure ZVA length can be divided by 16. This is required by
915 // the subsequent operations.
916 assert (zva_length % 16 == 0, "Unexpected ZVA Length");
917
918 __ tbz(base, 3, base_aligned);
919 __ str(zr, Address(__ post(base, 8)));
920 __ sub(cnt, cnt, 1);
921 __ bind(base_aligned);
922
923 // Ensure count >= zva_length * 2 so that it still deserves a zva after
924 // alignment.
925 Label small;
926 int low_limit = MAX2(zva_length * 2, (int)BlockZeroingLowLimit);
927 __ subs(rscratch1, cnt, low_limit >> 3);
928 __ br(Assembler::LT, small);
929 __ zero_dcache_blocks(base, cnt);
930 __ bind(small);
931 }
932
933 {
934 // Number of stp instructions we'll unroll
935 const int unroll =
936 MacroAssembler::zero_words_block_size / 2;
937 // Clear the remaining blocks.
938 Label loop;
939 __ subs(cnt, cnt, unroll * 2);
940 __ br(Assembler::LT, done);
941 __ bind(loop);
942 for (int i = 0; i < unroll; i++)
943 __ stp(zr, zr, __ post(base, 16));
944 __ subs(cnt, cnt, unroll * 2);
945 __ br(Assembler::GE, loop);
946 __ bind(done);
947 __ add(cnt, cnt, unroll * 2);
948 }
949
950 __ ret(lr);
951
952 // record the stub entry and end
953 store_archive_data(stub_id, start, __ pc());
954
955 return start;
956 }
957
958
959 typedef enum {
960 copy_forwards = 1,
961 copy_backwards = -1
962 } copy_direction;
963
964 // Helper object to reduce noise when telling the GC barriers how to perform loads and stores
965 // for arraycopy stubs.
966 class ArrayCopyBarrierSetHelper : StackObj {
967 BarrierSetAssembler* _bs_asm;
968 MacroAssembler* _masm;
969 DecoratorSet _decorators;
970 BasicType _type;
971 Register _gct1;
972 Register _gct2;
973 Register _gct3;
974 FloatRegister _gcvt1;
975 FloatRegister _gcvt2;
976 FloatRegister _gcvt3;
977
978 public:
979 ArrayCopyBarrierSetHelper(MacroAssembler* masm,
980 DecoratorSet decorators,
981 BasicType type,
982 Register gct1,
983 Register gct2,
984 Register gct3,
985 FloatRegister gcvt1,
986 FloatRegister gcvt2,
987 FloatRegister gcvt3)
988 : _bs_asm(BarrierSet::barrier_set()->barrier_set_assembler()),
989 _masm(masm),
990 _decorators(decorators),
991 _type(type),
992 _gct1(gct1),
993 _gct2(gct2),
994 _gct3(gct3),
995 _gcvt1(gcvt1),
996 _gcvt2(gcvt2),
997 _gcvt3(gcvt3) {
998 }
999
1000 void copy_load_at_32(FloatRegister dst1, FloatRegister dst2, Address src) {
1001 _bs_asm->copy_load_at(_masm, _decorators, _type, 32,
1002 dst1, dst2, src,
1003 _gct1, _gct2, _gcvt1);
1004 }
1005
1006 void copy_store_at_32(Address dst, FloatRegister src1, FloatRegister src2) {
1007 _bs_asm->copy_store_at(_masm, _decorators, _type, 32,
1008 dst, src1, src2,
1009 _gct1, _gct2, _gct3, _gcvt1, _gcvt2, _gcvt3);
1010 }
1011
1012 void copy_load_at_16(Register dst1, Register dst2, Address src) {
1013 _bs_asm->copy_load_at(_masm, _decorators, _type, 16,
1014 dst1, dst2, src,
1015 _gct1);
1016 }
1017
1018 void copy_store_at_16(Address dst, Register src1, Register src2) {
1019 _bs_asm->copy_store_at(_masm, _decorators, _type, 16,
1020 dst, src1, src2,
1021 _gct1, _gct2, _gct3);
1022 }
1023
1024 void copy_load_at_8(Register dst, Address src) {
1025 _bs_asm->copy_load_at(_masm, _decorators, _type, 8,
1026 dst, noreg, src,
1027 _gct1);
1028 }
1029
1030 void copy_store_at_8(Address dst, Register src) {
1031 _bs_asm->copy_store_at(_masm, _decorators, _type, 8,
1032 dst, src, noreg,
1033 _gct1, _gct2, _gct3);
1034 }
1035 };
1036
1037 // Bulk copy of blocks of 8 words.
1038 //
1039 // count is a count of words.
1040 //
1041 // Precondition: count >= 8
1042 //
1043 // Postconditions:
1044 //
1045 // The least significant bit of count contains the remaining count
1046 // of words to copy. The rest of count is trash.
1047 //
1048 // s and d are adjusted to point to the remaining words to copy
1049 //
1050 address generate_copy_longs(StubId stub_id, DecoratorSet decorators, Register s, Register d, Register count) {
1051 int entry_count = StubInfo::entry_count(stub_id);
1052 assert(entry_count == 1, "sanity check");
1053 address start = load_archive_data(stub_id);
1054 if (start != nullptr) {
1055 return start;
1056 }
1057 BasicType type;
1058 copy_direction direction;
1059
1060 switch (stub_id) {
1061 case StubId::stubgen_copy_byte_f_id:
1062 direction = copy_forwards;
1063 type = T_BYTE;
1064 break;
1065 case StubId::stubgen_copy_byte_b_id:
1066 direction = copy_backwards;
1067 type = T_BYTE;
1068 break;
1069 case StubId::stubgen_copy_oop_f_id:
1070 direction = copy_forwards;
1071 type = T_OBJECT;
1072 break;
1073 case StubId::stubgen_copy_oop_b_id:
1074 direction = copy_backwards;
1075 type = T_OBJECT;
1076 break;
1077 case StubId::stubgen_copy_oop_uninit_f_id:
1078 direction = copy_forwards;
1079 type = T_OBJECT;
1080 break;
1081 case StubId::stubgen_copy_oop_uninit_b_id:
1082 direction = copy_backwards;
1083 type = T_OBJECT;
1084 break;
1085 default:
1086 ShouldNotReachHere();
1087 }
1088
1089 int unit = wordSize * direction;
1090 int bias = (UseSIMDForMemoryOps ? 4:2) * wordSize;
1091
1092 const Register t0 = r3, t1 = r4, t2 = r5, t3 = r6,
1093 t4 = r7, t5 = r11, t6 = r12, t7 = r13;
1094 const Register stride = r14;
1095 const Register gct1 = rscratch1, gct2 = rscratch2, gct3 = r10;
1096 const FloatRegister gcvt1 = v6, gcvt2 = v7, gcvt3 = v16; // Note that v8-v15 are callee saved
1097 ArrayCopyBarrierSetHelper bs(_masm, decorators, type, gct1, gct2, gct3, gcvt1, gcvt2, gcvt3);
1098
1099 assert_different_registers(rscratch1, rscratch2, t0, t1, t2, t3, t4, t5, t6, t7);
1100 assert_different_registers(s, d, count, rscratch1, rscratch2);
1101
1102 Label again, drain;
1103
1104 __ align(CodeEntryAlignment);
1105
1106 StubCodeMark mark(this, stub_id);
1107
1108 start = __ pc();
1109
1110 Label unaligned_copy_long;
1111 if (AvoidUnalignedAccesses) {
1112 __ tbnz(d, 3, unaligned_copy_long);
1113 }
1114
1115 if (direction == copy_forwards) {
1116 __ sub(s, s, bias);
1117 __ sub(d, d, bias);
1118 }
1119
1120 #ifdef ASSERT
1121 // Make sure we are never given < 8 words
1122 {
1123 Label L;
1124 __ cmp(count, (u1)8);
1125 __ br(Assembler::GE, L);
1126 __ stop("genrate_copy_longs called with < 8 words");
1127 __ bind(L);
1128 }
1129 #endif
1130
1131 // Fill 8 registers
1132 if (UseSIMDForMemoryOps) {
1133 bs.copy_load_at_32(v0, v1, Address(s, 4 * unit));
1134 bs.copy_load_at_32(v2, v3, Address(__ pre(s, 8 * unit)));
1135 } else {
1136 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1137 bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1138 bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1139 bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1140 }
1141
1142 __ subs(count, count, 16);
1143 __ br(Assembler::LO, drain);
1144
1145 int prefetch = PrefetchCopyIntervalInBytes;
1146 bool use_stride = false;
1147 if (direction == copy_backwards) {
1148 use_stride = prefetch > 256;
1149 prefetch = -prefetch;
1150 if (use_stride) __ mov(stride, prefetch);
1151 }
1152
1153 __ bind(again);
1154
1155 if (PrefetchCopyIntervalInBytes > 0)
1156 __ prfm(use_stride ? Address(s, stride) : Address(s, prefetch), PLDL1KEEP);
1157
1158 if (UseSIMDForMemoryOps) {
1159 bs.copy_store_at_32(Address(d, 4 * unit), v0, v1);
1160 bs.copy_load_at_32(v0, v1, Address(s, 4 * unit));
1161 bs.copy_store_at_32(Address(__ pre(d, 8 * unit)), v2, v3);
1162 bs.copy_load_at_32(v2, v3, Address(__ pre(s, 8 * unit)));
1163 } else {
1164 bs.copy_store_at_16(Address(d, 2 * unit), t0, t1);
1165 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1166 bs.copy_store_at_16(Address(d, 4 * unit), t2, t3);
1167 bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1168 bs.copy_store_at_16(Address(d, 6 * unit), t4, t5);
1169 bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1170 bs.copy_store_at_16(Address(__ pre(d, 8 * unit)), t6, t7);
1171 bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1172 }
1173
1174 __ subs(count, count, 8);
1175 __ br(Assembler::HS, again);
1176
1177 // Drain
1178 __ bind(drain);
1179 if (UseSIMDForMemoryOps) {
1180 bs.copy_store_at_32(Address(d, 4 * unit), v0, v1);
1181 bs.copy_store_at_32(Address(__ pre(d, 8 * unit)), v2, v3);
1182 } else {
1183 bs.copy_store_at_16(Address(d, 2 * unit), t0, t1);
1184 bs.copy_store_at_16(Address(d, 4 * unit), t2, t3);
1185 bs.copy_store_at_16(Address(d, 6 * unit), t4, t5);
1186 bs.copy_store_at_16(Address(__ pre(d, 8 * unit)), t6, t7);
1187 }
1188
1189 {
1190 Label L1, L2;
1191 __ tbz(count, exact_log2(4), L1);
1192 if (UseSIMDForMemoryOps) {
1193 bs.copy_load_at_32(v0, v1, Address(__ pre(s, 4 * unit)));
1194 bs.copy_store_at_32(Address(__ pre(d, 4 * unit)), v0, v1);
1195 } else {
1196 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1197 bs.copy_load_at_16(t2, t3, Address(__ pre(s, 4 * unit)));
1198 bs.copy_store_at_16(Address(d, 2 * unit), t0, t1);
1199 bs.copy_store_at_16(Address(__ pre(d, 4 * unit)), t2, t3);
1200 }
1201 __ bind(L1);
1202
1203 if (direction == copy_forwards) {
1204 __ add(s, s, bias);
1205 __ add(d, d, bias);
1206 }
1207
1208 __ tbz(count, 1, L2);
1209 bs.copy_load_at_16(t0, t1, Address(__ adjust(s, 2 * unit, direction == copy_backwards)));
1210 bs.copy_store_at_16(Address(__ adjust(d, 2 * unit, direction == copy_backwards)), t0, t1);
1211 __ bind(L2);
1212 }
1213
1214 __ ret(lr);
1215
1216 if (AvoidUnalignedAccesses) {
1217 Label drain, again;
1218 // Register order for storing. Order is different for backward copy.
1219
1220 __ bind(unaligned_copy_long);
1221
1222 // source address is even aligned, target odd aligned
1223 //
1224 // when forward copying word pairs we read long pairs at offsets
1225 // {0, 2, 4, 6} (in long words). when backwards copying we read
1226 // long pairs at offsets {-2, -4, -6, -8}. We adjust the source
1227 // address by -2 in the forwards case so we can compute the
1228 // source offsets for both as {2, 4, 6, 8} * unit where unit = 1
1229 // or -1.
1230 //
1231 // when forward copying we need to store 1 word, 3 pairs and
1232 // then 1 word at offsets {0, 1, 3, 5, 7}. Rather than use a
1233 // zero offset We adjust the destination by -1 which means we
1234 // have to use offsets { 1, 2, 4, 6, 8} * unit for the stores.
1235 //
1236 // When backwards copyng we need to store 1 word, 3 pairs and
1237 // then 1 word at offsets {-1, -3, -5, -7, -8} i.e. we use
1238 // offsets {1, 3, 5, 7, 8} * unit.
1239
1240 if (direction == copy_forwards) {
1241 __ sub(s, s, 16);
1242 __ sub(d, d, 8);
1243 }
1244
1245 // Fill 8 registers
1246 //
1247 // for forwards copy s was offset by -16 from the original input
1248 // value of s so the register contents are at these offsets
1249 // relative to the 64 bit block addressed by that original input
1250 // and so on for each successive 64 byte block when s is updated
1251 //
1252 // t0 at offset 0, t1 at offset 8
1253 // t2 at offset 16, t3 at offset 24
1254 // t4 at offset 32, t5 at offset 40
1255 // t6 at offset 48, t7 at offset 56
1256
1257 // for backwards copy s was not offset so the register contents
1258 // are at these offsets into the preceding 64 byte block
1259 // relative to that original input and so on for each successive
1260 // preceding 64 byte block when s is updated. this explains the
1261 // slightly counter-intuitive looking pattern of register usage
1262 // in the stp instructions for backwards copy.
1263 //
1264 // t0 at offset -16, t1 at offset -8
1265 // t2 at offset -32, t3 at offset -24
1266 // t4 at offset -48, t5 at offset -40
1267 // t6 at offset -64, t7 at offset -56
1268
1269 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1270 bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1271 bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1272 bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1273
1274 __ subs(count, count, 16);
1275 __ br(Assembler::LO, drain);
1276
1277 int prefetch = PrefetchCopyIntervalInBytes;
1278 bool use_stride = false;
1279 if (direction == copy_backwards) {
1280 use_stride = prefetch > 256;
1281 prefetch = -prefetch;
1282 if (use_stride) __ mov(stride, prefetch);
1283 }
1284
1285 __ bind(again);
1286
1287 if (PrefetchCopyIntervalInBytes > 0)
1288 __ prfm(use_stride ? Address(s, stride) : Address(s, prefetch), PLDL1KEEP);
1289
1290 if (direction == copy_forwards) {
1291 // allowing for the offset of -8 the store instructions place
1292 // registers into the target 64 bit block at the following
1293 // offsets
1294 //
1295 // t0 at offset 0
1296 // t1 at offset 8, t2 at offset 16
1297 // t3 at offset 24, t4 at offset 32
1298 // t5 at offset 40, t6 at offset 48
1299 // t7 at offset 56
1300
1301 bs.copy_store_at_8(Address(d, 1 * unit), t0);
1302 bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
1303 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1304 bs.copy_store_at_16(Address(d, 4 * unit), t3, t4);
1305 bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1306 bs.copy_store_at_16(Address(d, 6 * unit), t5, t6);
1307 bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1308 bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t7);
1309 bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1310 } else {
1311 // d was not offset when we started so the registers are
1312 // written into the 64 bit block preceding d with the following
1313 // offsets
1314 //
1315 // t1 at offset -8
1316 // t3 at offset -24, t0 at offset -16
1317 // t5 at offset -48, t2 at offset -32
1318 // t7 at offset -56, t4 at offset -48
1319 // t6 at offset -64
1320 //
1321 // note that this matches the offsets previously noted for the
1322 // loads
1323
1324 bs.copy_store_at_8(Address(d, 1 * unit), t1);
1325 bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
1326 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1327 bs.copy_store_at_16(Address(d, 5 * unit), t5, t2);
1328 bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1329 bs.copy_store_at_16(Address(d, 7 * unit), t7, t4);
1330 bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1331 bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t6);
1332 bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1333 }
1334
1335 __ subs(count, count, 8);
1336 __ br(Assembler::HS, again);
1337
1338 // Drain
1339 //
1340 // this uses the same pattern of offsets and register arguments
1341 // as above
1342 __ bind(drain);
1343 if (direction == copy_forwards) {
1344 bs.copy_store_at_8(Address(d, 1 * unit), t0);
1345 bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
1346 bs.copy_store_at_16(Address(d, 4 * unit), t3, t4);
1347 bs.copy_store_at_16(Address(d, 6 * unit), t5, t6);
1348 bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t7);
1349 } else {
1350 bs.copy_store_at_8(Address(d, 1 * unit), t1);
1351 bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
1352 bs.copy_store_at_16(Address(d, 5 * unit), t5, t2);
1353 bs.copy_store_at_16(Address(d, 7 * unit), t7, t4);
1354 bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t6);
1355 }
1356 // now we need to copy any remaining part block which may
1357 // include a 4 word block subblock and/or a 2 word subblock.
1358 // bits 2 and 1 in the count are the tell-tale for whether we
1359 // have each such subblock
1360 {
1361 Label L1, L2;
1362 __ tbz(count, exact_log2(4), L1);
1363 // this is the same as above but copying only 4 longs hence
1364 // with only one intervening stp between the str instructions
1365 // but note that the offsets and registers still follow the
1366 // same pattern
1367 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1368 bs.copy_load_at_16(t2, t3, Address(__ pre(s, 4 * unit)));
1369 if (direction == copy_forwards) {
1370 bs.copy_store_at_8(Address(d, 1 * unit), t0);
1371 bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
1372 bs.copy_store_at_8(Address(__ pre(d, 4 * unit)), t3);
1373 } else {
1374 bs.copy_store_at_8(Address(d, 1 * unit), t1);
1375 bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
1376 bs.copy_store_at_8(Address(__ pre(d, 4 * unit)), t2);
1377 }
1378 __ bind(L1);
1379
1380 __ tbz(count, 1, L2);
1381 // this is the same as above but copying only 2 longs hence
1382 // there is no intervening stp between the str instructions
1383 // but note that the offset and register patterns are still
1384 // the same
1385 bs.copy_load_at_16(t0, t1, Address(__ pre(s, 2 * unit)));
1386 if (direction == copy_forwards) {
1387 bs.copy_store_at_8(Address(d, 1 * unit), t0);
1388 bs.copy_store_at_8(Address(__ pre(d, 2 * unit)), t1);
1389 } else {
1390 bs.copy_store_at_8(Address(d, 1 * unit), t1);
1391 bs.copy_store_at_8(Address(__ pre(d, 2 * unit)), t0);
1392 }
1393 __ bind(L2);
1394
1395 // for forwards copy we need to re-adjust the offsets we
1396 // applied so that s and d are follow the last words written
1397
1398 if (direction == copy_forwards) {
1399 __ add(s, s, 16);
1400 __ add(d, d, 8);
1401 }
1402
1403 }
1404
1405 __ ret(lr);
1406 }
1407
1408 // record the stub entry and end
1409 store_archive_data(stub_id, start, __ pc());
1410
1411 return start;
1412 }
1413
1414 // Small copy: less than 16 bytes.
1415 //
1416 // NB: Ignores all of the bits of count which represent more than 15
1417 // bytes, so a caller doesn't have to mask them.
1418
1419 void copy_memory_small(DecoratorSet decorators, BasicType type, Register s, Register d, Register count, int step) {
1420 bool is_backwards = step < 0;
1421 size_t granularity = g_uabs(step);
1422 int direction = is_backwards ? -1 : 1;
1423
1424 Label Lword, Lint, Lshort, Lbyte;
1425
1426 assert(granularity
1427 && granularity <= sizeof (jlong), "Impossible granularity in copy_memory_small");
1428
1429 const Register t0 = r3;
1430 const Register gct1 = rscratch1, gct2 = rscratch2, gct3 = r10;
1431 ArrayCopyBarrierSetHelper bs(_masm, decorators, type, gct1, gct2, gct3, fnoreg, fnoreg, fnoreg);
1432
1433 // ??? I don't know if this bit-test-and-branch is the right thing
1434 // to do. It does a lot of jumping, resulting in several
1435 // mispredicted branches. It might make more sense to do this
1436 // with something like Duff's device with a single computed branch.
1437
1438 __ tbz(count, 3 - exact_log2(granularity), Lword);
1439 bs.copy_load_at_8(t0, Address(__ adjust(s, direction * wordSize, is_backwards)));
1440 bs.copy_store_at_8(Address(__ adjust(d, direction * wordSize, is_backwards)), t0);
1441 __ bind(Lword);
1442
1443 if (granularity <= sizeof (jint)) {
1444 __ tbz(count, 2 - exact_log2(granularity), Lint);
1445 __ ldrw(t0, Address(__ adjust(s, sizeof (jint) * direction, is_backwards)));
1446 __ strw(t0, Address(__ adjust(d, sizeof (jint) * direction, is_backwards)));
1447 __ bind(Lint);
1448 }
1449
1450 if (granularity <= sizeof (jshort)) {
1451 __ tbz(count, 1 - exact_log2(granularity), Lshort);
1452 __ ldrh(t0, Address(__ adjust(s, sizeof (jshort) * direction, is_backwards)));
1453 __ strh(t0, Address(__ adjust(d, sizeof (jshort) * direction, is_backwards)));
1454 __ bind(Lshort);
1455 }
1456
1457 if (granularity <= sizeof (jbyte)) {
1458 __ tbz(count, 0, Lbyte);
1459 __ ldrb(t0, Address(__ adjust(s, sizeof (jbyte) * direction, is_backwards)));
1460 __ strb(t0, Address(__ adjust(d, sizeof (jbyte) * direction, is_backwards)));
1461 __ bind(Lbyte);
1462 }
1463 }
1464
1465 // All-singing all-dancing memory copy.
1466 //
1467 // Copy count units of memory from s to d. The size of a unit is
1468 // step, which can be positive or negative depending on the direction
1469 // of copy. If is_aligned is false, we align the source address.
1470 //
1471
1472 void copy_memory(DecoratorSet decorators, BasicType type, bool is_aligned,
1473 Register s, Register d, Register count, int step) {
1474 copy_direction direction = step < 0 ? copy_backwards : copy_forwards;
1475 bool is_backwards = step < 0;
1476 unsigned int granularity = g_uabs(step);
1477 const Register t0 = r3, t1 = r4;
1478
1479 // <= 80 (or 96 for SIMD) bytes do inline. Direction doesn't matter because we always
1480 // load all the data before writing anything
1481 Label copy4, copy8, copy16, copy32, copy80, copy_big, finish;
1482 const Register t2 = r5, t3 = r6, t4 = r7, t5 = r11;
1483 const Register t6 = r12, t7 = r13, t8 = r14, t9 = r15;
1484 const Register send = r17, dend = r16;
1485 const Register gct1 = rscratch1, gct2 = rscratch2, gct3 = r10;
1486 const FloatRegister gcvt1 = v6, gcvt2 = v7, gcvt3 = v16; // Note that v8-v15 are callee saved
1487 ArrayCopyBarrierSetHelper bs(_masm, decorators, type, gct1, gct2, gct3, gcvt1, gcvt2, gcvt3);
1488
1489 if (PrefetchCopyIntervalInBytes > 0)
1490 __ prfm(Address(s, 0), PLDL1KEEP);
1491 __ cmp(count, u1((UseSIMDForMemoryOps ? 96:80)/granularity));
1492 __ br(Assembler::HI, copy_big);
1493
1494 __ lea(send, Address(s, count, Address::lsl(exact_log2(granularity))));
1495 __ lea(dend, Address(d, count, Address::lsl(exact_log2(granularity))));
1496
1497 __ cmp(count, u1(16/granularity));
1498 __ br(Assembler::LS, copy16);
1499
1500 __ cmp(count, u1(64/granularity));
1501 __ br(Assembler::HI, copy80);
1502
1503 __ cmp(count, u1(32/granularity));
1504 __ br(Assembler::LS, copy32);
1505
1506 // 33..64 bytes
1507 if (UseSIMDForMemoryOps) {
1508 bs.copy_load_at_32(v0, v1, Address(s, 0));
1509 bs.copy_load_at_32(v2, v3, Address(send, -32));
1510 bs.copy_store_at_32(Address(d, 0), v0, v1);
1511 bs.copy_store_at_32(Address(dend, -32), v2, v3);
1512 } else {
1513 bs.copy_load_at_16(t0, t1, Address(s, 0));
1514 bs.copy_load_at_16(t2, t3, Address(s, 16));
1515 bs.copy_load_at_16(t4, t5, Address(send, -32));
1516 bs.copy_load_at_16(t6, t7, Address(send, -16));
1517
1518 bs.copy_store_at_16(Address(d, 0), t0, t1);
1519 bs.copy_store_at_16(Address(d, 16), t2, t3);
1520 bs.copy_store_at_16(Address(dend, -32), t4, t5);
1521 bs.copy_store_at_16(Address(dend, -16), t6, t7);
1522 }
1523 __ b(finish);
1524
1525 // 17..32 bytes
1526 __ bind(copy32);
1527 bs.copy_load_at_16(t0, t1, Address(s, 0));
1528 bs.copy_load_at_16(t6, t7, Address(send, -16));
1529
1530 bs.copy_store_at_16(Address(d, 0), t0, t1);
1531 bs.copy_store_at_16(Address(dend, -16), t6, t7);
1532 __ b(finish);
1533
1534 // 65..80/96 bytes
1535 // (96 bytes if SIMD because we do 32 byes per instruction)
1536 __ bind(copy80);
1537 if (UseSIMDForMemoryOps) {
1538 bs.copy_load_at_32(v0, v1, Address(s, 0));
1539 bs.copy_load_at_32(v2, v3, Address(s, 32));
1540 // Unaligned pointers can be an issue for copying.
1541 // The issue has more chances to happen when granularity of data is
1542 // less than 4(sizeof(jint)). Pointers for arrays of jint are at least
1543 // 4 byte aligned. Pointers for arrays of jlong are 8 byte aligned.
1544 // The most performance drop has been seen for the range 65-80 bytes.
1545 // For such cases using the pair of ldp/stp instead of the third pair of
1546 // ldpq/stpq fixes the performance issue.
1547 if (granularity < sizeof (jint)) {
1548 Label copy96;
1549 __ cmp(count, u1(80/granularity));
1550 __ br(Assembler::HI, copy96);
1551 bs.copy_load_at_16(t0, t1, Address(send, -16));
1552
1553 bs.copy_store_at_32(Address(d, 0), v0, v1);
1554 bs.copy_store_at_32(Address(d, 32), v2, v3);
1555
1556 bs.copy_store_at_16(Address(dend, -16), t0, t1);
1557 __ b(finish);
1558
1559 __ bind(copy96);
1560 }
1561 bs.copy_load_at_32(v4, v5, Address(send, -32));
1562
1563 bs.copy_store_at_32(Address(d, 0), v0, v1);
1564 bs.copy_store_at_32(Address(d, 32), v2, v3);
1565
1566 bs.copy_store_at_32(Address(dend, -32), v4, v5);
1567 } else {
1568 bs.copy_load_at_16(t0, t1, Address(s, 0));
1569 bs.copy_load_at_16(t2, t3, Address(s, 16));
1570 bs.copy_load_at_16(t4, t5, Address(s, 32));
1571 bs.copy_load_at_16(t6, t7, Address(s, 48));
1572 bs.copy_load_at_16(t8, t9, Address(send, -16));
1573
1574 bs.copy_store_at_16(Address(d, 0), t0, t1);
1575 bs.copy_store_at_16(Address(d, 16), t2, t3);
1576 bs.copy_store_at_16(Address(d, 32), t4, t5);
1577 bs.copy_store_at_16(Address(d, 48), t6, t7);
1578 bs.copy_store_at_16(Address(dend, -16), t8, t9);
1579 }
1580 __ b(finish);
1581
1582 // 0..16 bytes
1583 __ bind(copy16);
1584 __ cmp(count, u1(8/granularity));
1585 __ br(Assembler::LO, copy8);
1586
1587 // 8..16 bytes
1588 bs.copy_load_at_8(t0, Address(s, 0));
1589 bs.copy_load_at_8(t1, Address(send, -8));
1590 bs.copy_store_at_8(Address(d, 0), t0);
1591 bs.copy_store_at_8(Address(dend, -8), t1);
1592 __ b(finish);
1593
1594 if (granularity < 8) {
1595 // 4..7 bytes
1596 __ bind(copy8);
1597 __ tbz(count, 2 - exact_log2(granularity), copy4);
1598 __ ldrw(t0, Address(s, 0));
1599 __ ldrw(t1, Address(send, -4));
1600 __ strw(t0, Address(d, 0));
1601 __ strw(t1, Address(dend, -4));
1602 __ b(finish);
1603 if (granularity < 4) {
1604 // 0..3 bytes
1605 __ bind(copy4);
1606 __ cbz(count, finish); // get rid of 0 case
1607 if (granularity == 2) {
1608 __ ldrh(t0, Address(s, 0));
1609 __ strh(t0, Address(d, 0));
1610 } else { // granularity == 1
1611 // Now 1..3 bytes. Handle the 1 and 2 byte case by copying
1612 // the first and last byte.
1613 // Handle the 3 byte case by loading and storing base + count/2
1614 // (count == 1 (s+0)->(d+0), count == 2,3 (s+1) -> (d+1))
1615 // This does means in the 1 byte case we load/store the same
1616 // byte 3 times.
1617 __ lsr(count, count, 1);
1618 __ ldrb(t0, Address(s, 0));
1619 __ ldrb(t1, Address(send, -1));
1620 __ ldrb(t2, Address(s, count));
1621 __ strb(t0, Address(d, 0));
1622 __ strb(t1, Address(dend, -1));
1623 __ strb(t2, Address(d, count));
1624 }
1625 __ b(finish);
1626 }
1627 }
1628
1629 __ bind(copy_big);
1630 if (is_backwards) {
1631 __ lea(s, Address(s, count, Address::lsl(exact_log2(-step))));
1632 __ lea(d, Address(d, count, Address::lsl(exact_log2(-step))));
1633 }
1634
1635 // Now we've got the small case out of the way we can align the
1636 // source address on a 2-word boundary.
1637
1638 // Here we will materialize a count in r15, which is used by copy_memory_small
1639 // and the various generate_copy_longs stubs that we use for 2 word aligned bytes.
1640 // Up until here, we have used t9, which aliases r15, but from here on, that register
1641 // can not be used as a temp register, as it contains the count.
1642
1643 Label aligned;
1644
1645 if (is_aligned) {
1646 // We may have to adjust by 1 word to get s 2-word-aligned.
1647 __ tbz(s, exact_log2(wordSize), aligned);
1648 bs.copy_load_at_8(t0, Address(__ adjust(s, direction * wordSize, is_backwards)));
1649 bs.copy_store_at_8(Address(__ adjust(d, direction * wordSize, is_backwards)), t0);
1650 __ sub(count, count, wordSize/granularity);
1651 } else {
1652 if (is_backwards) {
1653 __ andr(r15, s, 2 * wordSize - 1);
1654 } else {
1655 __ neg(r15, s);
1656 __ andr(r15, r15, 2 * wordSize - 1);
1657 }
1658 // r15 is the byte adjustment needed to align s.
1659 __ cbz(r15, aligned);
1660 int shift = exact_log2(granularity);
1661 if (shift > 0) {
1662 __ lsr(r15, r15, shift);
1663 }
1664 __ sub(count, count, r15);
1665
1666 #if 0
1667 // ?? This code is only correct for a disjoint copy. It may or
1668 // may not make sense to use it in that case.
1669
1670 // Copy the first pair; s and d may not be aligned.
1671 __ ldp(t0, t1, Address(s, is_backwards ? -2 * wordSize : 0));
1672 __ stp(t0, t1, Address(d, is_backwards ? -2 * wordSize : 0));
1673
1674 // Align s and d, adjust count
1675 if (is_backwards) {
1676 __ sub(s, s, r15);
1677 __ sub(d, d, r15);
1678 } else {
1679 __ add(s, s, r15);
1680 __ add(d, d, r15);
1681 }
1682 #else
1683 copy_memory_small(decorators, type, s, d, r15, step);
1684 #endif
1685 }
1686
1687 __ bind(aligned);
1688
1689 // s is now 2-word-aligned.
1690
1691 // We have a count of units and some trailing bytes. Adjust the
1692 // count and do a bulk copy of words. If the shift is zero
1693 // perform a move instead to benefit from zero latency moves.
1694 int shift = exact_log2(wordSize/granularity);
1695 if (shift > 0) {
1696 __ lsr(r15, count, shift);
1697 } else {
1698 __ mov(r15, count);
1699 }
1700 if (direction == copy_forwards) {
1701 if (type != T_OBJECT) {
1702 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_byte_f()));
1703 __ blr(rscratch1);
1704 } else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
1705 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_uninit_f()));
1706 __ blr(rscratch1);
1707 } else {
1708 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_f()));
1709 __ blr(rscratch1);
1710 }
1711 } else {
1712 if (type != T_OBJECT) {
1713 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_byte_b()));
1714 __ blr(rscratch1);
1715 } else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
1716 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_uninit_b()));
1717 __ blr(rscratch1);
1718 } else {
1719 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_b()));
1720 __ blr(rscratch1);
1721 }
1722 }
1723
1724 // And the tail.
1725 copy_memory_small(decorators, type, s, d, count, step);
1726
1727 if (granularity >= 8) __ bind(copy8);
1728 if (granularity >= 4) __ bind(copy4);
1729 __ bind(finish);
1730 }
1731
1732
1733 void clobber_registers() {
1734 #ifdef ASSERT
1735 RegSet clobbered
1736 = MacroAssembler::call_clobbered_gp_registers() - rscratch1;
1737 __ mov(rscratch1, (uint64_t)0xdeadbeef);
1738 __ orr(rscratch1, rscratch1, rscratch1, Assembler::LSL, 32);
1739 for (RegSetIterator<Register> it = clobbered.begin(); *it != noreg; ++it) {
1740 __ mov(*it, rscratch1);
1741 }
1742 #endif
1743
1744 }
1745
1746 // Scan over array at a for count oops, verifying each one.
1747 // Preserves a and count, clobbers rscratch1 and rscratch2.
1748 void verify_oop_array (int size, Register a, Register count, Register temp) {
1749 Label loop, end;
1750 __ mov(rscratch1, a);
1751 __ mov(rscratch2, zr);
1752 __ bind(loop);
1753 __ cmp(rscratch2, count);
1754 __ br(Assembler::HS, end);
1755 if (size == wordSize) {
1756 __ ldr(temp, Address(a, rscratch2, Address::lsl(exact_log2(size))));
1757 __ verify_oop(temp);
1758 } else {
1759 __ ldrw(temp, Address(a, rscratch2, Address::lsl(exact_log2(size))));
1760 __ decode_heap_oop(temp); // calls verify_oop
1761 }
1762 __ add(rscratch2, rscratch2, 1);
1763 __ b(loop);
1764 __ bind(end);
1765 }
1766
1767 // Arguments:
1768 // stub_id - is used to name the stub and identify all details of
1769 // how to perform the copy.
1770 //
1771 // nopush_entry - is assigned to the stub's post push entry point
1772 // unless it is null
1773 //
1774 // Inputs:
1775 // c_rarg0 - source array address
1776 // c_rarg1 - destination array address
1777 // c_rarg2 - element count, treated as ssize_t, can be zero
1778 //
1779 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1780 // the hardware handle it. The two dwords within qwords that span
1781 // cache line boundaries will still be loaded and stored atomically.
1782 //
1783 // Side Effects: nopush_entry is set to the (post push) entry point
1784 // so it can be used by the corresponding conjoint
1785 // copy method
1786 //
1787 address generate_disjoint_copy(StubId stub_id, address *nopush_entry) {
1788 int size;
1789 bool aligned;
1790 bool is_oop;
1791 bool dest_uninitialized;
1792 switch (stub_id) {
1793 case StubId::stubgen_jbyte_disjoint_arraycopy_id:
1794 size = sizeof(jbyte);
1795 aligned = false;
1796 is_oop = false;
1797 dest_uninitialized = false;
1798 break;
1799 case StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id:
1800 size = sizeof(jbyte);
1801 aligned = true;
1802 is_oop = false;
1803 dest_uninitialized = false;
1804 break;
1805 case StubId::stubgen_jshort_disjoint_arraycopy_id:
1806 size = sizeof(jshort);
1807 aligned = false;
1808 is_oop = false;
1809 dest_uninitialized = false;
1810 break;
1811 case StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id:
1812 size = sizeof(jshort);
1813 aligned = true;
1814 is_oop = false;
1815 dest_uninitialized = false;
1816 break;
1817 case StubId::stubgen_jint_disjoint_arraycopy_id:
1818 size = sizeof(jint);
1819 aligned = false;
1820 is_oop = false;
1821 dest_uninitialized = false;
1822 break;
1823 case StubId::stubgen_arrayof_jint_disjoint_arraycopy_id:
1824 size = sizeof(jint);
1825 aligned = true;
1826 is_oop = false;
1827 dest_uninitialized = false;
1828 break;
1829 case StubId::stubgen_jlong_disjoint_arraycopy_id:
1830 // since this is always aligned we can (should!) use the same
1831 // stub as for case StubId::stubgen_arrayof_jlong_disjoint_arraycopy
1832 ShouldNotReachHere();
1833 break;
1834 case StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id:
1835 size = sizeof(jlong);
1836 aligned = true;
1837 is_oop = false;
1838 dest_uninitialized = false;
1839 break;
1840 case StubId::stubgen_oop_disjoint_arraycopy_id:
1841 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1842 aligned = !UseCompressedOops;
1843 is_oop = true;
1844 dest_uninitialized = false;
1845 break;
1846 case StubId::stubgen_arrayof_oop_disjoint_arraycopy_id:
1847 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1848 aligned = !UseCompressedOops;
1849 is_oop = true;
1850 dest_uninitialized = false;
1851 break;
1852 case StubId::stubgen_oop_disjoint_arraycopy_uninit_id:
1853 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1854 aligned = !UseCompressedOops;
1855 is_oop = true;
1856 dest_uninitialized = true;
1857 break;
1858 case StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id:
1859 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1860 aligned = !UseCompressedOops;
1861 is_oop = true;
1862 dest_uninitialized = true;
1863 break;
1864 default:
1865 ShouldNotReachHere();
1866 break;
1867 }
1868 // all stubs provide a 2nd entry which omits the frame push for
1869 // use when bailing out from a conjoint copy. However we may also
1870 // need some extra addressses for memory access protection.
1871 int entry_count = StubInfo::entry_count(stub_id);
1872 assert(entry_count == 2, "sanity check");
1873 assert(nopush_entry != nullptr, "all disjoint copy stubs export a nopush entry");
1874
1875 bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size);
1876 int extra_count = ((add_extras ? 1 : 0) * UnsafeMemoryAccess::COLUMN_COUNT);
1877 GrowableArray<address> entries;
1878 GrowableArray<address> extras;
1879 GrowableArray<address> *extras_ptr = (extra_count > 0 ? &extras : nullptr);
1880 address start = load_archive_data(stub_id, &entries, extras_ptr);
1881 if (start != nullptr) {
1882 assert(entries.length() == entry_count - 1,
1883 "unexpected entries count %d", entries.length());
1884 *nopush_entry = entries.at(0);
1885 assert(extras.length() == extra_count,
1886 "unexpected extra count %d", extras.length());
1887 if (add_extras) {
1888 // register one handler at offset 0
1889 register_unsafe_access_handlers(extras, 0, 1);
1890 }
1891 return start;
1892 }
1893
1894 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1895 RegSet saved_reg = RegSet::of(s, d, count);
1896
1897 __ align(CodeEntryAlignment);
1898 StubCodeMark mark(this, stub_id);
1899 start = __ pc();
1900 __ enter();
1901
1902 *nopush_entry = __ pc();
1903 entries.append(*nopush_entry);
1904
1905 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1906 BLOCK_COMMENT("Post-Push Entry:");
1907
1908 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
1909 if (dest_uninitialized) {
1910 decorators |= IS_DEST_UNINITIALIZED;
1911 }
1912 if (aligned) {
1913 decorators |= ARRAYCOPY_ALIGNED;
1914 }
1915
1916 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1917 bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_reg);
1918
1919 if (is_oop) {
1920 // save regs before copy_memory
1921 __ push(RegSet::of(d, count), sp);
1922 }
1923 {
1924 // UnsafeMemoryAccess page error: continue after unsafe access
1925 UnsafeMemoryAccessMark umam(this, add_extras, true);
1926 copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size);
1927 }
1928
1929 if (is_oop) {
1930 __ pop(RegSet::of(d, count), sp);
1931 if (VerifyOops)
1932 verify_oop_array(size, d, count, r16);
1933 }
1934
1935 bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1);
1936
1937 __ leave();
1938 __ mov(r0, zr); // return 0
1939 __ ret(lr);
1940
1941 address end = __ pc();
1942
1943 if (add_extras) {
1944 // retrieve the registered handler addresses
1945 retrieve_unsafe_access_handlers(start, end, extras);
1946 assert(extras.length() == extra_count
1947 , "incorrect handlers count %d", extras.length());
1948 }
1949
1950 // record the stub entry and end plus the no_push entry and any
1951 // extra handler addresses
1952 store_archive_data(stub_id, start, end, &entries, extras_ptr);
1953
1954 return start;
1955 }
1956
1957 // Arguments:
1958 // stub_id - is used to name the stub and identify all details of
1959 // how to perform the copy.
1960 //
1961 // nooverlap_target - identifes the (post push) entry for the
1962 // corresponding disjoint copy routine which can be
1963 // jumped to if the ranges do not actually overlap
1964 //
1965 // nopush_entry - is assigned to the stub's post push entry point
1966 // unless it is null
1967 //
1968 //
1969 // Inputs:
1970 // c_rarg0 - source array address
1971 // c_rarg1 - destination array address
1972 // c_rarg2 - element count, treated as ssize_t, can be zero
1973 //
1974 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1975 // the hardware handle it. The two dwords within qwords that span
1976 // cache line boundaries will still be loaded and stored atomically.
1977 //
1978 // Side Effects:
1979 // nopush_entry is set to the no-overlap entry point so it can be
1980 // used by some other conjoint copy method
1981 //
1982 address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) {
1983 int size;
1984 bool aligned;
1985 bool is_oop;
1986 bool dest_uninitialized;
1987 switch (stub_id) {
1988 case StubId::stubgen_jbyte_arraycopy_id:
1989 size = sizeof(jbyte);
1990 aligned = false;
1991 is_oop = false;
1992 dest_uninitialized = false;
1993 break;
1994 case StubId::stubgen_arrayof_jbyte_arraycopy_id:
1995 size = sizeof(jbyte);
1996 aligned = true;
1997 is_oop = false;
1998 dest_uninitialized = false;
1999 break;
2000 case StubId::stubgen_jshort_arraycopy_id:
2001 size = sizeof(jshort);
2002 aligned = false;
2003 is_oop = false;
2004 dest_uninitialized = false;
2005 break;
2006 case StubId::stubgen_arrayof_jshort_arraycopy_id:
2007 size = sizeof(jshort);
2008 aligned = true;
2009 is_oop = false;
2010 dest_uninitialized = false;
2011 break;
2012 case StubId::stubgen_jint_arraycopy_id:
2013 size = sizeof(jint);
2014 aligned = false;
2015 is_oop = false;
2016 dest_uninitialized = false;
2017 break;
2018 case StubId::stubgen_arrayof_jint_arraycopy_id:
2019 size = sizeof(jint);
2020 aligned = true;
2021 is_oop = false;
2022 dest_uninitialized = false;
2023 break;
2024 case StubId::stubgen_jlong_arraycopy_id:
2025 // since this is always aligned we can (should!) use the same
2026 // stub as for case StubId::stubgen_arrayof_jlong_disjoint_arraycopy
2027 ShouldNotReachHere();
2028 break;
2029 case StubId::stubgen_arrayof_jlong_arraycopy_id:
2030 size = sizeof(jlong);
2031 aligned = true;
2032 is_oop = false;
2033 dest_uninitialized = false;
2034 break;
2035 case StubId::stubgen_oop_arraycopy_id:
2036 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
2037 aligned = !UseCompressedOops;
2038 is_oop = true;
2039 dest_uninitialized = false;
2040 break;
2041 case StubId::stubgen_arrayof_oop_arraycopy_id:
2042 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
2043 aligned = !UseCompressedOops;
2044 is_oop = true;
2045 dest_uninitialized = false;
2046 break;
2047 case StubId::stubgen_oop_arraycopy_uninit_id:
2048 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
2049 aligned = !UseCompressedOops;
2050 is_oop = true;
2051 dest_uninitialized = true;
2052 break;
2053 case StubId::stubgen_arrayof_oop_arraycopy_uninit_id:
2054 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
2055 aligned = !UseCompressedOops;
2056 is_oop = true;
2057 dest_uninitialized = true;
2058 break;
2059 default:
2060 ShouldNotReachHere();
2061 }
2062 // only some conjoint stubs generate a 2nd entry
2063 int entry_count = StubInfo::entry_count(stub_id);
2064 int expected_entry_count = (nopush_entry == nullptr ? 1 : 2);
2065 assert(entry_count == expected_entry_count,
2066 "expected entry count %d does not match declared entry count %d for stub %s",
2067 expected_entry_count, entry_count, StubInfo::name(stub_id));
2068
2069 // We need to protect memory accesses in certain cases
2070 bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size);
2071 int extra_count = ((add_extras ? 1 : 0) * UnsafeMemoryAccess::COLUMN_COUNT);
2072 GrowableArray<address> entries;
2073 GrowableArray<address> extras;
2074 GrowableArray<address> *entries_ptr = (nopush_entry != nullptr ? &entries : nullptr);
2075 GrowableArray<address> *extras_ptr = (extra_count > 0 ? &extras : nullptr);
2076 address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
2077 if (start != nullptr) {
2078 assert(entries.length() == expected_entry_count - 1,
2079 "unexpected entries count %d", entries.length());
2080 assert(extras.length() == extra_count,
2081 "unexpected extra count %d", extras.length());
2082 if (nopush_entry != nullptr) {
2083 *nopush_entry = entries.at(0);
2084 }
2085 if (add_extras) {
2086 // register one handler at offset 0
2087 register_unsafe_access_handlers(extras, 0, 1);
2088 }
2089 return start;
2090 }
2091
2092 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
2093 RegSet saved_regs = RegSet::of(s, d, count);
2094 StubCodeMark mark(this, stub_id);
2095 start = __ pc();
2096 __ enter();
2097
2098 if (nopush_entry != nullptr) {
2099 *nopush_entry = __ pc();
2100 entries.append(*nopush_entry);
2101 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2102 BLOCK_COMMENT("Post-Push Entry:");
2103 }
2104
2105 // use fwd copy when (d-s) above_equal (count*size)
2106 Label L_overlapping;
2107 __ sub(rscratch1, d, s);
2108 __ cmp(rscratch1, count, Assembler::LSL, exact_log2(size));
2109 __ br(Assembler::LO, L_overlapping);
2110 __ b(RuntimeAddress(nooverlap_target));
2111 __ bind(L_overlapping);
2112
2113 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
2114 if (dest_uninitialized) {
2115 decorators |= IS_DEST_UNINITIALIZED;
2116 }
2117 if (aligned) {
2118 decorators |= ARRAYCOPY_ALIGNED;
2119 }
2120
2121 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2122 bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_regs);
2123
2124 if (is_oop) {
2125 // save regs before copy_memory
2126 __ push(RegSet::of(d, count), sp);
2127 }
2128 {
2129 // UnsafeMemoryAccess page error: continue after unsafe access
2130 UnsafeMemoryAccessMark umam(this, add_extras, true);
2131 copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size);
2132 }
2133 if (is_oop) {
2134 __ pop(RegSet::of(d, count), sp);
2135 if (VerifyOops)
2136 verify_oop_array(size, d, count, r16);
2137 }
2138 bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1);
2139 __ leave();
2140 __ mov(r0, zr); // return 0
2141 __ ret(lr);
2142
2143 assert(entries.length() == expected_entry_count - 1,
2144 "unexpected entries count %d", entries.length());
2145
2146 address end = __ pc();
2147
2148 if (add_extras) {
2149 // retrieve the registered handler addresses
2150 retrieve_unsafe_access_handlers(start, end, extras);
2151 assert(extras.length() == extra_count,
2152 "incorrect handlers count %d", extras.length());
2153 }
2154
2155 // record the stub entry and end plus any no_push entry and/or
2156 // extra handler addresses
2157 store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
2158
2159 return start;
2160 }
2161
2162 // Helper for generating a dynamic type check.
2163 // Smashes rscratch1, rscratch2.
2164 void generate_type_check(Register sub_klass,
2165 Register super_check_offset,
2166 Register super_klass,
2167 Register temp1,
2168 Register temp2,
2169 Register result,
2170 Label& L_success) {
2171 assert_different_registers(sub_klass, super_check_offset, super_klass);
2172
2173 BLOCK_COMMENT("type_check:");
2174
2175 Label L_miss;
2176
2177 __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, nullptr,
2178 super_check_offset);
2179 __ check_klass_subtype_slow_path(sub_klass, super_klass, temp1, temp2, &L_success, nullptr);
2180
2181 // Fall through on failure!
2182 __ BIND(L_miss);
2183 }
2184
2185 //
2186 // Generate checkcasting array copy stub
2187 //
2188 // Input:
2189 // c_rarg0 - source array address
2190 // c_rarg1 - destination array address
2191 // c_rarg2 - element count, treated as ssize_t, can be zero
2192 // c_rarg3 - size_t ckoff (super_check_offset)
2193 // c_rarg4 - oop ckval (super_klass)
2194 //
2195 // Output:
2196 // r0 == 0 - success
2197 // r0 == -1^K - failure, where K is partial transfer count
2198 //
2199 address generate_checkcast_copy(StubId stub_id, address *nopush_entry) {
2200 bool dest_uninitialized;
2201 switch (stub_id) {
2202 case StubId::stubgen_checkcast_arraycopy_id:
2203 dest_uninitialized = false;
2204 break;
2205 case StubId::stubgen_checkcast_arraycopy_uninit_id:
2206 dest_uninitialized = true;
2207 break;
2208 default:
2209 ShouldNotReachHere();
2210 }
2211
2212 // The normal stub provides a 2nd entry which omits the frame push
2213 // for use when bailing out from a disjoint copy.
2214 // Only some conjoint stubs generate a 2nd entry
2215 int entry_count = StubInfo::entry_count(stub_id);
2216 int expected_entry_count = (nopush_entry == nullptr ? 1 : 2);
2217 GrowableArray<address> entries;
2218 GrowableArray<address> *entries_ptr = (expected_entry_count == 1 ? nullptr : &entries);
2219 assert(entry_count == expected_entry_count,
2220 "expected entry count %d does not match declared entry count %d for stub %s",
2221 expected_entry_count, entry_count, StubInfo::name(stub_id));
2222 address start = load_archive_data(stub_id, entries_ptr);
2223 if (start != nullptr) {
2224 assert(entries.length() + 1 == expected_entry_count,
2225 "expected entry count %d does not match return entry count %d for stub %s",
2226 expected_entry_count, entries.length() + 1, StubInfo::name(stub_id));
2227 if (nopush_entry != nullptr) {
2228 *nopush_entry = entries.at(0);
2229 }
2230 return start;
2231 }
2232
2233 Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop;
2234
2235 // Input registers (after setup_arg_regs)
2236 const Register from = c_rarg0; // source array address
2237 const Register to = c_rarg1; // destination array address
2238 const Register count = c_rarg2; // elementscount
2239 const Register ckoff = c_rarg3; // super_check_offset
2240 const Register ckval = c_rarg4; // super_klass
2241
2242 RegSet wb_pre_saved_regs = RegSet::range(c_rarg0, c_rarg4);
2243
2244 // Registers used as temps (r19, r20, r21, r22 are save-on-entry)
2245 const Register copied_oop = r22; // actual oop copied
2246 const Register count_save = r21; // orig elementscount
2247 const Register start_to = r20; // destination array start address
2248 const Register r19_klass = r19; // oop._klass
2249
2250 // Registers used as gc temps (r5, r6, r7 are save-on-call)
2251 const Register gct1 = r5, gct2 = r6, gct3 = r7;
2252
2253 //---------------------------------------------------------------
2254 // Assembler stub will be used for this call to arraycopy
2255 // if the two arrays are subtypes of Object[] but the
2256 // destination array type is not equal to or a supertype
2257 // of the source type. Each element must be separately
2258 // checked.
2259
2260 assert_different_registers(from, to, count, ckoff, ckval, start_to,
2261 copied_oop, r19_klass, count_save, rscratch1);
2262
2263 __ align(CodeEntryAlignment);
2264 StubCodeMark mark(this, stub_id);
2265 start = __ pc();
2266
2267 __ enter(); // required for proper stackwalking of RuntimeStub frame
2268
2269 #ifdef ASSERT
2270 // caller guarantees that the arrays really are different
2271 // otherwise, we would have to make conjoint checks
2272 { Label L;
2273 __ b(L); // conjoint check not yet implemented
2274 __ stop("checkcast_copy within a single array");
2275 __ bind(L);
2276 }
2277 #endif //ASSERT
2278
2279 // Caller of this entry point must set up the argument registers.
2280 if (nopush_entry != nullptr) {
2281 *nopush_entry = __ pc();
2282 entries.append(*nopush_entry);
2283 BLOCK_COMMENT("Entry:");
2284 }
2285
2286 // Empty array: Nothing to do.
2287 __ cbz(count, L_done);
2288 __ push(RegSet::of(r19, r20, r21, r22), sp);
2289
2290 #ifdef ASSERT
2291 BLOCK_COMMENT("assert consistent ckoff/ckval");
2292 // The ckoff and ckval must be mutually consistent,
2293 // even though caller generates both.
2294 { Label L;
2295 int sco_offset = in_bytes(Klass::super_check_offset_offset());
2296 __ ldrw(start_to, Address(ckval, sco_offset));
2297 __ cmpw(ckoff, start_to);
2298 __ br(Assembler::EQ, L);
2299 __ stop("super_check_offset inconsistent");
2300 __ bind(L);
2301 }
2302 #endif //ASSERT
2303
2304 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT;
2305 bool is_oop = true;
2306 int element_size = UseCompressedOops ? 4 : 8;
2307 if (dest_uninitialized) {
2308 decorators |= IS_DEST_UNINITIALIZED;
2309 }
2310
2311 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2312 bs->arraycopy_prologue(_masm, decorators, is_oop, from, to, count, wb_pre_saved_regs);
2313
2314 // save the original count
2315 __ mov(count_save, count);
2316
2317 // Copy from low to high addresses
2318 __ mov(start_to, to); // Save destination array start address
2319 __ b(L_load_element);
2320
2321 // ======== begin loop ========
2322 // (Loop is rotated; its entry is L_load_element.)
2323 // Loop control:
2324 // for (; count != 0; count--) {
2325 // copied_oop = load_heap_oop(from++);
2326 // ... generate_type_check ...;
2327 // store_heap_oop(to++, copied_oop);
2328 // }
2329 __ align(OptoLoopAlignment);
2330
2331 __ BIND(L_store_element);
2332 bs->copy_store_at(_masm, decorators, T_OBJECT, element_size,
2333 __ post(to, element_size), copied_oop, noreg,
2334 gct1, gct2, gct3);
2335 __ sub(count, count, 1);
2336 __ cbz(count, L_do_card_marks);
2337
2338 // ======== loop entry is here ========
2339 __ BIND(L_load_element);
2340 bs->copy_load_at(_masm, decorators, T_OBJECT, element_size,
2341 copied_oop, noreg, __ post(from, element_size),
2342 gct1);
2343 __ cbz(copied_oop, L_store_element);
2344
2345 __ load_klass(r19_klass, copied_oop, rscratch1);// query the object klass
2346
2347 BLOCK_COMMENT("type_check:");
2348 generate_type_check(/*sub_klass*/r19_klass,
2349 /*super_check_offset*/ckoff,
2350 /*super_klass*/ckval,
2351 /*r_array_base*/gct1,
2352 /*temp2*/gct2,
2353 /*result*/r10, L_store_element);
2354
2355 // Fall through on failure!
2356
2357 // ======== end loop ========
2358
2359 // It was a real error; we must depend on the caller to finish the job.
2360 // Register count = remaining oops, count_orig = total oops.
2361 // Emit GC store barriers for the oops we have copied and report
2362 // their number to the caller.
2363
2364 __ subs(count, count_save, count); // K = partially copied oop count
2365 __ eon(count, count, zr); // report (-1^K) to caller
2366 __ br(Assembler::EQ, L_done_pop);
2367
2368 __ BIND(L_do_card_marks);
2369 bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, count_save, rscratch1);
2370
2371 __ bind(L_done_pop);
2372 __ pop(RegSet::of(r19, r20, r21, r22), sp);
2373 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
2374
2375 __ bind(L_done);
2376 __ mov(r0, count);
2377 __ leave();
2378 __ ret(lr);
2379
2380 // record the stub entry and end plus any no_push entry
2381 store_archive_data(stub_id, start, __ pc() , entries_ptr);
2382 return start;
2383 }
2384
2385 // Perform range checks on the proposed arraycopy.
2386 // Kills temp, but nothing else.
2387 // Also, clean the sign bits of src_pos and dst_pos.
2388 void arraycopy_range_checks(Register src, // source array oop (c_rarg0)
2389 Register src_pos, // source position (c_rarg1)
2390 Register dst, // destination array oo (c_rarg2)
2391 Register dst_pos, // destination position (c_rarg3)
2392 Register length,
2393 Register temp,
2394 Label& L_failed) {
2395 BLOCK_COMMENT("arraycopy_range_checks:");
2396
2397 assert_different_registers(rscratch1, temp);
2398
2399 // if (src_pos + length > arrayOop(src)->length()) FAIL;
2400 __ ldrw(rscratch1, Address(src, arrayOopDesc::length_offset_in_bytes()));
2401 __ addw(temp, length, src_pos);
2402 __ cmpw(temp, rscratch1);
2403 __ br(Assembler::HI, L_failed);
2404
2405 // if (dst_pos + length > arrayOop(dst)->length()) FAIL;
2406 __ ldrw(rscratch1, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2407 __ addw(temp, length, dst_pos);
2408 __ cmpw(temp, rscratch1);
2409 __ br(Assembler::HI, L_failed);
2410
2411 // Have to clean up high 32 bits of 'src_pos' and 'dst_pos'.
2412 __ movw(src_pos, src_pos);
2413 __ movw(dst_pos, dst_pos);
2414
2415 BLOCK_COMMENT("arraycopy_range_checks done");
2416 }
2417
2418 // These stubs get called from some dumb test routine.
2419 // I'll write them properly when they're called from
2420 // something that's actually doing something.
2421 static void fake_arraycopy_stub(address src, address dst, int count) {
2422 assert(count == 0, "huh?");
2423 }
2424
2425
2426 //
2427 // Generate 'unsafe' array copy stub
2428 // Though just as safe as the other stubs, it takes an unscaled
2429 // size_t argument instead of an element count.
2430 //
2431 // Input:
2432 // c_rarg0 - source array address
2433 // c_rarg1 - destination array address
2434 // c_rarg2 - byte count, treated as ssize_t, can be zero
2435 //
2436 // Examines the alignment of the operands and dispatches
2437 // to a long, int, short, or byte copy loop.
2438 //
2439 address generate_unsafe_copy(address byte_copy_entry,
2440 address short_copy_entry,
2441 address int_copy_entry,
2442 address long_copy_entry) {
2443 StubId stub_id = StubId::stubgen_unsafe_arraycopy_id;
2444 int entry_count = StubInfo::entry_count(stub_id);
2445 assert(entry_count == 1, "sanity check");
2446 address start = load_archive_data(stub_id);
2447 if (start != nullptr) {
2448 return start;
2449 }
2450 Label L_long_aligned, L_int_aligned, L_short_aligned;
2451 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
2452
2453 __ align(CodeEntryAlignment);
2454 StubCodeMark mark(this, stub_id);
2455 start = __ pc();
2456 __ enter(); // required for proper stackwalking of RuntimeStub frame
2457
2458 // bump this on entry, not on exit:
2459 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2460
2461 __ orr(rscratch1, s, d);
2462 __ orr(rscratch1, rscratch1, count);
2463
2464 __ andr(rscratch1, rscratch1, BytesPerLong-1);
2465 __ cbz(rscratch1, L_long_aligned);
2466 __ andr(rscratch1, rscratch1, BytesPerInt-1);
2467 __ cbz(rscratch1, L_int_aligned);
2468 __ tbz(rscratch1, 0, L_short_aligned);
2469 __ b(RuntimeAddress(byte_copy_entry));
2470
2471 __ BIND(L_short_aligned);
2472 __ lsr(count, count, LogBytesPerShort); // size => short_count
2473 __ b(RuntimeAddress(short_copy_entry));
2474 __ BIND(L_int_aligned);
2475 __ lsr(count, count, LogBytesPerInt); // size => int_count
2476 __ b(RuntimeAddress(int_copy_entry));
2477 __ BIND(L_long_aligned);
2478 __ lsr(count, count, LogBytesPerLong); // size => long_count
2479 __ b(RuntimeAddress(long_copy_entry));
2480
2481 // record the stub entry and end
2482 store_archive_data(stub_id, start, __ pc());
2483
2484 return start;
2485 }
2486
2487 //
2488 // Generate generic array copy stubs
2489 //
2490 // Input:
2491 // c_rarg0 - src oop
2492 // c_rarg1 - src_pos (32-bits)
2493 // c_rarg2 - dst oop
2494 // c_rarg3 - dst_pos (32-bits)
2495 // c_rarg4 - element count (32-bits)
2496 //
2497 // Output:
2498 // r0 == 0 - success
2499 // r0 == -1^K - failure, where K is partial transfer count
2500 //
2501 address generate_generic_copy(address byte_copy_entry, address short_copy_entry,
2502 address int_copy_entry, address oop_copy_entry,
2503 address long_copy_entry, address checkcast_copy_entry) {
2504 StubId stub_id = StubId::stubgen_generic_arraycopy_id;
2505 int entry_count = StubInfo::entry_count(stub_id);
2506 assert(entry_count == 1, "sanity check");
2507 address start = load_archive_data(stub_id);
2508 if (start != nullptr) {
2509 return start;
2510 }
2511 Label L_failed, L_objArray;
2512 Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
2513
2514 // Input registers
2515 const Register src = c_rarg0; // source array oop
2516 const Register src_pos = c_rarg1; // source position
2517 const Register dst = c_rarg2; // destination array oop
2518 const Register dst_pos = c_rarg3; // destination position
2519 const Register length = c_rarg4;
2520
2521
2522 // Registers used as temps
2523 const Register dst_klass = c_rarg5;
2524
2525 __ align(CodeEntryAlignment);
2526
2527 StubCodeMark mark(this, stub_id);
2528
2529 start = __ pc();
2530
2531 __ enter(); // required for proper stackwalking of RuntimeStub frame
2532
2533 // bump this on entry, not on exit:
2534 inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
2535
2536 //-----------------------------------------------------------------------
2537 // Assembler stub will be used for this call to arraycopy
2538 // if the following conditions are met:
2539 //
2540 // (1) src and dst must not be null.
2541 // (2) src_pos must not be negative.
2542 // (3) dst_pos must not be negative.
2543 // (4) length must not be negative.
2544 // (5) src klass and dst klass should be the same and not null.
2545 // (6) src and dst should be arrays.
2546 // (7) src_pos + length must not exceed length of src.
2547 // (8) dst_pos + length must not exceed length of dst.
2548 //
2549
2550 // if (src == nullptr) return -1;
2551 __ cbz(src, L_failed);
2552
2553 // if (src_pos < 0) return -1;
2554 __ tbnz(src_pos, 31, L_failed); // i.e. sign bit set
2555
2556 // if (dst == nullptr) return -1;
2557 __ cbz(dst, L_failed);
2558
2559 // if (dst_pos < 0) return -1;
2560 __ tbnz(dst_pos, 31, L_failed); // i.e. sign bit set
2561
2562 // registers used as temp
2563 const Register scratch_length = r16; // elements count to copy
2564 const Register scratch_src_klass = r17; // array klass
2565 const Register lh = r15; // layout helper
2566
2567 // if (length < 0) return -1;
2568 __ movw(scratch_length, length); // length (elements count, 32-bits value)
2569 __ tbnz(scratch_length, 31, L_failed); // i.e. sign bit set
2570
2571 __ load_narrow_klass(scratch_src_klass, src);
2572 #ifdef ASSERT
2573 // assert(src->klass() != nullptr);
2574 {
2575 BLOCK_COMMENT("assert klasses not null {");
2576 Label L1, L2;
2577 __ cbnz(scratch_src_klass, L2); // it is broken if klass is null
2578 __ bind(L1);
2579 __ stop("broken null klass");
2580 __ bind(L2);
2581 __ load_narrow_klass(rscratch1, dst);
2582 __ cbz(rscratch1, L1); // this would be broken also
2583 BLOCK_COMMENT("} assert klasses not null done");
2584 }
2585 #endif
2586 __ decode_klass_not_null(scratch_src_klass, scratch_src_klass, rscratch1);
2587
2588 // Load layout helper (32-bits)
2589 //
2590 // |array_tag| | header_size | element_type | |log2_element_size|
2591 // 32 30 24 16 8 2 0
2592 //
2593 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2594 //
2595
2596 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2597
2598 // Handle objArrays completely differently...
2599 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2600 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2601 __ movw(rscratch1, objArray_lh);
2602 __ eorw(rscratch2, lh, rscratch1);
2603 __ cbzw(rscratch2, L_objArray);
2604
2605 // if (src->klass() != dst->klass()) return -1;
2606 __ load_klass(rscratch2, dst, rscratch1);
2607 __ eor(rscratch2, rscratch2, scratch_src_klass);
2608 __ cbnz(rscratch2, L_failed);
2609
2610 // if (!src->is_Array()) return -1;
2611 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2612
2613 // At this point, it is known to be a typeArray (array_tag 0x3).
2614 #ifdef ASSERT
2615 {
2616 BLOCK_COMMENT("assert primitive array {");
2617 Label L;
2618 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2619 __ cmpw(lh, rscratch2);
2620 __ br(Assembler::GE, L);
2621 __ stop("must be a primitive array");
2622 __ bind(L);
2623 BLOCK_COMMENT("} assert primitive array done");
2624 }
2625 #endif
2626
2627 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2628 rscratch2, L_failed);
2629
2630 // TypeArrayKlass
2631 //
2632 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2633 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2634 //
2635
2636 const Register rscratch1_offset = rscratch1; // array offset
2637 const Register r15_elsize = lh; // element size
2638
2639 __ ubfx(rscratch1_offset, lh, Klass::_lh_header_size_shift,
2640 exact_log2(Klass::_lh_header_size_mask+1)); // array_offset
2641 __ add(src, src, rscratch1_offset); // src array offset
2642 __ add(dst, dst, rscratch1_offset); // dst array offset
2643 BLOCK_COMMENT("choose copy loop based on element size");
2644
2645 // next registers should be set before the jump to corresponding stub
2646 const Register from = c_rarg0; // source array address
2647 const Register to = c_rarg1; // destination array address
2648 const Register count = c_rarg2; // elements count
2649
2650 // 'from', 'to', 'count' registers should be set in such order
2651 // since they are the same as 'src', 'src_pos', 'dst'.
2652
2653 assert(Klass::_lh_log2_element_size_shift == 0, "fix this code");
2654
2655 // The possible values of elsize are 0-3, i.e. exact_log2(element
2656 // size in bytes). We do a simple bitwise binary search.
2657 __ BIND(L_copy_bytes);
2658 __ tbnz(r15_elsize, 1, L_copy_ints);
2659 __ tbnz(r15_elsize, 0, L_copy_shorts);
2660 __ lea(from, Address(src, src_pos));// src_addr
2661 __ lea(to, Address(dst, dst_pos));// dst_addr
2662 __ movw(count, scratch_length); // length
2663 __ b(RuntimeAddress(byte_copy_entry));
2664
2665 __ BIND(L_copy_shorts);
2666 __ lea(from, Address(src, src_pos, Address::lsl(1)));// src_addr
2667 __ lea(to, Address(dst, dst_pos, Address::lsl(1)));// dst_addr
2668 __ movw(count, scratch_length); // length
2669 __ b(RuntimeAddress(short_copy_entry));
2670
2671 __ BIND(L_copy_ints);
2672 __ tbnz(r15_elsize, 0, L_copy_longs);
2673 __ lea(from, Address(src, src_pos, Address::lsl(2)));// src_addr
2674 __ lea(to, Address(dst, dst_pos, Address::lsl(2)));// dst_addr
2675 __ movw(count, scratch_length); // length
2676 __ b(RuntimeAddress(int_copy_entry));
2677
2678 __ BIND(L_copy_longs);
2679 #ifdef ASSERT
2680 {
2681 BLOCK_COMMENT("assert long copy {");
2682 Label L;
2683 __ andw(lh, lh, Klass::_lh_log2_element_size_mask); // lh -> r15_elsize
2684 __ cmpw(r15_elsize, LogBytesPerLong);
2685 __ br(Assembler::EQ, L);
2686 __ stop("must be long copy, but elsize is wrong");
2687 __ bind(L);
2688 BLOCK_COMMENT("} assert long copy done");
2689 }
2690 #endif
2691 __ lea(from, Address(src, src_pos, Address::lsl(3)));// src_addr
2692 __ lea(to, Address(dst, dst_pos, Address::lsl(3)));// dst_addr
2693 __ movw(count, scratch_length); // length
2694 __ b(RuntimeAddress(long_copy_entry));
2695
2696 // ObjArrayKlass
2697 __ BIND(L_objArray);
2698 // live at this point: scratch_src_klass, scratch_length, src[_pos], dst[_pos]
2699
2700 Label L_plain_copy, L_checkcast_copy;
2701 // test array classes for subtyping
2702 __ load_klass(r15, dst, rscratch1);
2703 __ cmp(scratch_src_klass, r15); // usual case is exact equality
2704 __ br(Assembler::NE, L_checkcast_copy);
2705
2706 // Identically typed arrays can be copied without element-wise checks.
2707 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2708 rscratch2, L_failed);
2709
2710 __ lea(from, Address(src, src_pos, Address::lsl(LogBytesPerHeapOop)));
2711 __ add(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2712 __ lea(to, Address(dst, dst_pos, Address::lsl(LogBytesPerHeapOop)));
2713 __ add(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2714 __ movw(count, scratch_length); // length
2715 __ BIND(L_plain_copy);
2716 __ b(RuntimeAddress(oop_copy_entry));
2717
2718 __ BIND(L_checkcast_copy);
2719 // live at this point: scratch_src_klass, scratch_length, r15 (dst_klass)
2720 {
2721 // Before looking at dst.length, make sure dst is also an objArray.
2722 __ ldrw(rscratch1, Address(r15, lh_offset));
2723 __ movw(rscratch2, objArray_lh);
2724 __ eorw(rscratch1, rscratch1, rscratch2);
2725 __ cbnzw(rscratch1, L_failed);
2726
2727 // It is safe to examine both src.length and dst.length.
2728 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2729 r15, L_failed);
2730
2731 __ load_klass(dst_klass, dst, rscratch1); // reload
2732
2733 // Marshal the base address arguments now, freeing registers.
2734 __ lea(from, Address(src, src_pos, Address::lsl(LogBytesPerHeapOop)));
2735 __ add(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2736 __ lea(to, Address(dst, dst_pos, Address::lsl(LogBytesPerHeapOop)));
2737 __ add(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2738 __ movw(count, length); // length (reloaded)
2739 Register sco_temp = c_rarg3; // this register is free now
2740 assert_different_registers(from, to, count, sco_temp,
2741 dst_klass, scratch_src_klass);
2742 // assert_clean_int(count, sco_temp);
2743
2744 // Generate the type check.
2745 const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2746 __ ldrw(sco_temp, Address(dst_klass, sco_offset));
2747
2748 // Smashes rscratch1, rscratch2
2749 generate_type_check(scratch_src_klass, sco_temp, dst_klass, /*temps*/ noreg, noreg, noreg,
2750 L_plain_copy);
2751
2752 // Fetch destination element klass from the ObjArrayKlass header.
2753 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2754 __ ldr(dst_klass, Address(dst_klass, ek_offset));
2755 __ ldrw(sco_temp, Address(dst_klass, sco_offset));
2756
2757 // the checkcast_copy loop needs two extra arguments:
2758 assert(c_rarg3 == sco_temp, "#3 already in place");
2759 // Set up arguments for checkcast_copy_entry.
2760 __ mov(c_rarg4, dst_klass); // dst.klass.element_klass
2761 __ b(RuntimeAddress(checkcast_copy_entry));
2762 }
2763
2764 __ BIND(L_failed);
2765 __ mov(r0, -1);
2766 __ leave(); // required for proper stackwalking of RuntimeStub frame
2767 __ ret(lr);
2768
2769 // record the stub entry and end
2770 store_archive_data(stub_id, start, __ pc());
2771
2772 return start;
2773 }
2774
2775 //
2776 // Generate stub for array fill. If "aligned" is true, the
2777 // "to" address is assumed to be heapword aligned.
2778 //
2779 // Arguments for generated stub:
2780 // to: c_rarg0
2781 // value: c_rarg1
2782 // count: c_rarg2 treated as signed
2783 //
2784 address generate_fill(StubId stub_id) {
2785 BasicType t;
2786 bool aligned;
2787
2788 switch (stub_id) {
2789 case StubId::stubgen_jbyte_fill_id:
2790 t = T_BYTE;
2791 aligned = false;
2792 break;
2793 case StubId::stubgen_jshort_fill_id:
2794 t = T_SHORT;
2795 aligned = false;
2796 break;
2797 case StubId::stubgen_jint_fill_id:
2798 t = T_INT;
2799 aligned = false;
2800 break;
2801 case StubId::stubgen_arrayof_jbyte_fill_id:
2802 t = T_BYTE;
2803 aligned = true;
2804 break;
2805 case StubId::stubgen_arrayof_jshort_fill_id:
2806 t = T_SHORT;
2807 aligned = true;
2808 break;
2809 case StubId::stubgen_arrayof_jint_fill_id:
2810 t = T_INT;
2811 aligned = true;
2812 break;
2813 default:
2814 ShouldNotReachHere();
2815 };
2816 int entry_count = StubInfo::entry_count(stub_id);
2817 assert(entry_count == 1, "sanity check");
2818 address start = load_archive_data(stub_id);
2819 if (start != nullptr) {
2820 return start;
2821 }
2822 __ align(CodeEntryAlignment);
2823 StubCodeMark mark(this, stub_id);
2824 start = __ pc();
2825
2826 BLOCK_COMMENT("Entry:");
2827
2828 const Register to = c_rarg0; // source array address
2829 const Register value = c_rarg1; // value
2830 const Register count = c_rarg2; // elements count
2831
2832 const Register bz_base = r10; // base for block_zero routine
2833 const Register cnt_words = r11; // temp register
2834
2835 __ enter();
2836
2837 Label L_fill_elements, L_exit1;
2838
2839 int shift = -1;
2840 switch (t) {
2841 case T_BYTE:
2842 shift = 0;
2843 __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
2844 __ bfi(value, value, 8, 8); // 8 bit -> 16 bit
2845 __ bfi(value, value, 16, 16); // 16 bit -> 32 bit
2846 __ br(Assembler::LO, L_fill_elements);
2847 break;
2848 case T_SHORT:
2849 shift = 1;
2850 __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
2851 __ bfi(value, value, 16, 16); // 16 bit -> 32 bit
2852 __ br(Assembler::LO, L_fill_elements);
2853 break;
2854 case T_INT:
2855 shift = 2;
2856 __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
2857 __ br(Assembler::LO, L_fill_elements);
2858 break;
2859 default: ShouldNotReachHere();
2860 }
2861
2862 // Align source address at 8 bytes address boundary.
2863 Label L_skip_align1, L_skip_align2, L_skip_align4;
2864 if (!aligned) {
2865 switch (t) {
2866 case T_BYTE:
2867 // One byte misalignment happens only for byte arrays.
2868 __ tbz(to, 0, L_skip_align1);
2869 __ strb(value, Address(__ post(to, 1)));
2870 __ subw(count, count, 1);
2871 __ bind(L_skip_align1);
2872 // Fallthrough
2873 case T_SHORT:
2874 // Two bytes misalignment happens only for byte and short (char) arrays.
2875 __ tbz(to, 1, L_skip_align2);
2876 __ strh(value, Address(__ post(to, 2)));
2877 __ subw(count, count, 2 >> shift);
2878 __ bind(L_skip_align2);
2879 // Fallthrough
2880 case T_INT:
2881 // Align to 8 bytes, we know we are 4 byte aligned to start.
2882 __ tbz(to, 2, L_skip_align4);
2883 __ strw(value, Address(__ post(to, 4)));
2884 __ subw(count, count, 4 >> shift);
2885 __ bind(L_skip_align4);
2886 break;
2887 default: ShouldNotReachHere();
2888 }
2889 }
2890
2891 //
2892 // Fill large chunks
2893 //
2894 __ lsrw(cnt_words, count, 3 - shift); // number of words
2895 __ bfi(value, value, 32, 32); // 32 bit -> 64 bit
2896 __ subw(count, count, cnt_words, Assembler::LSL, 3 - shift);
2897 if (UseBlockZeroing) {
2898 Label non_block_zeroing, rest;
2899 // If the fill value is zero we can use the fast zero_words().
2900 __ cbnz(value, non_block_zeroing);
2901 __ mov(bz_base, to);
2902 __ add(to, to, cnt_words, Assembler::LSL, LogBytesPerWord);
2903 address tpc = __ zero_words(bz_base, cnt_words);
2904 if (tpc == nullptr) {
2905 fatal("CodeCache is full at generate_fill");
2906 }
2907 __ b(rest);
2908 __ bind(non_block_zeroing);
2909 __ fill_words(to, cnt_words, value);
2910 __ bind(rest);
2911 } else {
2912 __ fill_words(to, cnt_words, value);
2913 }
2914
2915 // Remaining count is less than 8 bytes. Fill it by a single store.
2916 // Note that the total length is no less than 8 bytes.
2917 if (t == T_BYTE || t == T_SHORT) {
2918 Label L_exit1;
2919 __ cbzw(count, L_exit1);
2920 __ add(to, to, count, Assembler::LSL, shift); // points to the end
2921 __ str(value, Address(to, -8)); // overwrite some elements
2922 __ bind(L_exit1);
2923 __ leave();
2924 __ ret(lr);
2925 }
2926
2927 // Handle copies less than 8 bytes.
2928 Label L_fill_2, L_fill_4, L_exit2;
2929 __ bind(L_fill_elements);
2930 switch (t) {
2931 case T_BYTE:
2932 __ tbz(count, 0, L_fill_2);
2933 __ strb(value, Address(__ post(to, 1)));
2934 __ bind(L_fill_2);
2935 __ tbz(count, 1, L_fill_4);
2936 __ strh(value, Address(__ post(to, 2)));
2937 __ bind(L_fill_4);
2938 __ tbz(count, 2, L_exit2);
2939 __ strw(value, Address(to));
2940 break;
2941 case T_SHORT:
2942 __ tbz(count, 0, L_fill_4);
2943 __ strh(value, Address(__ post(to, 2)));
2944 __ bind(L_fill_4);
2945 __ tbz(count, 1, L_exit2);
2946 __ strw(value, Address(to));
2947 break;
2948 case T_INT:
2949 __ cbzw(count, L_exit2);
2950 __ strw(value, Address(to));
2951 break;
2952 default: ShouldNotReachHere();
2953 }
2954 __ bind(L_exit2);
2955 __ leave();
2956 __ ret(lr);
2957
2958 // record the stub entry and end
2959 store_archive_data(stub_id, start, __ pc());
2960
2961 return start;
2962 }
2963
2964 address generate_unsafecopy_common_error_exit() {
2965 StubId stub_id = StubId::stubgen_unsafecopy_common_id;
2966 int entry_count = StubInfo::entry_count(stub_id);
2967 assert(entry_count == 1, "sanity check");
2968 address start = load_archive_data(stub_id);
2969 if (start != nullptr) {
2970 return start;
2971 }
2972 __ align(CodeEntryAlignment);
2973 StubCodeMark mark(this, stub_id);
2974 start = __ pc();
2975 __ leave();
2976 __ mov(r0, 0);
2977 __ ret(lr);
2978
2979 // record the stub entry and end
2980 store_archive_data(stub_id, start, __ pc());
2981
2982 return start;
2983 }
2984
2985 //
2986 // Generate 'unsafe' set memory stub
2987 // Though just as safe as the other stubs, it takes an unscaled
2988 // size_t (# bytes) argument instead of an element count.
2989 //
2990 // This fill operation is atomicity preserving: as long as the
2991 // address supplied is sufficiently aligned, all writes of up to 64
2992 // bits in size are single-copy atomic.
2993 //
2994 // Input:
2995 // c_rarg0 - destination array address
2996 // c_rarg1 - byte count (size_t)
2997 // c_rarg2 - byte value
2998 //
2999 address generate_unsafe_setmemory() {
3000 StubId stub_id = StubId::stubgen_unsafe_setmemory_id;
3001 int entry_count = StubInfo::entry_count(stub_id);
3002 assert(entry_count == 1, "sanity check");
3003 // we expect one set of extra unsafememory access handler entries
3004 GrowableArray<address> extras;
3005 int extra_count = 1 * UnsafeMemoryAccess::COLUMN_COUNT;
3006 address start = load_archive_data(stub_id, nullptr, &extras);
3007 if (start != nullptr) {
3008 assert(extras.length() == extra_count,
3009 "unexpected extra entry count %d", extras.length());
3010 register_unsafe_access_handlers(extras, 0, 1);
3011 return start;
3012 }
3013
3014 __ align(CodeEntryAlignment);
3015 StubCodeMark mark(this, stub_id);
3016 start = __ pc();
3017
3018 Register dest = c_rarg0, count = c_rarg1, value = c_rarg2;
3019 Label tail;
3020
3021 {
3022 UnsafeMemoryAccessMark umam(this, true, false);
3023
3024 __ enter(); // required for proper stackwalking of RuntimeStub frame
3025
3026 __ dup(v0, __ T16B, value);
3027
3028 if (AvoidUnalignedAccesses) {
3029 __ cmp(count, (u1)16);
3030 __ br(__ LO, tail);
3031
3032 __ mov(rscratch1, 16);
3033 __ andr(rscratch2, dest, 15);
3034 __ sub(rscratch1, rscratch1, rscratch2); // Bytes needed to 16-align dest
3035 __ strq(v0, Address(dest));
3036 __ sub(count, count, rscratch1);
3037 __ add(dest, dest, rscratch1);
3038 }
3039
3040 __ subs(count, count, (u1)64);
3041 __ br(__ LO, tail);
3042 {
3043 Label again;
3044 __ bind(again);
3045 __ stpq(v0, v0, Address(dest));
3046 __ stpq(v0, v0, Address(dest, 32));
3047
3048 __ subs(count, count, 64);
3049 __ add(dest, dest, 64);
3050 __ br(__ HS, again);
3051 }
3052
3053 __ bind(tail);
3054 // The count of bytes is off by 64, but we don't need to correct
3055 // it because we're only going to use the least-significant few
3056 // count bits from here on.
3057 // __ add(count, count, 64);
3058
3059 {
3060 Label dont;
3061 __ tbz(count, exact_log2(32), dont);
3062 __ stpq(v0, v0, __ post(dest, 32));
3063 __ bind(dont);
3064 }
3065 {
3066 Label dont;
3067 __ tbz(count, exact_log2(16), dont);
3068 __ strq(v0, __ post(dest, 16));
3069 __ bind(dont);
3070 }
3071 {
3072 Label dont;
3073 __ tbz(count, exact_log2(8), dont);
3074 __ strd(v0, __ post(dest, 8));
3075 __ bind(dont);
3076 }
3077
3078 Label finished;
3079 __ tst(count, 7);
3080 __ br(__ EQ, finished);
3081
3082 {
3083 Label dont;
3084 __ tbz(count, exact_log2(4), dont);
3085 __ strs(v0, __ post(dest, 4));
3086 __ bind(dont);
3087 }
3088 {
3089 Label dont;
3090 __ tbz(count, exact_log2(2), dont);
3091 __ bfi(value, value, 8, 8);
3092 __ strh(value, __ post(dest, 2));
3093 __ bind(dont);
3094 }
3095 {
3096 Label dont;
3097 __ tbz(count, exact_log2(1), dont);
3098 __ strb(value, Address(dest));
3099 __ bind(dont);
3100 }
3101
3102 __ bind(finished);
3103 __ leave();
3104 __ ret(lr);
3105 // have to exit the block and destroy the UnsafeMemoryAccessMark
3106 // in order to retrieve the handler end address
3107 }
3108
3109 // install saved handler addresses in extras
3110 address end = __ pc();
3111 retrieve_unsafe_access_handlers(start, end, extras);
3112 assert(extras.length() == extra_count,
3113 "incorrect handlers count %d", extras.length());
3114 // record the stub entry and end plus the extras
3115 store_archive_data(stub_id, start, end, nullptr, &extras);
3116
3117 return start;
3118 }
3119
3120 address generate_data_cache_writeback() {
3121 const Register line = c_rarg0; // address of line to write back
3122
3123 StubId stub_id = StubId::stubgen_data_cache_writeback_id;
3124 int entry_count = StubInfo::entry_count(stub_id);
3125 assert(entry_count == 1, "sanity check");
3126 address start = load_archive_data(stub_id);
3127 if (start != nullptr) {
3128 return start;
3129 }
3130 __ align(CodeEntryAlignment);
3131 StubCodeMark mark(this, stub_id);
3132
3133 start = __ pc();
3134 __ enter();
3135 __ cache_wb(Address(line, 0));
3136 __ leave();
3137 __ ret(lr);
3138
3139 // record the stub entry and end
3140 store_archive_data(stub_id, start, __ pc());
3141
3142 return start;
3143 }
3144
3145 address generate_data_cache_writeback_sync() {
3146 StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id;
3147 int entry_count = StubInfo::entry_count(stub_id);
3148 assert(entry_count == 1, "sanity check");
3149 address start = load_archive_data(stub_id);
3150 if (start != nullptr) {
3151 return start;
3152 }
3153 const Register is_pre = c_rarg0; // pre or post sync
3154 __ align(CodeEntryAlignment);
3155 StubCodeMark mark(this, stub_id);
3156
3157 // pre wbsync is a no-op
3158 // post wbsync translates to an sfence
3159
3160 Label skip;
3161 start = __ pc();
3162 __ enter();
3163 __ cbnz(is_pre, skip);
3164 __ cache_wbsync(false);
3165 __ bind(skip);
3166 __ leave();
3167 __ ret(lr);
3168
3169 // record the stub entry and end
3170 store_archive_data(stub_id, start, __ pc());
3171
3172 return start;
3173 }
3174
3175 void generate_arraycopy_stubs() {
3176 // Some copy stubs publish a normal entry and then a 2nd 'fallback'
3177 // entry immediately following their stack push. This can be used
3178 // as a post-push branch target for compatible stubs when they
3179 // identify a special case that can be handled by the fallback
3180 // stub e.g a disjoint copy stub may be use as a special case
3181 // fallback for its compatible conjoint copy stub.
3182 //
3183 // A no push entry is always returned in the following local and
3184 // then published by assigning to the appropriate entry field in
3185 // class StubRoutines. The entry value is then passed to the
3186 // generator for the compatible stub. That means the entry must be
3187 // listed when saving to/restoring from the AOT cache, ensuring
3188 // that the inter-stub jumps are noted at AOT-cache save and
3189 // relocated at AOT cache load.
3190 address nopush_entry;
3191
3192 // generate the common exit first so later stubs can rely on it if
3193 // they want an UnsafeMemoryAccess exit non-local to the stub
3194 StubRoutines::_unsafecopy_common_exit = generate_unsafecopy_common_error_exit();
3195 // register the stub as the default exit with class UnsafeMemoryAccess
3196 UnsafeMemoryAccess::set_common_exit_stub_pc(StubRoutines::_unsafecopy_common_exit);
3197
3198 // generate and publish arch64-specific bulk copy routines first
3199 // so we can call them from other copy stubs
3200 StubRoutines::aarch64::_copy_byte_f = generate_copy_longs(StubId::stubgen_copy_byte_f_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
3201 StubRoutines::aarch64::_copy_byte_b = generate_copy_longs(StubId::stubgen_copy_byte_b_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
3202
3203 StubRoutines::aarch64::_copy_oop_f = generate_copy_longs(StubId::stubgen_copy_oop_f_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
3204 StubRoutines::aarch64::_copy_oop_b = generate_copy_longs(StubId::stubgen_copy_oop_b_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
3205
3206 StubRoutines::aarch64::_copy_oop_uninit_f = generate_copy_longs(StubId::stubgen_copy_oop_uninit_f_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, r0, r1, r15);
3207 StubRoutines::aarch64::_copy_oop_uninit_b = generate_copy_longs(StubId::stubgen_copy_oop_uninit_b_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, r0, r1, r15);
3208
3209 StubRoutines::aarch64::_zero_blocks = generate_zero_blocks();
3210
3211 //*** jbyte
3212 // Always need aligned and unaligned versions
3213 StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &nopush_entry);
3214 // disjoint nopush entry is needed by conjoint copy
3215 StubRoutines::_jbyte_disjoint_arraycopy_nopush = nopush_entry;
3216 StubRoutines::_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, StubRoutines::_jbyte_disjoint_arraycopy_nopush, &nopush_entry);
3217 // conjoint nopush entry is needed by generic/unsafe copy
3218 StubRoutines::_jbyte_arraycopy_nopush = nopush_entry;
3219 StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &nopush_entry);
3220 // disjoint arrayof nopush entry is needed by conjoint copy
3221 StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush = nopush_entry;
3222 StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush, nullptr);
3223
3224 //*** jshort
3225 // Always need aligned and unaligned versions
3226 StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &nopush_entry);
3227 // disjoint nopush entry is needed by conjoint copy
3228 StubRoutines::_jshort_disjoint_arraycopy_nopush = nopush_entry;
3229 StubRoutines::_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, StubRoutines::_jshort_disjoint_arraycopy_nopush, &nopush_entry);
3230 // conjoint nopush entry is used by generic/unsafe copy
3231 StubRoutines::_jshort_arraycopy_nopush = nopush_entry;
3232 StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &nopush_entry);
3233 // disjoint arrayof nopush entry is needed by conjoint copy
3234 StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush = nopush_entry;
3235 StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush, nullptr);
3236
3237 //*** jint
3238 // Aligned versions
3239 StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &nopush_entry);
3240 // disjoint arrayof nopush entry is needed by conjoint copy
3241 StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush = nopush_entry;
3242 StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush, nullptr);
3243 // In 64 bit we need both aligned and unaligned versions of jint arraycopy.
3244 // jint_arraycopy_nopush always points to the unaligned version
3245 StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &nopush_entry);
3246 // disjoint nopush entry is needed by conjoint copy
3247 StubRoutines::_jint_disjoint_arraycopy_nopush = nopush_entry;
3248 StubRoutines::_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, StubRoutines::_jint_disjoint_arraycopy_nopush, &nopush_entry);
3249 // conjoint nopush entry is needed by generic/unsafe copy
3250 StubRoutines::_jint_arraycopy_nopush = nopush_entry;
3251
3252 //*** jlong
3253 // It is always aligned
3254 StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &nopush_entry);
3255 // disjoint arrayof nopush entry is needed by conjoint copy
3256 StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush = nopush_entry;
3257 StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush, &nopush_entry);
3258 // conjoint nopush entry is needed by generic/unsafe copy
3259 StubRoutines::_jlong_arraycopy_nopush = nopush_entry;
3260 // disjoint normal/nopush and conjoint normal entries are not
3261 // generated since the arrayof versions are the same
3262 StubRoutines::_jlong_disjoint_arraycopy = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
3263 StubRoutines::_jlong_disjoint_arraycopy_nopush = StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush;
3264 StubRoutines::_jlong_arraycopy = StubRoutines::_arrayof_jlong_arraycopy;
3265
3266 //*** oops
3267 {
3268 StubRoutines::_arrayof_oop_disjoint_arraycopy
3269 = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &nopush_entry);
3270 // disjoint arrayof nopush entry is needed by conjoint copy
3271 StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush = nopush_entry;
3272 StubRoutines::_arrayof_oop_arraycopy
3273 = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush, &nopush_entry);
3274 // conjoint arrayof nopush entry is needed by generic/unsafe copy
3275 StubRoutines::_oop_arraycopy_nopush = nopush_entry;
3276 // Aligned versions without pre-barriers
3277 StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit
3278 = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
3279 // disjoint arrayof+uninit nopush entry is needed by conjoint copy
3280 StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush = nopush_entry;
3281 // note that we don't need a returned nopush entry because the
3282 // generic/unsafe copy does not cater for uninit arrays.
3283 StubRoutines::_arrayof_oop_arraycopy_uninit
3284 = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush, nullptr);
3285 }
3286
3287 // for oop copies reuse arrayof entries for non-arrayof cases
3288 StubRoutines::_oop_disjoint_arraycopy = StubRoutines::_arrayof_oop_disjoint_arraycopy;
3289 StubRoutines::_oop_disjoint_arraycopy_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush;
3290 StubRoutines::_oop_arraycopy = StubRoutines::_arrayof_oop_arraycopy;
3291 StubRoutines::_oop_disjoint_arraycopy_uninit = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
3292 StubRoutines::_oop_disjoint_arraycopy_uninit_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush;
3293 StubRoutines::_oop_arraycopy_uninit = StubRoutines::_arrayof_oop_arraycopy_uninit;
3294
3295 StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &nopush_entry);
3296 // checkcast nopush entry is needed by generic copy
3297 StubRoutines::_checkcast_arraycopy_nopush = nopush_entry;
3298 // note that we don't need a returned nopush entry because the
3299 // generic copy does not cater for uninit arrays.
3300 StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_uninit_id, nullptr);
3301
3302 // unsafe arraycopy may fallback on conjoint stubs
3303 StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(StubRoutines::_jbyte_arraycopy_nopush,
3304 StubRoutines::_jshort_arraycopy_nopush,
3305 StubRoutines::_jint_arraycopy_nopush,
3306 StubRoutines::_jlong_arraycopy_nopush);
3307
3308 // generic arraycopy may fallback on conjoint stubs
3309 StubRoutines::_generic_arraycopy = generate_generic_copy(StubRoutines::_jbyte_arraycopy_nopush,
3310 StubRoutines::_jshort_arraycopy_nopush,
3311 StubRoutines::_jint_arraycopy_nopush,
3312 StubRoutines::_oop_arraycopy_nopush,
3313 StubRoutines::_jlong_arraycopy_nopush,
3314 StubRoutines::_checkcast_arraycopy_nopush);
3315
3316 StubRoutines::_jbyte_fill = generate_fill(StubId::stubgen_jbyte_fill_id);
3317 StubRoutines::_jshort_fill = generate_fill(StubId::stubgen_jshort_fill_id);
3318 StubRoutines::_jint_fill = generate_fill(StubId::stubgen_jint_fill_id);
3319 StubRoutines::_arrayof_jbyte_fill = generate_fill(StubId::stubgen_arrayof_jbyte_fill_id);
3320 StubRoutines::_arrayof_jshort_fill = generate_fill(StubId::stubgen_arrayof_jshort_fill_id);
3321 StubRoutines::_arrayof_jint_fill = generate_fill(StubId::stubgen_arrayof_jint_fill_id);
3322 }
3323
3324 void generate_math_stubs() { Unimplemented(); }
3325
3326 // Arguments:
3327 //
3328 // Inputs:
3329 // c_rarg0 - source byte array address
3330 // c_rarg1 - destination byte array address
3331 // c_rarg2 - sessionKe (key) in little endian int array
3332 //
3333 address generate_aescrypt_encryptBlock() {
3334 assert(UseAES, "need AES cryptographic extension support");
3335 StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id;
3336 int entry_count = StubInfo::entry_count(stub_id);
3337 assert(entry_count == 1, "sanity check");
3338 address start = load_archive_data(stub_id);
3339 if (start != nullptr) {
3340 return start;
3341 }
3342 __ align(CodeEntryAlignment);
3343 StubCodeMark mark(this, stub_id);
3344
3345 const Register from = c_rarg0; // source array address
3346 const Register to = c_rarg1; // destination array address
3347 const Register key = c_rarg2; // key array address
3348 const Register keylen = rscratch1;
3349
3350 start = __ pc();
3351 __ enter();
3352
3353 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3354
3355 __ aesenc_loadkeys(key, keylen);
3356 __ aesecb_encrypt(from, to, keylen);
3357
3358 __ mov(r0, 0);
3359
3360 __ leave();
3361 __ ret(lr);
3362
3363 // record the stub entry and end
3364 store_archive_data(stub_id, start, __ pc());
3365
3366 return start;
3367 }
3368
3369 // Arguments:
3370 //
3371 // Inputs:
3372 // c_rarg0 - source byte array address
3373 // c_rarg1 - destination byte array address
3374 // c_rarg2 - sessionKd (key) in little endian int array
3375 //
3376 address generate_aescrypt_decryptBlock() {
3377 assert(UseAES, "need AES cryptographic extension support");
3378 StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id;
3379 int entry_count = StubInfo::entry_count(stub_id);
3380 assert(entry_count == 1, "sanity check");
3381 address start = load_archive_data(stub_id);
3382 if (start != nullptr) {
3383 return start;
3384 }
3385 __ align(CodeEntryAlignment);
3386 StubCodeMark mark(this, stub_id);
3387 Label L_doLast;
3388
3389 const Register from = c_rarg0; // source array address
3390 const Register to = c_rarg1; // destination array address
3391 const Register key = c_rarg2; // key array address
3392 const Register keylen = rscratch1;
3393
3394 start = __ pc();
3395 __ enter(); // required for proper stackwalking of RuntimeStub frame
3396
3397 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3398
3399 __ aesecb_decrypt(from, to, key, keylen);
3400
3401 __ mov(r0, 0);
3402
3403 __ leave();
3404 __ ret(lr);
3405
3406 // record the stub entry and end
3407 store_archive_data(stub_id, start, __ pc());
3408
3409 return start;
3410 }
3411
3412 // Arguments:
3413 //
3414 // Inputs:
3415 // c_rarg0 - source byte array address
3416 // c_rarg1 - destination byte array address
3417 // c_rarg2 - sessionKe (key) in little endian int array
3418 // c_rarg3 - r vector byte array address
3419 // c_rarg4 - input length
3420 //
3421 // Output:
3422 // x0 - input length
3423 //
3424 address generate_cipherBlockChaining_encryptAESCrypt() {
3425 assert(UseAES, "need AES cryptographic extension support");
3426 StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id;
3427 int entry_count = StubInfo::entry_count(stub_id);
3428 assert(entry_count == 1, "sanity check");
3429 address start = load_archive_data(stub_id);
3430 if (start != nullptr) {
3431 return start;
3432 }
3433 __ align(CodeEntryAlignment);
3434 StubCodeMark mark(this, stub_id);
3435
3436 Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52;
3437
3438 const Register from = c_rarg0; // source array address
3439 const Register to = c_rarg1; // destination array address
3440 const Register key = c_rarg2; // key array address
3441 const Register rvec = c_rarg3; // r byte array initialized from initvector array address
3442 // and left with the results of the last encryption block
3443 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
3444 const Register keylen = rscratch1;
3445
3446 start = __ pc();
3447
3448 __ enter();
3449
3450 __ movw(rscratch2, len_reg);
3451
3452 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3453
3454 __ ld1(v0, __ T16B, rvec);
3455
3456 __ cmpw(keylen, 52);
3457 __ br(Assembler::CC, L_loadkeys_44);
3458 __ br(Assembler::EQ, L_loadkeys_52);
3459
3460 __ ld1(v17, v18, __ T16B, __ post(key, 32));
3461 __ rev32(v17, __ T16B, v17);
3462 __ rev32(v18, __ T16B, v18);
3463 __ BIND(L_loadkeys_52);
3464 __ ld1(v19, v20, __ T16B, __ post(key, 32));
3465 __ rev32(v19, __ T16B, v19);
3466 __ rev32(v20, __ T16B, v20);
3467 __ BIND(L_loadkeys_44);
3468 __ ld1(v21, v22, v23, v24, __ T16B, __ post(key, 64));
3469 __ rev32(v21, __ T16B, v21);
3470 __ rev32(v22, __ T16B, v22);
3471 __ rev32(v23, __ T16B, v23);
3472 __ rev32(v24, __ T16B, v24);
3473 __ ld1(v25, v26, v27, v28, __ T16B, __ post(key, 64));
3474 __ rev32(v25, __ T16B, v25);
3475 __ rev32(v26, __ T16B, v26);
3476 __ rev32(v27, __ T16B, v27);
3477 __ rev32(v28, __ T16B, v28);
3478 __ ld1(v29, v30, v31, __ T16B, key);
3479 __ rev32(v29, __ T16B, v29);
3480 __ rev32(v30, __ T16B, v30);
3481 __ rev32(v31, __ T16B, v31);
3482
3483 __ BIND(L_aes_loop);
3484 __ ld1(v1, __ T16B, __ post(from, 16));
3485 __ eor(v0, __ T16B, v0, v1);
3486
3487 __ br(Assembler::CC, L_rounds_44);
3488 __ br(Assembler::EQ, L_rounds_52);
3489
3490 __ aese(v0, v17); __ aesmc(v0, v0);
3491 __ aese(v0, v18); __ aesmc(v0, v0);
3492 __ BIND(L_rounds_52);
3493 __ aese(v0, v19); __ aesmc(v0, v0);
3494 __ aese(v0, v20); __ aesmc(v0, v0);
3495 __ BIND(L_rounds_44);
3496 __ aese(v0, v21); __ aesmc(v0, v0);
3497 __ aese(v0, v22); __ aesmc(v0, v0);
3498 __ aese(v0, v23); __ aesmc(v0, v0);
3499 __ aese(v0, v24); __ aesmc(v0, v0);
3500 __ aese(v0, v25); __ aesmc(v0, v0);
3501 __ aese(v0, v26); __ aesmc(v0, v0);
3502 __ aese(v0, v27); __ aesmc(v0, v0);
3503 __ aese(v0, v28); __ aesmc(v0, v0);
3504 __ aese(v0, v29); __ aesmc(v0, v0);
3505 __ aese(v0, v30);
3506 __ eor(v0, __ T16B, v0, v31);
3507
3508 __ st1(v0, __ T16B, __ post(to, 16));
3509
3510 __ subw(len_reg, len_reg, 16);
3511 __ cbnzw(len_reg, L_aes_loop);
3512
3513 __ st1(v0, __ T16B, rvec);
3514
3515 __ mov(r0, rscratch2);
3516
3517 __ leave();
3518 __ ret(lr);
3519
3520 // record the stub entry and end
3521 store_archive_data(stub_id, start, __ pc());
3522
3523 return start;
3524 }
3525
3526 // Arguments:
3527 //
3528 // Inputs:
3529 // c_rarg0 - source byte array address
3530 // c_rarg1 - destination byte array address
3531 // c_rarg2 - sessionKd (key) in little endian int array
3532 // c_rarg3 - r vector byte array address
3533 // c_rarg4 - input length
3534 //
3535 // Output:
3536 // r0 - input length
3537 //
3538 address generate_cipherBlockChaining_decryptAESCrypt() {
3539 assert(UseAES, "need AES cryptographic extension support");
3540 StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
3541 int entry_count = StubInfo::entry_count(stub_id);
3542 assert(entry_count == 1, "sanity check");
3543 address start = load_archive_data(stub_id);
3544 if (start != nullptr) {
3545 return start;
3546 }
3547 __ align(CodeEntryAlignment);
3548 StubCodeMark mark(this, stub_id);
3549
3550 Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52;
3551
3552 const Register from = c_rarg0; // source array address
3553 const Register to = c_rarg1; // destination array address
3554 const Register key = c_rarg2; // key array address
3555 const Register rvec = c_rarg3; // r byte array initialized from initvector array address
3556 // and left with the results of the last encryption block
3557 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
3558 const Register keylen = rscratch1;
3559
3560 start = __ pc();
3561
3562 __ enter();
3563
3564 __ movw(rscratch2, len_reg);
3565
3566 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3567
3568 __ ld1(v2, __ T16B, rvec);
3569
3570 __ ld1(v31, __ T16B, __ post(key, 16));
3571 __ rev32(v31, __ T16B, v31);
3572
3573 __ cmpw(keylen, 52);
3574 __ br(Assembler::CC, L_loadkeys_44);
3575 __ br(Assembler::EQ, L_loadkeys_52);
3576
3577 __ ld1(v17, v18, __ T16B, __ post(key, 32));
3578 __ rev32(v17, __ T16B, v17);
3579 __ rev32(v18, __ T16B, v18);
3580 __ BIND(L_loadkeys_52);
3581 __ ld1(v19, v20, __ T16B, __ post(key, 32));
3582 __ rev32(v19, __ T16B, v19);
3583 __ rev32(v20, __ T16B, v20);
3584 __ BIND(L_loadkeys_44);
3585 __ ld1(v21, v22, v23, v24, __ T16B, __ post(key, 64));
3586 __ rev32(v21, __ T16B, v21);
3587 __ rev32(v22, __ T16B, v22);
3588 __ rev32(v23, __ T16B, v23);
3589 __ rev32(v24, __ T16B, v24);
3590 __ ld1(v25, v26, v27, v28, __ T16B, __ post(key, 64));
3591 __ rev32(v25, __ T16B, v25);
3592 __ rev32(v26, __ T16B, v26);
3593 __ rev32(v27, __ T16B, v27);
3594 __ rev32(v28, __ T16B, v28);
3595 __ ld1(v29, v30, __ T16B, key);
3596 __ rev32(v29, __ T16B, v29);
3597 __ rev32(v30, __ T16B, v30);
3598
3599 __ BIND(L_aes_loop);
3600 __ ld1(v0, __ T16B, __ post(from, 16));
3601 __ orr(v1, __ T16B, v0, v0);
3602
3603 __ br(Assembler::CC, L_rounds_44);
3604 __ br(Assembler::EQ, L_rounds_52);
3605
3606 __ aesd(v0, v17); __ aesimc(v0, v0);
3607 __ aesd(v0, v18); __ aesimc(v0, v0);
3608 __ BIND(L_rounds_52);
3609 __ aesd(v0, v19); __ aesimc(v0, v0);
3610 __ aesd(v0, v20); __ aesimc(v0, v0);
3611 __ BIND(L_rounds_44);
3612 __ aesd(v0, v21); __ aesimc(v0, v0);
3613 __ aesd(v0, v22); __ aesimc(v0, v0);
3614 __ aesd(v0, v23); __ aesimc(v0, v0);
3615 __ aesd(v0, v24); __ aesimc(v0, v0);
3616 __ aesd(v0, v25); __ aesimc(v0, v0);
3617 __ aesd(v0, v26); __ aesimc(v0, v0);
3618 __ aesd(v0, v27); __ aesimc(v0, v0);
3619 __ aesd(v0, v28); __ aesimc(v0, v0);
3620 __ aesd(v0, v29); __ aesimc(v0, v0);
3621 __ aesd(v0, v30);
3622 __ eor(v0, __ T16B, v0, v31);
3623 __ eor(v0, __ T16B, v0, v2);
3624
3625 __ st1(v0, __ T16B, __ post(to, 16));
3626 __ orr(v2, __ T16B, v1, v1);
3627
3628 __ subw(len_reg, len_reg, 16);
3629 __ cbnzw(len_reg, L_aes_loop);
3630
3631 __ st1(v2, __ T16B, rvec);
3632
3633 __ mov(r0, rscratch2);
3634
3635 __ leave();
3636 __ ret(lr);
3637
3638 // record the stub entry and end
3639 store_archive_data(stub_id, start, __ pc());
3640
3641 return start;
3642 }
3643
3644 // Big-endian 128-bit + 64-bit -> 128-bit addition.
3645 // Inputs: 128-bits. in is preserved.
3646 // The least-significant 64-bit word is in the upper dword of each vector.
3647 // inc (the 64-bit increment) is preserved. Its lower dword must be zero.
3648 // Output: result
3649 void be_add_128_64(FloatRegister result, FloatRegister in,
3650 FloatRegister inc, FloatRegister tmp) {
3651 assert_different_registers(result, tmp, inc);
3652
3653 __ addv(result, __ T2D, in, inc); // Add inc to the least-significant dword of
3654 // input
3655 __ cm(__ HI, tmp, __ T2D, inc, result);// Check for result overflowing
3656 __ ext(tmp, __ T16B, tmp, tmp, 0x08); // Swap LSD of comparison result to MSD and
3657 // MSD == 0 (must be!) to LSD
3658 __ subv(result, __ T2D, result, tmp); // Subtract -1 from MSD if there was an overflow
3659 }
3660
3661 // CTR AES crypt.
3662 // Arguments:
3663 //
3664 // Inputs:
3665 // c_rarg0 - source byte array address
3666 // c_rarg1 - destination byte array address
3667 // c_rarg2 - sessionKe (key) in little endian int array
3668 // c_rarg3 - counter vector byte array address
3669 // c_rarg4 - input length
3670 // c_rarg5 - saved encryptedCounter start
3671 // c_rarg6 - saved used length
3672 //
3673 // Output:
3674 // r0 - input length
3675 //
3676 address generate_counterMode_AESCrypt() {
3677 StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
3678 int entry_count = StubInfo::entry_count(stub_id);
3679 assert(entry_count == 1, "sanity check");
3680 address start = load_archive_data(stub_id);
3681 if (start != nullptr) {
3682 return start;
3683 }
3684 const Register in = c_rarg0;
3685 const Register out = c_rarg1;
3686 const Register key = c_rarg2;
3687 const Register counter = c_rarg3;
3688 const Register saved_len = c_rarg4, len = r10;
3689 const Register saved_encrypted_ctr = c_rarg5;
3690 const Register used_ptr = c_rarg6, used = r12;
3691
3692 const Register offset = r7;
3693 const Register keylen = r11;
3694
3695 const unsigned char block_size = 16;
3696 const int bulk_width = 4;
3697 // NB: bulk_width can be 4 or 8. 8 gives slightly faster
3698 // performance with larger data sizes, but it also means that the
3699 // fast path isn't used until you have at least 8 blocks, and up
3700 // to 127 bytes of data will be executed on the slow path. For
3701 // that reason, and also so as not to blow away too much icache, 4
3702 // blocks seems like a sensible compromise.
3703
3704 // Algorithm:
3705 //
3706 // if (len == 0) {
3707 // goto DONE;
3708 // }
3709 // int result = len;
3710 // do {
3711 // if (used >= blockSize) {
3712 // if (len >= bulk_width * blockSize) {
3713 // CTR_large_block();
3714 // if (len == 0)
3715 // goto DONE;
3716 // }
3717 // for (;;) {
3718 // 16ByteVector v0 = counter;
3719 // embeddedCipher.encryptBlock(v0, 0, encryptedCounter, 0);
3720 // used = 0;
3721 // if (len < blockSize)
3722 // break; /* goto NEXT */
3723 // 16ByteVector v1 = load16Bytes(in, offset);
3724 // v1 = v1 ^ encryptedCounter;
3725 // store16Bytes(out, offset);
3726 // used = blockSize;
3727 // offset += blockSize;
3728 // len -= blockSize;
3729 // if (len == 0)
3730 // goto DONE;
3731 // }
3732 // }
3733 // NEXT:
3734 // out[outOff++] = (byte)(in[inOff++] ^ encryptedCounter[used++]);
3735 // len--;
3736 // } while (len != 0);
3737 // DONE:
3738 // return result;
3739 //
3740 // CTR_large_block()
3741 // Wide bulk encryption of whole blocks.
3742
3743 __ align(CodeEntryAlignment);
3744 StubCodeMark mark(this, stub_id);
3745 start = __ pc();
3746 __ enter();
3747
3748 Label DONE, CTR_large_block, large_block_return;
3749 __ ldrw(used, Address(used_ptr));
3750 __ cbzw(saved_len, DONE);
3751
3752 __ mov(len, saved_len);
3753 __ mov(offset, 0);
3754
3755 // Compute #rounds for AES based on the length of the key array
3756 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3757
3758 __ aesenc_loadkeys(key, keylen);
3759
3760 {
3761 Label L_CTR_loop, NEXT;
3762
3763 __ bind(L_CTR_loop);
3764
3765 __ cmp(used, block_size);
3766 __ br(__ LO, NEXT);
3767
3768 // Maybe we have a lot of data
3769 __ subsw(rscratch1, len, bulk_width * block_size);
3770 __ br(__ HS, CTR_large_block);
3771 __ BIND(large_block_return);
3772 __ cbzw(len, DONE);
3773
3774 // Setup the counter
3775 __ movi(v4, __ T4S, 0);
3776 __ movi(v5, __ T4S, 1);
3777 __ ins(v4, __ S, v5, 2, 2); // v4 contains { 0, 1 }
3778
3779 // 128-bit big-endian increment
3780 __ ld1(v0, __ T16B, counter);
3781 __ rev64(v16, __ T16B, v0);
3782 be_add_128_64(v16, v16, v4, /*tmp*/v5);
3783 __ rev64(v16, __ T16B, v16);
3784 __ st1(v16, __ T16B, counter);
3785 // Previous counter value is in v0
3786 // v4 contains { 0, 1 }
3787
3788 {
3789 // We have fewer than bulk_width blocks of data left. Encrypt
3790 // them one by one until there is less than a full block
3791 // remaining, being careful to save both the encrypted counter
3792 // and the counter.
3793
3794 Label inner_loop;
3795 __ bind(inner_loop);
3796 // Counter to encrypt is in v0
3797 __ aesecb_encrypt(noreg, noreg, keylen);
3798 __ st1(v0, __ T16B, saved_encrypted_ctr);
3799
3800 // Do we have a remaining full block?
3801
3802 __ mov(used, 0);
3803 __ cmp(len, block_size);
3804 __ br(__ LO, NEXT);
3805
3806 // Yes, we have a full block
3807 __ ldrq(v1, Address(in, offset));
3808 __ eor(v1, __ T16B, v1, v0);
3809 __ strq(v1, Address(out, offset));
3810 __ mov(used, block_size);
3811 __ add(offset, offset, block_size);
3812
3813 __ subw(len, len, block_size);
3814 __ cbzw(len, DONE);
3815
3816 // Increment the counter, store it back
3817 __ orr(v0, __ T16B, v16, v16);
3818 __ rev64(v16, __ T16B, v16);
3819 be_add_128_64(v16, v16, v4, /*tmp*/v5);
3820 __ rev64(v16, __ T16B, v16);
3821 __ st1(v16, __ T16B, counter); // Save the incremented counter back
3822
3823 __ b(inner_loop);
3824 }
3825
3826 __ BIND(NEXT);
3827
3828 // Encrypt a single byte, and loop.
3829 // We expect this to be a rare event.
3830 __ ldrb(rscratch1, Address(in, offset));
3831 __ ldrb(rscratch2, Address(saved_encrypted_ctr, used));
3832 __ eor(rscratch1, rscratch1, rscratch2);
3833 __ strb(rscratch1, Address(out, offset));
3834 __ add(offset, offset, 1);
3835 __ add(used, used, 1);
3836 __ subw(len, len,1);
3837 __ cbnzw(len, L_CTR_loop);
3838 }
3839
3840 __ bind(DONE);
3841 __ strw(used, Address(used_ptr));
3842 __ mov(r0, saved_len);
3843
3844 __ leave(); // required for proper stackwalking of RuntimeStub frame
3845 __ ret(lr);
3846
3847 // Bulk encryption
3848
3849 __ BIND (CTR_large_block);
3850 assert(bulk_width == 4 || bulk_width == 8, "must be");
3851
3852 if (bulk_width == 8) {
3853 __ sub(sp, sp, 4 * 16);
3854 __ st1(v12, v13, v14, v15, __ T16B, Address(sp));
3855 }
3856 __ sub(sp, sp, 4 * 16);
3857 __ st1(v8, v9, v10, v11, __ T16B, Address(sp));
3858 RegSet saved_regs = (RegSet::of(in, out, offset)
3859 + RegSet::of(saved_encrypted_ctr, used_ptr, len));
3860 __ push(saved_regs, sp);
3861 __ andr(len, len, -16 * bulk_width); // 8/4 encryptions, 16 bytes per encryption
3862 __ add(in, in, offset);
3863 __ add(out, out, offset);
3864
3865 // Keys should already be loaded into the correct registers
3866
3867 __ ld1(v0, __ T16B, counter); // v0 contains the first counter
3868 __ rev64(v16, __ T16B, v0); // v16 contains byte-reversed counter
3869
3870 // AES/CTR loop
3871 {
3872 Label L_CTR_loop;
3873 __ BIND(L_CTR_loop);
3874
3875 // Setup the counters
3876 __ movi(v8, __ T4S, 0);
3877 __ movi(v9, __ T4S, 1);
3878 __ ins(v8, __ S, v9, 2, 2); // v8 contains { 0, 1 }
3879
3880 for (int i = 0; i < bulk_width; i++) {
3881 FloatRegister v0_ofs = as_FloatRegister(v0->encoding() + i);
3882 __ rev64(v0_ofs, __ T16B, v16);
3883 be_add_128_64(v16, v16, v8, /*tmp*/v9);
3884 }
3885
3886 __ ld1(v8, v9, v10, v11, __ T16B, __ post(in, 4 * 16));
3887
3888 // Encrypt the counters
3889 __ aesecb_encrypt(noreg, noreg, keylen, v0, bulk_width);
3890
3891 if (bulk_width == 8) {
3892 __ ld1(v12, v13, v14, v15, __ T16B, __ post(in, 4 * 16));
3893 }
3894
3895 // XOR the encrypted counters with the inputs
3896 for (int i = 0; i < bulk_width; i++) {
3897 FloatRegister v0_ofs = as_FloatRegister(v0->encoding() + i);
3898 FloatRegister v8_ofs = as_FloatRegister(v8->encoding() + i);
3899 __ eor(v0_ofs, __ T16B, v0_ofs, v8_ofs);
3900 }
3901
3902 // Write the encrypted data
3903 __ st1(v0, v1, v2, v3, __ T16B, __ post(out, 4 * 16));
3904 if (bulk_width == 8) {
3905 __ st1(v4, v5, v6, v7, __ T16B, __ post(out, 4 * 16));
3906 }
3907
3908 __ subw(len, len, 16 * bulk_width);
3909 __ cbnzw(len, L_CTR_loop);
3910 }
3911
3912 // Save the counter back where it goes
3913 __ rev64(v16, __ T16B, v16);
3914 __ st1(v16, __ T16B, counter);
3915
3916 __ pop(saved_regs, sp);
3917
3918 __ ld1(v8, v9, v10, v11, __ T16B, __ post(sp, 4 * 16));
3919 if (bulk_width == 8) {
3920 __ ld1(v12, v13, v14, v15, __ T16B, __ post(sp, 4 * 16));
3921 }
3922
3923 __ andr(rscratch1, len, -16 * bulk_width);
3924 __ sub(len, len, rscratch1);
3925 __ add(offset, offset, rscratch1);
3926 __ mov(used, 16);
3927 __ strw(used, Address(used_ptr));
3928 __ b(large_block_return);
3929
3930 // record the stub entry and end
3931 store_archive_data(stub_id, start, __ pc());
3932
3933 return start;
3934 }
3935
3936 // Vector AES Galois Counter Mode implementation. Parameters:
3937 //
3938 // in = c_rarg0
3939 // len = c_rarg1
3940 // ct = c_rarg2 - ciphertext that ghash will read (in for encrypt, out for decrypt)
3941 // out = c_rarg3
3942 // key = c_rarg4
3943 // state = c_rarg5 - GHASH.state
3944 // subkeyHtbl = c_rarg6 - powers of H
3945 // counter = c_rarg7 - 16 bytes of CTR
3946 // return - number of processed bytes
3947 address generate_galoisCounterMode_AESCrypt() {
3948 Label ghash_polynomial; // local data generated after code
3949 StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
3950 int entry_count = StubInfo::entry_count(stub_id);
3951 assert(entry_count == 1, "sanity check");
3952 address start = load_archive_data(stub_id);
3953 if (start != nullptr) {
3954 return start;
3955 }
3956 __ align(CodeEntryAlignment);
3957 StubCodeMark mark(this, stub_id);
3958 start = __ pc();
3959 __ enter();
3960
3961 const Register in = c_rarg0;
3962 const Register len = c_rarg1;
3963 const Register ct = c_rarg2;
3964 const Register out = c_rarg3;
3965 // and updated with the incremented counter in the end
3966
3967 const Register key = c_rarg4;
3968 const Register state = c_rarg5;
3969
3970 const Register subkeyHtbl = c_rarg6;
3971
3972 const Register counter = c_rarg7;
3973
3974 const Register keylen = r10;
3975 // Save state before entering routine
3976 __ sub(sp, sp, 4 * 16);
3977 __ st1(v12, v13, v14, v15, __ T16B, Address(sp));
3978 __ sub(sp, sp, 4 * 16);
3979 __ st1(v8, v9, v10, v11, __ T16B, Address(sp));
3980
3981 // __ andr(len, len, -512);
3982 __ andr(len, len, -16 * 8); // 8 encryptions, 16 bytes per encryption
3983 __ str(len, __ pre(sp, -2 * wordSize));
3984
3985 Label DONE;
3986 __ cbz(len, DONE);
3987
3988 // Compute #rounds for AES based on the length of the key array
3989 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3990
3991 __ aesenc_loadkeys(key, keylen);
3992 __ ld1(v0, __ T16B, counter); // v0 contains the first counter
3993 __ rev32(v16, __ T16B, v0); // v16 contains byte-reversed counter
3994
3995 // AES/CTR loop
3996 {
3997 Label L_CTR_loop;
3998 __ BIND(L_CTR_loop);
3999
4000 // Setup the counters
4001 __ movi(v8, __ T4S, 0);
4002 __ movi(v9, __ T4S, 1);
4003 __ ins(v8, __ S, v9, 3, 3); // v8 contains { 0, 0, 0, 1 }
4004
4005 assert(v0->encoding() < v8->encoding(), "");
4006 for (int i = v0->encoding(); i < v8->encoding(); i++) {
4007 FloatRegister f = as_FloatRegister(i);
4008 __ rev32(f, __ T16B, v16);
4009 __ addv(v16, __ T4S, v16, v8);
4010 }
4011
4012 __ ld1(v8, v9, v10, v11, __ T16B, __ post(in, 4 * 16));
4013
4014 // Encrypt the counters
4015 __ aesecb_encrypt(noreg, noreg, keylen, v0, /*unrolls*/8);
4016
4017 __ ld1(v12, v13, v14, v15, __ T16B, __ post(in, 4 * 16));
4018
4019 // XOR the encrypted counters with the inputs
4020 for (int i = 0; i < 8; i++) {
4021 FloatRegister v0_ofs = as_FloatRegister(v0->encoding() + i);
4022 FloatRegister v8_ofs = as_FloatRegister(v8->encoding() + i);
4023 __ eor(v0_ofs, __ T16B, v0_ofs, v8_ofs);
4024 }
4025 __ st1(v0, v1, v2, v3, __ T16B, __ post(out, 4 * 16));
4026 __ st1(v4, v5, v6, v7, __ T16B, __ post(out, 4 * 16));
4027
4028 __ subw(len, len, 16 * 8);
4029 __ cbnzw(len, L_CTR_loop);
4030 }
4031
4032 __ rev32(v16, __ T16B, v16);
4033 __ st1(v16, __ T16B, counter);
4034
4035 __ ldr(len, Address(sp));
4036 __ lsr(len, len, exact_log2(16)); // We want the count of blocks
4037
4038 // GHASH/CTR loop
4039 __ ghash_processBlocks_wide(ghash_polynomial, state, subkeyHtbl, ct,
4040 len, /*unrolls*/4);
4041
4042 #ifdef ASSERT
4043 { Label L;
4044 __ cmp(len, (unsigned char)0);
4045 __ br(Assembler::EQ, L);
4046 __ stop("stubGenerator: abort");
4047 __ bind(L);
4048 }
4049 #endif
4050
4051 __ bind(DONE);
4052 // Return the number of bytes processed
4053 __ ldr(r0, __ post(sp, 2 * wordSize));
4054
4055 __ ld1(v8, v9, v10, v11, __ T16B, __ post(sp, 4 * 16));
4056 __ ld1(v12, v13, v14, v15, __ T16B, __ post(sp, 4 * 16));
4057
4058 __ leave(); // required for proper stackwalking of RuntimeStub frame
4059 __ ret(lr);
4060
4061 // bind label and generate polynomial data
4062 __ align(wordSize * 2);
4063 __ bind(ghash_polynomial);
4064 __ emit_int64(0x87); // The low-order bits of the field
4065 // polynomial (i.e. p = z^7+z^2+z+1)
4066 // repeated in the low and high parts of a
4067 // 128-bit vector
4068 __ emit_int64(0x87);
4069
4070 // record the stub entry and end
4071 store_archive_data(stub_id, start, __ pc());
4072
4073 return start;
4074 }
4075
4076 class Cached64Bytes {
4077 private:
4078 MacroAssembler *_masm;
4079 Register _regs[8];
4080
4081 public:
4082 Cached64Bytes(MacroAssembler *masm, RegSet rs): _masm(masm) {
4083 assert(rs.size() == 8, "%u registers are used to cache 16 4-byte data", rs.size());
4084 auto it = rs.begin();
4085 for (auto &r: _regs) {
4086 r = *it;
4087 ++it;
4088 }
4089 }
4090
4091 void gen_loads(Register base) {
4092 for (int i = 0; i < 8; i += 2) {
4093 __ ldp(_regs[i], _regs[i + 1], Address(base, 8 * i));
4094 }
4095 }
4096
4097 // Generate code extracting i-th unsigned word (4 bytes) from cached 64 bytes.
4098 void extract_u32(Register dest, int i) {
4099 __ ubfx(dest, _regs[i / 2], 32 * (i % 2), 32);
4100 }
4101 };
4102
4103 // Utility routines for md5.
4104 // Clobbers r10 and r11.
4105 void md5_FF(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
4106 int k, int s, int t) {
4107 Register rscratch3 = r10;
4108 Register rscratch4 = r11;
4109
4110 __ eorw(rscratch3, r3, r4);
4111 __ movw(rscratch2, t);
4112 __ andw(rscratch3, rscratch3, r2);
4113 __ addw(rscratch4, r1, rscratch2);
4114 reg_cache.extract_u32(rscratch1, k);
4115 __ eorw(rscratch3, rscratch3, r4);
4116 __ addw(rscratch4, rscratch4, rscratch1);
4117 __ addw(rscratch3, rscratch3, rscratch4);
4118 __ rorw(rscratch2, rscratch3, 32 - s);
4119 __ addw(r1, rscratch2, r2);
4120 }
4121
4122 void md5_GG(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
4123 int k, int s, int t) {
4124 Register rscratch3 = r10;
4125 Register rscratch4 = r11;
4126
4127 reg_cache.extract_u32(rscratch1, k);
4128 __ movw(rscratch2, t);
4129 __ addw(rscratch4, r1, rscratch2);
4130 __ addw(rscratch4, rscratch4, rscratch1);
4131 __ bicw(rscratch2, r3, r4);
4132 __ andw(rscratch3, r2, r4);
4133 __ addw(rscratch2, rscratch2, rscratch4);
4134 __ addw(rscratch2, rscratch2, rscratch3);
4135 __ rorw(rscratch2, rscratch2, 32 - s);
4136 __ addw(r1, rscratch2, r2);
4137 }
4138
4139 void md5_HH(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
4140 int k, int s, int t) {
4141 Register rscratch3 = r10;
4142 Register rscratch4 = r11;
4143
4144 __ eorw(rscratch3, r3, r4);
4145 __ movw(rscratch2, t);
4146 __ addw(rscratch4, r1, rscratch2);
4147 reg_cache.extract_u32(rscratch1, k);
4148 __ eorw(rscratch3, rscratch3, r2);
4149 __ addw(rscratch4, rscratch4, rscratch1);
4150 __ addw(rscratch3, rscratch3, rscratch4);
4151 __ rorw(rscratch2, rscratch3, 32 - s);
4152 __ addw(r1, rscratch2, r2);
4153 }
4154
4155 void md5_II(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
4156 int k, int s, int t) {
4157 Register rscratch3 = r10;
4158 Register rscratch4 = r11;
4159
4160 __ movw(rscratch3, t);
4161 __ ornw(rscratch2, r2, r4);
4162 __ addw(rscratch4, r1, rscratch3);
4163 reg_cache.extract_u32(rscratch1, k);
4164 __ eorw(rscratch3, rscratch2, r3);
4165 __ addw(rscratch4, rscratch4, rscratch1);
4166 __ addw(rscratch3, rscratch3, rscratch4);
4167 __ rorw(rscratch2, rscratch3, 32 - s);
4168 __ addw(r1, rscratch2, r2);
4169 }
4170
4171 // Arguments:
4172 //
4173 // Inputs:
4174 // c_rarg0 - byte[] source+offset
4175 // c_rarg1 - int[] SHA.state
4176 // c_rarg2 - int offset
4177 // c_rarg3 - int limit
4178 //
4179 address generate_md5_implCompress(StubId stub_id) {
4180 bool multi_block;
4181 switch (stub_id) {
4182 case StubId::stubgen_md5_implCompress_id:
4183 multi_block = false;
4184 break;
4185 case StubId::stubgen_md5_implCompressMB_id:
4186 multi_block = true;
4187 break;
4188 default:
4189 ShouldNotReachHere();
4190 }
4191 int entry_count = StubInfo::entry_count(stub_id);
4192 assert(entry_count == 1, "sanity check");
4193 address start = load_archive_data(stub_id);
4194 if (start != nullptr) {
4195 return start;
4196 }
4197 __ align(CodeEntryAlignment);
4198
4199 StubCodeMark mark(this, stub_id);
4200 start = __ pc();
4201
4202 Register buf = c_rarg0;
4203 Register state = c_rarg1;
4204 Register ofs = c_rarg2;
4205 Register limit = c_rarg3;
4206 Register a = r4;
4207 Register b = r5;
4208 Register c = r6;
4209 Register d = r7;
4210 Register rscratch3 = r10;
4211 Register rscratch4 = r11;
4212
4213 Register state_regs[2] = { r12, r13 };
4214 RegSet saved_regs = RegSet::range(r16, r22) - r18_tls;
4215 Cached64Bytes reg_cache(_masm, RegSet::of(r14, r15) + saved_regs); // using 8 registers
4216
4217 __ push(saved_regs, sp);
4218
4219 __ ldp(state_regs[0], state_regs[1], Address(state));
4220 __ ubfx(a, state_regs[0], 0, 32);
4221 __ ubfx(b, state_regs[0], 32, 32);
4222 __ ubfx(c, state_regs[1], 0, 32);
4223 __ ubfx(d, state_regs[1], 32, 32);
4224
4225 Label md5_loop;
4226 __ BIND(md5_loop);
4227
4228 reg_cache.gen_loads(buf);
4229
4230 // Round 1
4231 md5_FF(reg_cache, a, b, c, d, 0, 7, 0xd76aa478);
4232 md5_FF(reg_cache, d, a, b, c, 1, 12, 0xe8c7b756);
4233 md5_FF(reg_cache, c, d, a, b, 2, 17, 0x242070db);
4234 md5_FF(reg_cache, b, c, d, a, 3, 22, 0xc1bdceee);
4235 md5_FF(reg_cache, a, b, c, d, 4, 7, 0xf57c0faf);
4236 md5_FF(reg_cache, d, a, b, c, 5, 12, 0x4787c62a);
4237 md5_FF(reg_cache, c, d, a, b, 6, 17, 0xa8304613);
4238 md5_FF(reg_cache, b, c, d, a, 7, 22, 0xfd469501);
4239 md5_FF(reg_cache, a, b, c, d, 8, 7, 0x698098d8);
4240 md5_FF(reg_cache, d, a, b, c, 9, 12, 0x8b44f7af);
4241 md5_FF(reg_cache, c, d, a, b, 10, 17, 0xffff5bb1);
4242 md5_FF(reg_cache, b, c, d, a, 11, 22, 0x895cd7be);
4243 md5_FF(reg_cache, a, b, c, d, 12, 7, 0x6b901122);
4244 md5_FF(reg_cache, d, a, b, c, 13, 12, 0xfd987193);
4245 md5_FF(reg_cache, c, d, a, b, 14, 17, 0xa679438e);
4246 md5_FF(reg_cache, b, c, d, a, 15, 22, 0x49b40821);
4247
4248 // Round 2
4249 md5_GG(reg_cache, a, b, c, d, 1, 5, 0xf61e2562);
4250 md5_GG(reg_cache, d, a, b, c, 6, 9, 0xc040b340);
4251 md5_GG(reg_cache, c, d, a, b, 11, 14, 0x265e5a51);
4252 md5_GG(reg_cache, b, c, d, a, 0, 20, 0xe9b6c7aa);
4253 md5_GG(reg_cache, a, b, c, d, 5, 5, 0xd62f105d);
4254 md5_GG(reg_cache, d, a, b, c, 10, 9, 0x02441453);
4255 md5_GG(reg_cache, c, d, a, b, 15, 14, 0xd8a1e681);
4256 md5_GG(reg_cache, b, c, d, a, 4, 20, 0xe7d3fbc8);
4257 md5_GG(reg_cache, a, b, c, d, 9, 5, 0x21e1cde6);
4258 md5_GG(reg_cache, d, a, b, c, 14, 9, 0xc33707d6);
4259 md5_GG(reg_cache, c, d, a, b, 3, 14, 0xf4d50d87);
4260 md5_GG(reg_cache, b, c, d, a, 8, 20, 0x455a14ed);
4261 md5_GG(reg_cache, a, b, c, d, 13, 5, 0xa9e3e905);
4262 md5_GG(reg_cache, d, a, b, c, 2, 9, 0xfcefa3f8);
4263 md5_GG(reg_cache, c, d, a, b, 7, 14, 0x676f02d9);
4264 md5_GG(reg_cache, b, c, d, a, 12, 20, 0x8d2a4c8a);
4265
4266 // Round 3
4267 md5_HH(reg_cache, a, b, c, d, 5, 4, 0xfffa3942);
4268 md5_HH(reg_cache, d, a, b, c, 8, 11, 0x8771f681);
4269 md5_HH(reg_cache, c, d, a, b, 11, 16, 0x6d9d6122);
4270 md5_HH(reg_cache, b, c, d, a, 14, 23, 0xfde5380c);
4271 md5_HH(reg_cache, a, b, c, d, 1, 4, 0xa4beea44);
4272 md5_HH(reg_cache, d, a, b, c, 4, 11, 0x4bdecfa9);
4273 md5_HH(reg_cache, c, d, a, b, 7, 16, 0xf6bb4b60);
4274 md5_HH(reg_cache, b, c, d, a, 10, 23, 0xbebfbc70);
4275 md5_HH(reg_cache, a, b, c, d, 13, 4, 0x289b7ec6);
4276 md5_HH(reg_cache, d, a, b, c, 0, 11, 0xeaa127fa);
4277 md5_HH(reg_cache, c, d, a, b, 3, 16, 0xd4ef3085);
4278 md5_HH(reg_cache, b, c, d, a, 6, 23, 0x04881d05);
4279 md5_HH(reg_cache, a, b, c, d, 9, 4, 0xd9d4d039);
4280 md5_HH(reg_cache, d, a, b, c, 12, 11, 0xe6db99e5);
4281 md5_HH(reg_cache, c, d, a, b, 15, 16, 0x1fa27cf8);
4282 md5_HH(reg_cache, b, c, d, a, 2, 23, 0xc4ac5665);
4283
4284 // Round 4
4285 md5_II(reg_cache, a, b, c, d, 0, 6, 0xf4292244);
4286 md5_II(reg_cache, d, a, b, c, 7, 10, 0x432aff97);
4287 md5_II(reg_cache, c, d, a, b, 14, 15, 0xab9423a7);
4288 md5_II(reg_cache, b, c, d, a, 5, 21, 0xfc93a039);
4289 md5_II(reg_cache, a, b, c, d, 12, 6, 0x655b59c3);
4290 md5_II(reg_cache, d, a, b, c, 3, 10, 0x8f0ccc92);
4291 md5_II(reg_cache, c, d, a, b, 10, 15, 0xffeff47d);
4292 md5_II(reg_cache, b, c, d, a, 1, 21, 0x85845dd1);
4293 md5_II(reg_cache, a, b, c, d, 8, 6, 0x6fa87e4f);
4294 md5_II(reg_cache, d, a, b, c, 15, 10, 0xfe2ce6e0);
4295 md5_II(reg_cache, c, d, a, b, 6, 15, 0xa3014314);
4296 md5_II(reg_cache, b, c, d, a, 13, 21, 0x4e0811a1);
4297 md5_II(reg_cache, a, b, c, d, 4, 6, 0xf7537e82);
4298 md5_II(reg_cache, d, a, b, c, 11, 10, 0xbd3af235);
4299 md5_II(reg_cache, c, d, a, b, 2, 15, 0x2ad7d2bb);
4300 md5_II(reg_cache, b, c, d, a, 9, 21, 0xeb86d391);
4301
4302 __ addw(a, state_regs[0], a);
4303 __ ubfx(rscratch2, state_regs[0], 32, 32);
4304 __ addw(b, rscratch2, b);
4305 __ addw(c, state_regs[1], c);
4306 __ ubfx(rscratch4, state_regs[1], 32, 32);
4307 __ addw(d, rscratch4, d);
4308
4309 __ orr(state_regs[0], a, b, Assembler::LSL, 32);
4310 __ orr(state_regs[1], c, d, Assembler::LSL, 32);
4311
4312 if (multi_block) {
4313 __ add(buf, buf, 64);
4314 __ add(ofs, ofs, 64);
4315 __ cmp(ofs, limit);
4316 __ br(Assembler::LE, md5_loop);
4317 __ mov(c_rarg0, ofs); // return ofs
4318 }
4319
4320 // write hash values back in the correct order
4321 __ stp(state_regs[0], state_regs[1], Address(state));
4322
4323 __ pop(saved_regs, sp);
4324
4325 __ ret(lr);
4326
4327 // record the stub entry and end
4328 store_archive_data(stub_id, start, __ pc());
4329
4330 return start;
4331 }
4332
4333 // Arguments:
4334 //
4335 // Inputs:
4336 // c_rarg0 - byte[] source+offset
4337 // c_rarg1 - int[] SHA.state
4338 // c_rarg2 - int offset
4339 // c_rarg3 - int limit
4340 //
4341 address generate_sha1_implCompress(StubId stub_id) {
4342 bool multi_block;
4343 switch (stub_id) {
4344 case StubId::stubgen_sha1_implCompress_id:
4345 multi_block = false;
4346 break;
4347 case StubId::stubgen_sha1_implCompressMB_id:
4348 multi_block = true;
4349 break;
4350 default:
4351 ShouldNotReachHere();
4352 }
4353 int entry_count = StubInfo::entry_count(stub_id);
4354 assert(entry_count == 1, "sanity check");
4355 address start = load_archive_data(stub_id);
4356 if (start != nullptr) {
4357 return start;
4358 }
4359 __ align(CodeEntryAlignment);
4360
4361 StubCodeMark mark(this, stub_id);
4362 start = __ pc();
4363
4364 Register buf = c_rarg0;
4365 Register state = c_rarg1;
4366 Register ofs = c_rarg2;
4367 Register limit = c_rarg3;
4368
4369 Label keys;
4370 Label sha1_loop;
4371
4372 // load the keys into v0..v3
4373 __ adr(rscratch1, keys);
4374 __ ld4r(v0, v1, v2, v3, __ T4S, Address(rscratch1));
4375 // load 5 words state into v6, v7
4376 __ ldrq(v6, Address(state, 0));
4377 __ ldrs(v7, Address(state, 16));
4378
4379
4380 __ BIND(sha1_loop);
4381 // load 64 bytes of data into v16..v19
4382 __ ld1(v16, v17, v18, v19, __ T4S, multi_block ? __ post(buf, 64) : buf);
4383 __ rev32(v16, __ T16B, v16);
4384 __ rev32(v17, __ T16B, v17);
4385 __ rev32(v18, __ T16B, v18);
4386 __ rev32(v19, __ T16B, v19);
4387
4388 // do the sha1
4389 __ addv(v4, __ T4S, v16, v0);
4390 __ orr(v20, __ T16B, v6, v6);
4391
4392 FloatRegister d0 = v16;
4393 FloatRegister d1 = v17;
4394 FloatRegister d2 = v18;
4395 FloatRegister d3 = v19;
4396
4397 for (int round = 0; round < 20; round++) {
4398 FloatRegister tmp1 = (round & 1) ? v4 : v5;
4399 FloatRegister tmp2 = (round & 1) ? v21 : v22;
4400 FloatRegister tmp3 = round ? ((round & 1) ? v22 : v21) : v7;
4401 FloatRegister tmp4 = (round & 1) ? v5 : v4;
4402 FloatRegister key = (round < 4) ? v0 : ((round < 9) ? v1 : ((round < 14) ? v2 : v3));
4403
4404 if (round < 16) __ sha1su0(d0, __ T4S, d1, d2);
4405 if (round < 19) __ addv(tmp1, __ T4S, d1, key);
4406 __ sha1h(tmp2, __ T4S, v20);
4407 if (round < 5)
4408 __ sha1c(v20, __ T4S, tmp3, tmp4);
4409 else if (round < 10 || round >= 15)
4410 __ sha1p(v20, __ T4S, tmp3, tmp4);
4411 else
4412 __ sha1m(v20, __ T4S, tmp3, tmp4);
4413 if (round < 16) __ sha1su1(d0, __ T4S, d3);
4414
4415 tmp1 = d0; d0 = d1; d1 = d2; d2 = d3; d3 = tmp1;
4416 }
4417
4418 __ addv(v7, __ T2S, v7, v21);
4419 __ addv(v6, __ T4S, v6, v20);
4420
4421 if (multi_block) {
4422 __ add(ofs, ofs, 64);
4423 __ cmp(ofs, limit);
4424 __ br(Assembler::LE, sha1_loop);
4425 __ mov(c_rarg0, ofs); // return ofs
4426 }
4427
4428 __ strq(v6, Address(state, 0));
4429 __ strs(v7, Address(state, 16));
4430
4431 __ ret(lr);
4432
4433 __ bind(keys);
4434 __ emit_int32(0x5a827999);
4435 __ emit_int32(0x6ed9eba1);
4436 __ emit_int32(0x8f1bbcdc);
4437 __ emit_int32(0xca62c1d6);
4438
4439 // record the stub entry and end
4440 store_archive_data(stub_id, start, __ pc());
4441
4442 return start;
4443 }
4444
4445
4446 // Arguments:
4447 //
4448 // Inputs:
4449 // c_rarg0 - byte[] source+offset
4450 // c_rarg1 - int[] SHA.state
4451 // c_rarg2 - int offset
4452 // c_rarg3 - int limit
4453 //
4454 address generate_sha256_implCompress(StubId stub_id) {
4455 bool multi_block;
4456 switch (stub_id) {
4457 case StubId::stubgen_sha256_implCompress_id:
4458 multi_block = false;
4459 break;
4460 case StubId::stubgen_sha256_implCompressMB_id:
4461 multi_block = true;
4462 break;
4463 default:
4464 ShouldNotReachHere();
4465 }
4466 int entry_count = StubInfo::entry_count(stub_id);
4467 assert(entry_count == 1, "sanity check");
4468 address start = load_archive_data(stub_id);
4469 if (start != nullptr) {
4470 return start;
4471 }
4472 __ align(CodeEntryAlignment);
4473 StubCodeMark mark(this, stub_id);
4474 start = __ pc();
4475
4476 Register buf = c_rarg0;
4477 Register state = c_rarg1;
4478 Register ofs = c_rarg2;
4479 Register limit = c_rarg3;
4480
4481 Label sha1_loop;
4482
4483 __ stpd(v8, v9, __ pre(sp, -32));
4484 __ stpd(v10, v11, Address(sp, 16));
4485
4486 // dga == v0
4487 // dgb == v1
4488 // dg0 == v2
4489 // dg1 == v3
4490 // dg2 == v4
4491 // t0 == v6
4492 // t1 == v7
4493
4494 // load 16 keys to v16..v31
4495 __ lea(rscratch1, ExternalAddress((address)_sha256_round_consts));
4496 __ ld1(v16, v17, v18, v19, __ T4S, __ post(rscratch1, 64));
4497 __ ld1(v20, v21, v22, v23, __ T4S, __ post(rscratch1, 64));
4498 __ ld1(v24, v25, v26, v27, __ T4S, __ post(rscratch1, 64));
4499 __ ld1(v28, v29, v30, v31, __ T4S, rscratch1);
4500
4501 // load 8 words (256 bits) state
4502 __ ldpq(v0, v1, state);
4503
4504 __ BIND(sha1_loop);
4505 // load 64 bytes of data into v8..v11
4506 __ ld1(v8, v9, v10, v11, __ T4S, multi_block ? __ post(buf, 64) : buf);
4507 __ rev32(v8, __ T16B, v8);
4508 __ rev32(v9, __ T16B, v9);
4509 __ rev32(v10, __ T16B, v10);
4510 __ rev32(v11, __ T16B, v11);
4511
4512 __ addv(v6, __ T4S, v8, v16);
4513 __ orr(v2, __ T16B, v0, v0);
4514 __ orr(v3, __ T16B, v1, v1);
4515
4516 FloatRegister d0 = v8;
4517 FloatRegister d1 = v9;
4518 FloatRegister d2 = v10;
4519 FloatRegister d3 = v11;
4520
4521
4522 for (int round = 0; round < 16; round++) {
4523 FloatRegister tmp1 = (round & 1) ? v6 : v7;
4524 FloatRegister tmp2 = (round & 1) ? v7 : v6;
4525 FloatRegister tmp3 = (round & 1) ? v2 : v4;
4526 FloatRegister tmp4 = (round & 1) ? v4 : v2;
4527
4528 if (round < 12) __ sha256su0(d0, __ T4S, d1);
4529 __ orr(v4, __ T16B, v2, v2);
4530 if (round < 15)
4531 __ addv(tmp1, __ T4S, d1, as_FloatRegister(round + 17));
4532 __ sha256h(v2, __ T4S, v3, tmp2);
4533 __ sha256h2(v3, __ T4S, v4, tmp2);
4534 if (round < 12) __ sha256su1(d0, __ T4S, d2, d3);
4535
4536 tmp1 = d0; d0 = d1; d1 = d2; d2 = d3; d3 = tmp1;
4537 }
4538
4539 __ addv(v0, __ T4S, v0, v2);
4540 __ addv(v1, __ T4S, v1, v3);
4541
4542 if (multi_block) {
4543 __ add(ofs, ofs, 64);
4544 __ cmp(ofs, limit);
4545 __ br(Assembler::LE, sha1_loop);
4546 __ mov(c_rarg0, ofs); // return ofs
4547 }
4548
4549 __ ldpd(v10, v11, Address(sp, 16));
4550 __ ldpd(v8, v9, __ post(sp, 32));
4551
4552 __ stpq(v0, v1, state);
4553
4554 __ ret(lr);
4555
4556 // record the stub entry and end
4557 store_archive_data(stub_id, start, __ pc());
4558
4559 return start;
4560 }
4561
4562 // Double rounds for sha512.
4563 void sha512_dround(int dr,
4564 FloatRegister vi0, FloatRegister vi1,
4565 FloatRegister vi2, FloatRegister vi3,
4566 FloatRegister vi4, FloatRegister vrc0,
4567 FloatRegister vrc1, FloatRegister vin0,
4568 FloatRegister vin1, FloatRegister vin2,
4569 FloatRegister vin3, FloatRegister vin4) {
4570 if (dr < 36) {
4571 __ ld1(vrc1, __ T2D, __ post(rscratch2, 16));
4572 }
4573 __ addv(v5, __ T2D, vrc0, vin0);
4574 __ ext(v6, __ T16B, vi2, vi3, 8);
4575 __ ext(v5, __ T16B, v5, v5, 8);
4576 __ ext(v7, __ T16B, vi1, vi2, 8);
4577 __ addv(vi3, __ T2D, vi3, v5);
4578 if (dr < 32) {
4579 __ ext(v5, __ T16B, vin3, vin4, 8);
4580 __ sha512su0(vin0, __ T2D, vin1);
4581 }
4582 __ sha512h(vi3, __ T2D, v6, v7);
4583 if (dr < 32) {
4584 __ sha512su1(vin0, __ T2D, vin2, v5);
4585 }
4586 __ addv(vi4, __ T2D, vi1, vi3);
4587 __ sha512h2(vi3, __ T2D, vi1, vi0);
4588 }
4589
4590 // Arguments:
4591 //
4592 // Inputs:
4593 // c_rarg0 - byte[] source+offset
4594 // c_rarg1 - int[] SHA.state
4595 // c_rarg2 - int offset
4596 // c_rarg3 - int limit
4597 //
4598 address generate_sha512_implCompress(StubId stub_id) {
4599 bool multi_block;
4600 switch (stub_id) {
4601 case StubId::stubgen_sha512_implCompress_id:
4602 multi_block = false;
4603 break;
4604 case StubId::stubgen_sha512_implCompressMB_id:
4605 multi_block = true;
4606 break;
4607 default:
4608 ShouldNotReachHere();
4609 }
4610 int entry_count = StubInfo::entry_count(stub_id);
4611 assert(entry_count == 1, "sanity check");
4612 address start = load_archive_data(stub_id);
4613 if (start != nullptr) {
4614 return start;
4615 }
4616 __ align(CodeEntryAlignment);
4617 StubCodeMark mark(this, stub_id);
4618 start = __ pc();
4619
4620 Register buf = c_rarg0;
4621 Register state = c_rarg1;
4622 Register ofs = c_rarg2;
4623 Register limit = c_rarg3;
4624
4625 __ stpd(v8, v9, __ pre(sp, -64));
4626 __ stpd(v10, v11, Address(sp, 16));
4627 __ stpd(v12, v13, Address(sp, 32));
4628 __ stpd(v14, v15, Address(sp, 48));
4629
4630 Label sha512_loop;
4631
4632 // load state
4633 __ ld1(v8, v9, v10, v11, __ T2D, state);
4634
4635 // load first 4 round constants
4636 __ lea(rscratch1, ExternalAddress((address)_sha512_round_consts));
4637 __ ld1(v20, v21, v22, v23, __ T2D, __ post(rscratch1, 64));
4638
4639 __ BIND(sha512_loop);
4640 // load 128B of data into v12..v19
4641 __ ld1(v12, v13, v14, v15, __ T2D, __ post(buf, 64));
4642 __ ld1(v16, v17, v18, v19, __ T2D, __ post(buf, 64));
4643 __ rev64(v12, __ T16B, v12);
4644 __ rev64(v13, __ T16B, v13);
4645 __ rev64(v14, __ T16B, v14);
4646 __ rev64(v15, __ T16B, v15);
4647 __ rev64(v16, __ T16B, v16);
4648 __ rev64(v17, __ T16B, v17);
4649 __ rev64(v18, __ T16B, v18);
4650 __ rev64(v19, __ T16B, v19);
4651
4652 __ mov(rscratch2, rscratch1);
4653
4654 __ mov(v0, __ T16B, v8);
4655 __ mov(v1, __ T16B, v9);
4656 __ mov(v2, __ T16B, v10);
4657 __ mov(v3, __ T16B, v11);
4658
4659 sha512_dround( 0, v0, v1, v2, v3, v4, v20, v24, v12, v13, v19, v16, v17);
4660 sha512_dround( 1, v3, v0, v4, v2, v1, v21, v25, v13, v14, v12, v17, v18);
4661 sha512_dround( 2, v2, v3, v1, v4, v0, v22, v26, v14, v15, v13, v18, v19);
4662 sha512_dround( 3, v4, v2, v0, v1, v3, v23, v27, v15, v16, v14, v19, v12);
4663 sha512_dround( 4, v1, v4, v3, v0, v2, v24, v28, v16, v17, v15, v12, v13);
4664 sha512_dround( 5, v0, v1, v2, v3, v4, v25, v29, v17, v18, v16, v13, v14);
4665 sha512_dround( 6, v3, v0, v4, v2, v1, v26, v30, v18, v19, v17, v14, v15);
4666 sha512_dround( 7, v2, v3, v1, v4, v0, v27, v31, v19, v12, v18, v15, v16);
4667 sha512_dround( 8, v4, v2, v0, v1, v3, v28, v24, v12, v13, v19, v16, v17);
4668 sha512_dround( 9, v1, v4, v3, v0, v2, v29, v25, v13, v14, v12, v17, v18);
4669 sha512_dround(10, v0, v1, v2, v3, v4, v30, v26, v14, v15, v13, v18, v19);
4670 sha512_dround(11, v3, v0, v4, v2, v1, v31, v27, v15, v16, v14, v19, v12);
4671 sha512_dround(12, v2, v3, v1, v4, v0, v24, v28, v16, v17, v15, v12, v13);
4672 sha512_dround(13, v4, v2, v0, v1, v3, v25, v29, v17, v18, v16, v13, v14);
4673 sha512_dround(14, v1, v4, v3, v0, v2, v26, v30, v18, v19, v17, v14, v15);
4674 sha512_dround(15, v0, v1, v2, v3, v4, v27, v31, v19, v12, v18, v15, v16);
4675 sha512_dround(16, v3, v0, v4, v2, v1, v28, v24, v12, v13, v19, v16, v17);
4676 sha512_dround(17, v2, v3, v1, v4, v0, v29, v25, v13, v14, v12, v17, v18);
4677 sha512_dround(18, v4, v2, v0, v1, v3, v30, v26, v14, v15, v13, v18, v19);
4678 sha512_dround(19, v1, v4, v3, v0, v2, v31, v27, v15, v16, v14, v19, v12);
4679 sha512_dround(20, v0, v1, v2, v3, v4, v24, v28, v16, v17, v15, v12, v13);
4680 sha512_dround(21, v3, v0, v4, v2, v1, v25, v29, v17, v18, v16, v13, v14);
4681 sha512_dround(22, v2, v3, v1, v4, v0, v26, v30, v18, v19, v17, v14, v15);
4682 sha512_dround(23, v4, v2, v0, v1, v3, v27, v31, v19, v12, v18, v15, v16);
4683 sha512_dround(24, v1, v4, v3, v0, v2, v28, v24, v12, v13, v19, v16, v17);
4684 sha512_dround(25, v0, v1, v2, v3, v4, v29, v25, v13, v14, v12, v17, v18);
4685 sha512_dround(26, v3, v0, v4, v2, v1, v30, v26, v14, v15, v13, v18, v19);
4686 sha512_dround(27, v2, v3, v1, v4, v0, v31, v27, v15, v16, v14, v19, v12);
4687 sha512_dround(28, v4, v2, v0, v1, v3, v24, v28, v16, v17, v15, v12, v13);
4688 sha512_dround(29, v1, v4, v3, v0, v2, v25, v29, v17, v18, v16, v13, v14);
4689 sha512_dround(30, v0, v1, v2, v3, v4, v26, v30, v18, v19, v17, v14, v15);
4690 sha512_dround(31, v3, v0, v4, v2, v1, v27, v31, v19, v12, v18, v15, v16);
4691 sha512_dround(32, v2, v3, v1, v4, v0, v28, v24, v12, v0, v0, v0, v0);
4692 sha512_dround(33, v4, v2, v0, v1, v3, v29, v25, v13, v0, v0, v0, v0);
4693 sha512_dround(34, v1, v4, v3, v0, v2, v30, v26, v14, v0, v0, v0, v0);
4694 sha512_dround(35, v0, v1, v2, v3, v4, v31, v27, v15, v0, v0, v0, v0);
4695 sha512_dround(36, v3, v0, v4, v2, v1, v24, v0, v16, v0, v0, v0, v0);
4696 sha512_dround(37, v2, v3, v1, v4, v0, v25, v0, v17, v0, v0, v0, v0);
4697 sha512_dround(38, v4, v2, v0, v1, v3, v26, v0, v18, v0, v0, v0, v0);
4698 sha512_dround(39, v1, v4, v3, v0, v2, v27, v0, v19, v0, v0, v0, v0);
4699
4700 __ addv(v8, __ T2D, v8, v0);
4701 __ addv(v9, __ T2D, v9, v1);
4702 __ addv(v10, __ T2D, v10, v2);
4703 __ addv(v11, __ T2D, v11, v3);
4704
4705 if (multi_block) {
4706 __ add(ofs, ofs, 128);
4707 __ cmp(ofs, limit);
4708 __ br(Assembler::LE, sha512_loop);
4709 __ mov(c_rarg0, ofs); // return ofs
4710 }
4711
4712 __ st1(v8, v9, v10, v11, __ T2D, state);
4713
4714 __ ldpd(v14, v15, Address(sp, 48));
4715 __ ldpd(v12, v13, Address(sp, 32));
4716 __ ldpd(v10, v11, Address(sp, 16));
4717 __ ldpd(v8, v9, __ post(sp, 64));
4718
4719 __ ret(lr);
4720
4721 // record the stub entry and end
4722 store_archive_data(stub_id, start, __ pc());
4723
4724 return start;
4725 }
4726
4727 // Execute one round of keccak of two computations in parallel.
4728 // One of the states should be loaded into the lower halves of
4729 // the vector registers v0-v24, the other should be loaded into
4730 // the upper halves of those registers. The ld1r instruction loads
4731 // the round constant into both halves of register v31.
4732 // Intermediate results c0...c5 and d0...d5 are computed
4733 // in registers v25...v30.
4734 // All vector instructions that are used operate on both register
4735 // halves in parallel.
4736 // If only a single computation is needed, one can only load the lower halves.
4737 void keccak_round(Register rscratch1) {
4738 __ eor3(v29, __ T16B, v4, v9, v14); // c4 = a4 ^ a9 ^ a14
4739 __ eor3(v26, __ T16B, v1, v6, v11); // c1 = a1 ^ a16 ^ a11
4740 __ eor3(v28, __ T16B, v3, v8, v13); // c3 = a3 ^ a8 ^a13
4741 __ eor3(v25, __ T16B, v0, v5, v10); // c0 = a0 ^ a5 ^ a10
4742 __ eor3(v27, __ T16B, v2, v7, v12); // c2 = a2 ^ a7 ^ a12
4743 __ eor3(v29, __ T16B, v29, v19, v24); // c4 ^= a19 ^ a24
4744 __ eor3(v26, __ T16B, v26, v16, v21); // c1 ^= a16 ^ a21
4745 __ eor3(v28, __ T16B, v28, v18, v23); // c3 ^= a18 ^ a23
4746 __ eor3(v25, __ T16B, v25, v15, v20); // c0 ^= a15 ^ a20
4747 __ eor3(v27, __ T16B, v27, v17, v22); // c2 ^= a17 ^ a22
4748
4749 __ rax1(v30, __ T2D, v29, v26); // d0 = c4 ^ rol(c1, 1)
4750 __ rax1(v26, __ T2D, v26, v28); // d2 = c1 ^ rol(c3, 1)
4751 __ rax1(v28, __ T2D, v28, v25); // d4 = c3 ^ rol(c0, 1)
4752 __ rax1(v25, __ T2D, v25, v27); // d1 = c0 ^ rol(c2, 1)
4753 __ rax1(v27, __ T2D, v27, v29); // d3 = c2 ^ rol(c4, 1)
4754
4755 __ eor(v0, __ T16B, v0, v30); // a0 = a0 ^ d0
4756 __ xar(v29, __ T2D, v1, v25, (64 - 1)); // a10' = rol((a1^d1), 1)
4757 __ xar(v1, __ T2D, v6, v25, (64 - 44)); // a1 = rol(a6^d1), 44)
4758 __ xar(v6, __ T2D, v9, v28, (64 - 20)); // a6 = rol((a9^d4), 20)
4759 __ xar(v9, __ T2D, v22, v26, (64 - 61)); // a9 = rol((a22^d2), 61)
4760 __ xar(v22, __ T2D, v14, v28, (64 - 39)); // a22 = rol((a14^d4), 39)
4761 __ xar(v14, __ T2D, v20, v30, (64 - 18)); // a14 = rol((a20^d0), 18)
4762 __ xar(v31, __ T2D, v2, v26, (64 - 62)); // a20' = rol((a2^d2), 62)
4763 __ xar(v2, __ T2D, v12, v26, (64 - 43)); // a2 = rol((a12^d2), 43)
4764 __ xar(v12, __ T2D, v13, v27, (64 - 25)); // a12 = rol((a13^d3), 25)
4765 __ xar(v13, __ T2D, v19, v28, (64 - 8)); // a13 = rol((a19^d4), 8)
4766 __ xar(v19, __ T2D, v23, v27, (64 - 56)); // a19 = rol((a23^d3), 56)
4767 __ xar(v23, __ T2D, v15, v30, (64 - 41)); // a23 = rol((a15^d0), 41)
4768 __ xar(v15, __ T2D, v4, v28, (64 - 27)); // a15 = rol((a4^d4), 27)
4769 __ xar(v28, __ T2D, v24, v28, (64 - 14)); // a4' = rol((a24^d4), 14)
4770 __ xar(v24, __ T2D, v21, v25, (64 - 2)); // a24 = rol((a21^d1), 2)
4771 __ xar(v8, __ T2D, v8, v27, (64 - 55)); // a21' = rol((a8^d3), 55)
4772 __ xar(v4, __ T2D, v16, v25, (64 - 45)); // a8' = rol((a16^d1), 45)
4773 __ xar(v16, __ T2D, v5, v30, (64 - 36)); // a16 = rol((a5^d0), 36)
4774 __ xar(v5, __ T2D, v3, v27, (64 - 28)); // a5 = rol((a3^d3), 28)
4775 __ xar(v27, __ T2D, v18, v27, (64 - 21)); // a3' = rol((a18^d3), 21)
4776 __ xar(v3, __ T2D, v17, v26, (64 - 15)); // a18' = rol((a17^d2), 15)
4777 __ xar(v25, __ T2D, v11, v25, (64 - 10)); // a17' = rol((a11^d1), 10)
4778 __ xar(v26, __ T2D, v7, v26, (64 - 6)); // a11' = rol((a7^d2), 6)
4779 __ xar(v30, __ T2D, v10, v30, (64 - 3)); // a7' = rol((a10^d0), 3)
4780
4781 __ bcax(v20, __ T16B, v31, v22, v8); // a20 = a20' ^ (~a21 & a22')
4782 __ bcax(v21, __ T16B, v8, v23, v22); // a21 = a21' ^ (~a22 & a23)
4783 __ bcax(v22, __ T16B, v22, v24, v23); // a22 = a22 ^ (~a23 & a24)
4784 __ bcax(v23, __ T16B, v23, v31, v24); // a23 = a23 ^ (~a24 & a20')
4785 __ bcax(v24, __ T16B, v24, v8, v31); // a24 = a24 ^ (~a20' & a21')
4786
4787 __ ld1r(v31, __ T2D, __ post(rscratch1, 8)); // rc = round_constants[i]
4788
4789 __ bcax(v17, __ T16B, v25, v19, v3); // a17 = a17' ^ (~a18' & a19)
4790 __ bcax(v18, __ T16B, v3, v15, v19); // a18 = a18' ^ (~a19 & a15')
4791 __ bcax(v19, __ T16B, v19, v16, v15); // a19 = a19 ^ (~a15 & a16)
4792 __ bcax(v15, __ T16B, v15, v25, v16); // a15 = a15 ^ (~a16 & a17')
4793 __ bcax(v16, __ T16B, v16, v3, v25); // a16 = a16 ^ (~a17' & a18')
4794
4795 __ bcax(v10, __ T16B, v29, v12, v26); // a10 = a10' ^ (~a11' & a12)
4796 __ bcax(v11, __ T16B, v26, v13, v12); // a11 = a11' ^ (~a12 & a13)
4797 __ bcax(v12, __ T16B, v12, v14, v13); // a12 = a12 ^ (~a13 & a14)
4798 __ bcax(v13, __ T16B, v13, v29, v14); // a13 = a13 ^ (~a14 & a10')
4799 __ bcax(v14, __ T16B, v14, v26, v29); // a14 = a14 ^ (~a10' & a11')
4800
4801 __ bcax(v7, __ T16B, v30, v9, v4); // a7 = a7' ^ (~a8' & a9)
4802 __ bcax(v8, __ T16B, v4, v5, v9); // a8 = a8' ^ (~a9 & a5)
4803 __ bcax(v9, __ T16B, v9, v6, v5); // a9 = a9 ^ (~a5 & a6)
4804 __ bcax(v5, __ T16B, v5, v30, v6); // a5 = a5 ^ (~a6 & a7)
4805 __ bcax(v6, __ T16B, v6, v4, v30); // a6 = a6 ^ (~a7 & a8')
4806
4807 __ bcax(v3, __ T16B, v27, v0, v28); // a3 = a3' ^ (~a4' & a0)
4808 __ bcax(v4, __ T16B, v28, v1, v0); // a4 = a4' ^ (~a0 & a1)
4809 __ bcax(v0, __ T16B, v0, v2, v1); // a0 = a0 ^ (~a1 & a2)
4810 __ bcax(v1, __ T16B, v1, v27, v2); // a1 = a1 ^ (~a2 & a3)
4811 __ bcax(v2, __ T16B, v2, v28, v27); // a2 = a2 ^ (~a3 & a4')
4812
4813 __ eor(v0, __ T16B, v0, v31); // a0 = a0 ^ rc
4814 }
4815
4816 // Arguments:
4817 //
4818 // Inputs:
4819 // c_rarg0 - byte[] source+offset
4820 // c_rarg1 - byte[] SHA.state
4821 // c_rarg2 - int block_size
4822 // c_rarg3 - int offset
4823 // c_rarg4 - int limit
4824 //
4825 address generate_sha3_implCompress(StubId stub_id) {
4826 bool multi_block;
4827 switch (stub_id) {
4828 case StubId::stubgen_sha3_implCompress_id:
4829 multi_block = false;
4830 break;
4831 case StubId::stubgen_sha3_implCompressMB_id:
4832 multi_block = true;
4833 break;
4834 default:
4835 ShouldNotReachHere();
4836 }
4837 int entry_count = StubInfo::entry_count(stub_id);
4838 assert(entry_count == 1, "sanity check");
4839 address start = load_archive_data(stub_id);
4840 if (start != nullptr) {
4841 return start;
4842 }
4843 __ align(CodeEntryAlignment);
4844 StubCodeMark mark(this, stub_id);
4845 start = __ pc();
4846
4847 Register buf = c_rarg0;
4848 Register state = c_rarg1;
4849 Register block_size = c_rarg2;
4850 Register ofs = c_rarg3;
4851 Register limit = c_rarg4;
4852
4853 Label sha3_loop, rounds24_loop;
4854 Label sha3_512_or_sha3_384, shake128;
4855
4856 __ stpd(v8, v9, __ pre(sp, -64));
4857 __ stpd(v10, v11, Address(sp, 16));
4858 __ stpd(v12, v13, Address(sp, 32));
4859 __ stpd(v14, v15, Address(sp, 48));
4860
4861 // load state
4862 __ add(rscratch1, state, 32);
4863 __ ld1(v0, v1, v2, v3, __ T1D, state);
4864 __ ld1(v4, v5, v6, v7, __ T1D, __ post(rscratch1, 32));
4865 __ ld1(v8, v9, v10, v11, __ T1D, __ post(rscratch1, 32));
4866 __ ld1(v12, v13, v14, v15, __ T1D, __ post(rscratch1, 32));
4867 __ ld1(v16, v17, v18, v19, __ T1D, __ post(rscratch1, 32));
4868 __ ld1(v20, v21, v22, v23, __ T1D, __ post(rscratch1, 32));
4869 __ ld1(v24, __ T1D, rscratch1);
4870
4871 __ BIND(sha3_loop);
4872
4873 // 24 keccak rounds
4874 __ movw(rscratch2, 24);
4875
4876 // load round_constants base
4877 __ lea(rscratch1, ExternalAddress((address) _sha3_round_consts));
4878
4879 // load input
4880 __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32));
4881 __ ld1(v29, v30, v31, __ T8B, __ post(buf, 24));
4882 __ eor(v0, __ T8B, v0, v25);
4883 __ eor(v1, __ T8B, v1, v26);
4884 __ eor(v2, __ T8B, v2, v27);
4885 __ eor(v3, __ T8B, v3, v28);
4886 __ eor(v4, __ T8B, v4, v29);
4887 __ eor(v5, __ T8B, v5, v30);
4888 __ eor(v6, __ T8B, v6, v31);
4889
4890 // block_size == 72, SHA3-512; block_size == 104, SHA3-384
4891 __ tbz(block_size, 7, sha3_512_or_sha3_384);
4892
4893 __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32));
4894 __ ld1(v29, v30, v31, __ T8B, __ post(buf, 24));
4895 __ eor(v7, __ T8B, v7, v25);
4896 __ eor(v8, __ T8B, v8, v26);
4897 __ eor(v9, __ T8B, v9, v27);
4898 __ eor(v10, __ T8B, v10, v28);
4899 __ eor(v11, __ T8B, v11, v29);
4900 __ eor(v12, __ T8B, v12, v30);
4901 __ eor(v13, __ T8B, v13, v31);
4902
4903 __ ld1(v25, v26, v27, __ T8B, __ post(buf, 24));
4904 __ eor(v14, __ T8B, v14, v25);
4905 __ eor(v15, __ T8B, v15, v26);
4906 __ eor(v16, __ T8B, v16, v27);
4907
4908 // block_size == 136, bit4 == 0 and bit5 == 0, SHA3-256 or SHAKE256
4909 __ andw(c_rarg5, block_size, 48);
4910 __ cbzw(c_rarg5, rounds24_loop);
4911
4912 __ tbnz(block_size, 5, shake128);
4913 // block_size == 144, bit5 == 0, SHA3-224
4914 __ ldrd(v28, __ post(buf, 8));
4915 __ eor(v17, __ T8B, v17, v28);
4916 __ b(rounds24_loop);
4917
4918 __ BIND(shake128);
4919 __ ld1(v28, v29, v30, v31, __ T8B, __ post(buf, 32));
4920 __ eor(v17, __ T8B, v17, v28);
4921 __ eor(v18, __ T8B, v18, v29);
4922 __ eor(v19, __ T8B, v19, v30);
4923 __ eor(v20, __ T8B, v20, v31);
4924 __ b(rounds24_loop); // block_size == 168, SHAKE128
4925
4926 __ BIND(sha3_512_or_sha3_384);
4927 __ ld1(v25, v26, __ T8B, __ post(buf, 16));
4928 __ eor(v7, __ T8B, v7, v25);
4929 __ eor(v8, __ T8B, v8, v26);
4930 __ tbz(block_size, 5, rounds24_loop); // SHA3-512
4931
4932 // SHA3-384
4933 __ ld1(v27, v28, v29, v30, __ T8B, __ post(buf, 32));
4934 __ eor(v9, __ T8B, v9, v27);
4935 __ eor(v10, __ T8B, v10, v28);
4936 __ eor(v11, __ T8B, v11, v29);
4937 __ eor(v12, __ T8B, v12, v30);
4938
4939 __ BIND(rounds24_loop);
4940 __ subw(rscratch2, rscratch2, 1);
4941
4942 keccak_round(rscratch1);
4943
4944 __ cbnzw(rscratch2, rounds24_loop);
4945
4946 if (multi_block) {
4947 __ add(ofs, ofs, block_size);
4948 __ cmp(ofs, limit);
4949 __ br(Assembler::LE, sha3_loop);
4950 __ mov(c_rarg0, ofs); // return ofs
4951 }
4952
4953 __ st1(v0, v1, v2, v3, __ T1D, __ post(state, 32));
4954 __ st1(v4, v5, v6, v7, __ T1D, __ post(state, 32));
4955 __ st1(v8, v9, v10, v11, __ T1D, __ post(state, 32));
4956 __ st1(v12, v13, v14, v15, __ T1D, __ post(state, 32));
4957 __ st1(v16, v17, v18, v19, __ T1D, __ post(state, 32));
4958 __ st1(v20, v21, v22, v23, __ T1D, __ post(state, 32));
4959 __ st1(v24, __ T1D, state);
4960
4961 // restore callee-saved registers
4962 __ ldpd(v14, v15, Address(sp, 48));
4963 __ ldpd(v12, v13, Address(sp, 32));
4964 __ ldpd(v10, v11, Address(sp, 16));
4965 __ ldpd(v8, v9, __ post(sp, 64));
4966
4967 __ ret(lr);
4968
4969 // record the stub entry and end
4970 store_archive_data(stub_id, start, __ pc());
4971
4972 return start;
4973 }
4974
4975 // Inputs:
4976 // c_rarg0 - long[] state0
4977 // c_rarg1 - long[] state1
4978 address generate_double_keccak() {
4979 StubId stub_id = StubId::stubgen_double_keccak_id;
4980 int entry_count = StubInfo::entry_count(stub_id);
4981 assert(entry_count == 1, "sanity check");
4982 address start = load_archive_data(stub_id);
4983 if (start != nullptr) {
4984 return start;
4985 }
4986 // Implements the double_keccak() method of the
4987 // sun.security.provider.SHA3Parallel class
4988 __ align(CodeEntryAlignment);
4989 StubCodeMark mark(this, stub_id);
4990 start = __ pc();
4991 __ enter();
4992
4993 Register state0 = c_rarg0;
4994 Register state1 = c_rarg1;
4995
4996 Label rounds24_loop;
4997
4998 // save callee-saved registers
4999 __ stpd(v8, v9, __ pre(sp, -64));
5000 __ stpd(v10, v11, Address(sp, 16));
5001 __ stpd(v12, v13, Address(sp, 32));
5002 __ stpd(v14, v15, Address(sp, 48));
5003
5004 // load states
5005 __ add(rscratch1, state0, 32);
5006 __ ld4(v0, v1, v2, v3, __ D, 0, state0);
5007 __ ld4(v4, v5, v6, v7, __ D, 0, __ post(rscratch1, 32));
5008 __ ld4(v8, v9, v10, v11, __ D, 0, __ post(rscratch1, 32));
5009 __ ld4(v12, v13, v14, v15, __ D, 0, __ post(rscratch1, 32));
5010 __ ld4(v16, v17, v18, v19, __ D, 0, __ post(rscratch1, 32));
5011 __ ld4(v20, v21, v22, v23, __ D, 0, __ post(rscratch1, 32));
5012 __ ld1(v24, __ D, 0, rscratch1);
5013 __ add(rscratch1, state1, 32);
5014 __ ld4(v0, v1, v2, v3, __ D, 1, state1);
5015 __ ld4(v4, v5, v6, v7, __ D, 1, __ post(rscratch1, 32));
5016 __ ld4(v8, v9, v10, v11, __ D, 1, __ post(rscratch1, 32));
5017 __ ld4(v12, v13, v14, v15, __ D, 1, __ post(rscratch1, 32));
5018 __ ld4(v16, v17, v18, v19, __ D, 1, __ post(rscratch1, 32));
5019 __ ld4(v20, v21, v22, v23, __ D, 1, __ post(rscratch1, 32));
5020 __ ld1(v24, __ D, 1, rscratch1);
5021
5022 // 24 keccak rounds
5023 __ movw(rscratch2, 24);
5024
5025 // load round_constants base
5026 __ lea(rscratch1, ExternalAddress((address) _double_keccak_round_consts));
5027
5028 __ BIND(rounds24_loop);
5029 __ subw(rscratch2, rscratch2, 1);
5030 keccak_round(rscratch1);
5031 __ cbnzw(rscratch2, rounds24_loop);
5032
5033 __ st4(v0, v1, v2, v3, __ D, 0, __ post(state0, 32));
5034 __ st4(v4, v5, v6, v7, __ D, 0, __ post(state0, 32));
5035 __ st4(v8, v9, v10, v11, __ D, 0, __ post(state0, 32));
5036 __ st4(v12, v13, v14, v15, __ D, 0, __ post(state0, 32));
5037 __ st4(v16, v17, v18, v19, __ D, 0, __ post(state0, 32));
5038 __ st4(v20, v21, v22, v23, __ D, 0, __ post(state0, 32));
5039 __ st1(v24, __ D, 0, state0);
5040 __ st4(v0, v1, v2, v3, __ D, 1, __ post(state1, 32));
5041 __ st4(v4, v5, v6, v7, __ D, 1, __ post(state1, 32));
5042 __ st4(v8, v9, v10, v11, __ D, 1, __ post(state1, 32));
5043 __ st4(v12, v13, v14, v15, __ D, 1, __ post(state1, 32));
5044 __ st4(v16, v17, v18, v19, __ D, 1, __ post(state1, 32));
5045 __ st4(v20, v21, v22, v23, __ D, 1, __ post(state1, 32));
5046 __ st1(v24, __ D, 1, state1);
5047
5048 // restore callee-saved vector registers
5049 __ ldpd(v14, v15, Address(sp, 48));
5050 __ ldpd(v12, v13, Address(sp, 32));
5051 __ ldpd(v10, v11, Address(sp, 16));
5052 __ ldpd(v8, v9, __ post(sp, 64));
5053
5054 __ leave(); // required for proper stackwalking of RuntimeStub frame
5055
5056 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
5057 __ ret(lr);
5058
5059 // record the stub entry and end
5060 store_archive_data(stub_id, start, __ pc());
5061
5062 return start;
5063 }
5064
5065 // ChaCha20 block function. This version parallelizes the 32-bit
5066 // state elements on each of 16 vectors, producing 4 blocks of
5067 // keystream at a time.
5068 //
5069 // state (int[16]) = c_rarg0
5070 // keystream (byte[256]) = c_rarg1
5071 // return - number of bytes of produced keystream (always 256)
5072 //
5073 // This implementation takes each 32-bit integer from the state
5074 // array and broadcasts it across all 4 32-bit lanes of a vector register
5075 // (e.g. state[0] is replicated on all 4 lanes of v4, state[1] to all 4 lanes
5076 // of v5, etc.). Once all 16 elements have been broadcast onto 16 vectors,
5077 // the quarter round schedule is implemented as outlined in RFC 7539 section
5078 // 2.3. However, instead of sequentially processing the 3 quarter round
5079 // operations represented by one QUARTERROUND function, we instead stack all
5080 // the adds, xors and left-rotations from the first 4 quarter rounds together
5081 // and then do the same for the second set of 4 quarter rounds. This removes
5082 // some latency that would otherwise be incurred by waiting for an add to
5083 // complete before performing an xor (which depends on the result of the
5084 // add), etc. An adjustment happens between the first and second groups of 4
5085 // quarter rounds, but this is done only in the inputs to the macro functions
5086 // that generate the assembly instructions - these adjustments themselves are
5087 // not part of the resulting assembly.
5088 // The 4 registers v0-v3 are used during the quarter round operations as
5089 // scratch registers. Once the 20 rounds are complete, these 4 scratch
5090 // registers become the vectors involved in adding the start state back onto
5091 // the post-QR working state. After the adds are complete, each of the 16
5092 // vectors write their first lane back to the keystream buffer, followed
5093 // by the second lane from all vectors and so on.
5094 address generate_chacha20Block_blockpar() {
5095 StubId stub_id = StubId::stubgen_chacha20Block_id;
5096 int entry_count = StubInfo::entry_count(stub_id);
5097 assert(entry_count == 1, "sanity check");
5098 address start = load_archive_data(stub_id);
5099 if (start != nullptr) {
5100 return start;
5101 }
5102 Label L_twoRounds, L_cc20_const;
5103 __ align(CodeEntryAlignment);
5104 StubCodeMark mark(this, stub_id);
5105 start = __ pc();
5106 __ enter();
5107
5108 int i, j;
5109 const Register state = c_rarg0;
5110 const Register keystream = c_rarg1;
5111 const Register loopCtr = r10;
5112 const Register tmpAddr = r11;
5113 const FloatRegister ctrAddOverlay = v28;
5114 const FloatRegister lrot8Tbl = v29;
5115
5116 // Organize SIMD registers in an array that facilitates
5117 // putting repetitive opcodes into loop structures. It is
5118 // important that each grouping of 4 registers is monotonically
5119 // increasing to support the requirements of multi-register
5120 // instructions (e.g. ld4r, st4, etc.)
5121 const FloatRegister workSt[16] = {
5122 v4, v5, v6, v7, v16, v17, v18, v19,
5123 v20, v21, v22, v23, v24, v25, v26, v27
5124 };
5125
5126 // Pull in constant data. The first 16 bytes are the add overlay
5127 // which is applied to the vector holding the counter (state[12]).
5128 // The second 16 bytes is the index register for the 8-bit left
5129 // rotation tbl instruction.
5130 __ adr(tmpAddr, L_cc20_const);
5131 __ ldpq(ctrAddOverlay, lrot8Tbl, Address(tmpAddr));
5132
5133 // Load from memory and interlace across 16 SIMD registers,
5134 // With each word from memory being broadcast to all lanes of
5135 // each successive SIMD register.
5136 // Addr(0) -> All lanes in workSt[i]
5137 // Addr(4) -> All lanes workSt[i + 1], etc.
5138 __ mov(tmpAddr, state);
5139 for (i = 0; i < 16; i += 4) {
5140 __ ld4r(workSt[i], workSt[i + 1], workSt[i + 2], workSt[i + 3], __ T4S,
5141 __ post(tmpAddr, 16));
5142 }
5143 __ addv(workSt[12], __ T4S, workSt[12], ctrAddOverlay); // Add ctr overlay
5144
5145 // Before entering the loop, create 5 4-register arrays. These
5146 // will hold the 4 registers that represent the a/b/c/d fields
5147 // in the quarter round operation. For instance the "b" field
5148 // for the first 4 quarter round operations is the set of v16/v17/v18/v19,
5149 // but in the second 4 quarter rounds it gets adjusted to v17/v18/v19/v16
5150 // since it is part of a diagonal organization. The aSet and scratch
5151 // register sets are defined at declaration time because they do not change
5152 // organization at any point during the 20-round processing.
5153 FloatRegister aSet[4] = { v4, v5, v6, v7 };
5154 FloatRegister bSet[4];
5155 FloatRegister cSet[4];
5156 FloatRegister dSet[4];
5157 FloatRegister scratch[4] = { v0, v1, v2, v3 };
5158
5159 // Set up the 10 iteration loop and perform all 8 quarter round ops
5160 __ mov(loopCtr, 10);
5161 __ BIND(L_twoRounds);
5162
5163 // Set to columnar organization and do the following 4 quarter-rounds:
5164 // QUARTERROUND(0, 4, 8, 12)
5165 // QUARTERROUND(1, 5, 9, 13)
5166 // QUARTERROUND(2, 6, 10, 14)
5167 // QUARTERROUND(3, 7, 11, 15)
5168 __ cc20_set_qr_registers(bSet, workSt, 4, 5, 6, 7);
5169 __ cc20_set_qr_registers(cSet, workSt, 8, 9, 10, 11);
5170 __ cc20_set_qr_registers(dSet, workSt, 12, 13, 14, 15);
5171
5172 __ cc20_qr_add4(aSet, bSet); // a += b
5173 __ cc20_qr_xor4(dSet, aSet, dSet); // d ^= a
5174 __ cc20_qr_lrot4(dSet, dSet, 16, lrot8Tbl); // d <<<= 16
5175
5176 __ cc20_qr_add4(cSet, dSet); // c += d
5177 __ cc20_qr_xor4(bSet, cSet, scratch); // b ^= c (scratch)
5178 __ cc20_qr_lrot4(scratch, bSet, 12, lrot8Tbl); // b <<<= 12
5179
5180 __ cc20_qr_add4(aSet, bSet); // a += b
5181 __ cc20_qr_xor4(dSet, aSet, dSet); // d ^= a
5182 __ cc20_qr_lrot4(dSet, dSet, 8, lrot8Tbl); // d <<<= 8
5183
5184 __ cc20_qr_add4(cSet, dSet); // c += d
5185 __ cc20_qr_xor4(bSet, cSet, scratch); // b ^= c (scratch)
5186 __ cc20_qr_lrot4(scratch, bSet, 7, lrot8Tbl); // b <<<= 12
5187
5188 // Set to diagonal organization and do the next 4 quarter-rounds:
5189 // QUARTERROUND(0, 5, 10, 15)
5190 // QUARTERROUND(1, 6, 11, 12)
5191 // QUARTERROUND(2, 7, 8, 13)
5192 // QUARTERROUND(3, 4, 9, 14)
5193 __ cc20_set_qr_registers(bSet, workSt, 5, 6, 7, 4);
5194 __ cc20_set_qr_registers(cSet, workSt, 10, 11, 8, 9);
5195 __ cc20_set_qr_registers(dSet, workSt, 15, 12, 13, 14);
5196
5197 __ cc20_qr_add4(aSet, bSet); // a += b
5198 __ cc20_qr_xor4(dSet, aSet, dSet); // d ^= a
5199 __ cc20_qr_lrot4(dSet, dSet, 16, lrot8Tbl); // d <<<= 16
5200
5201 __ cc20_qr_add4(cSet, dSet); // c += d
5202 __ cc20_qr_xor4(bSet, cSet, scratch); // b ^= c (scratch)
5203 __ cc20_qr_lrot4(scratch, bSet, 12, lrot8Tbl); // b <<<= 12
5204
5205 __ cc20_qr_add4(aSet, bSet); // a += b
5206 __ cc20_qr_xor4(dSet, aSet, dSet); // d ^= a
5207 __ cc20_qr_lrot4(dSet, dSet, 8, lrot8Tbl); // d <<<= 8
5208
5209 __ cc20_qr_add4(cSet, dSet); // c += d
5210 __ cc20_qr_xor4(bSet, cSet, scratch); // b ^= c (scratch)
5211 __ cc20_qr_lrot4(scratch, bSet, 7, lrot8Tbl); // b <<<= 12
5212
5213 // Decrement and iterate
5214 __ sub(loopCtr, loopCtr, 1);
5215 __ cbnz(loopCtr, L_twoRounds);
5216
5217 __ mov(tmpAddr, state);
5218
5219 // Add the starting state back to the post-loop keystream
5220 // state. We read/interlace the state array from memory into
5221 // 4 registers similar to what we did in the beginning. Then
5222 // add the counter overlay onto workSt[12] at the end.
5223 for (i = 0; i < 16; i += 4) {
5224 __ ld4r(v0, v1, v2, v3, __ T4S, __ post(tmpAddr, 16));
5225 __ addv(workSt[i], __ T4S, workSt[i], v0);
5226 __ addv(workSt[i + 1], __ T4S, workSt[i + 1], v1);
5227 __ addv(workSt[i + 2], __ T4S, workSt[i + 2], v2);
5228 __ addv(workSt[i + 3], __ T4S, workSt[i + 3], v3);
5229 }
5230 __ addv(workSt[12], __ T4S, workSt[12], ctrAddOverlay); // Add ctr overlay
5231
5232 // Write working state into the keystream buffer. This is accomplished
5233 // by taking the lane "i" from each of the four vectors and writing
5234 // it to consecutive 4-byte offsets, then post-incrementing by 16 and
5235 // repeating with the next 4 vectors until all 16 vectors have been used.
5236 // Then move to the next lane and repeat the process until all lanes have
5237 // been written.
5238 for (i = 0; i < 4; i++) {
5239 for (j = 0; j < 16; j += 4) {
5240 __ st4(workSt[j], workSt[j + 1], workSt[j + 2], workSt[j + 3], __ S, i,
5241 __ post(keystream, 16));
5242 }
5243 }
5244
5245 __ mov(r0, 256); // Return length of output keystream
5246 __ leave();
5247 __ ret(lr);
5248
5249 // bind label and generate local constant data used by this stub
5250 // The constant data is broken into two 128-bit segments to be loaded
5251 // onto FloatRegisters. The first 128 bits are a counter add overlay
5252 // that adds +0/+1/+2/+3 to the vector holding replicated state[12].
5253 // The second 128-bits is a table constant used for 8-bit left rotations.
5254 __ BIND(L_cc20_const);
5255 __ emit_int64(0x0000000100000000UL);
5256 __ emit_int64(0x0000000300000002UL);
5257 __ emit_int64(0x0605040702010003UL);
5258 __ emit_int64(0x0E0D0C0F0A09080BUL);
5259
5260 // record the stub entry and end
5261 store_archive_data(stub_id, start, __ pc());
5262
5263 return start;
5264 }
5265
5266 // Helpers to schedule parallel operation bundles across vector
5267 // register sequences of size 2, 4 or 8.
5268
5269 // Implement various primitive computations across vector sequences
5270
5271 template<int N>
5272 void vs_addv(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5273 const VSeq<N>& v1, const VSeq<N>& v2) {
5274 // output must not be constant
5275 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5276 // output cannot overwrite pending inputs
5277 assert(!vs_write_before_read(v, v1), "output overwrites input");
5278 assert(!vs_write_before_read(v, v2), "output overwrites input");
5279 for (int i = 0; i < N; i++) {
5280 __ addv(v[i], T, v1[i], v2[i]);
5281 }
5282 }
5283
5284 template<int N>
5285 void vs_subv(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5286 const VSeq<N>& v1, const VSeq<N>& v2) {
5287 // output must not be constant
5288 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5289 // output cannot overwrite pending inputs
5290 assert(!vs_write_before_read(v, v1), "output overwrites input");
5291 assert(!vs_write_before_read(v, v2), "output overwrites input");
5292 for (int i = 0; i < N; i++) {
5293 __ subv(v[i], T, v1[i], v2[i]);
5294 }
5295 }
5296
5297 template<int N>
5298 void vs_mulv(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5299 const VSeq<N>& v1, const VSeq<N>& v2) {
5300 // output must not be constant
5301 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5302 // output cannot overwrite pending inputs
5303 assert(!vs_write_before_read(v, v1), "output overwrites input");
5304 assert(!vs_write_before_read(v, v2), "output overwrites input");
5305 for (int i = 0; i < N; i++) {
5306 __ mulv(v[i], T, v1[i], v2[i]);
5307 }
5308 }
5309
5310 template<int N>
5311 void vs_negr(const VSeq<N>& v, Assembler::SIMD_Arrangement T, const VSeq<N>& v1) {
5312 // output must not be constant
5313 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5314 // output cannot overwrite pending inputs
5315 assert(!vs_write_before_read(v, v1), "output overwrites input");
5316 for (int i = 0; i < N; i++) {
5317 __ negr(v[i], T, v1[i]);
5318 }
5319 }
5320
5321 template<int N>
5322 void vs_shl(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5323 const VSeq<N>& v1, int shift) {
5324 // output must not be constant
5325 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5326 // output cannot overwrite pending inputs
5327 assert(!vs_write_before_read(v, v1), "output overwrites input");
5328
5329 for (int i = 0; i < N; i++) {
5330 __ shl(v[i], T, v1[i], shift);
5331 }
5332 }
5333
5334 template<int N>
5335 void vs_ushr(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5336 const VSeq<N>& v1, int shift) {
5337 // output must not be constant
5338 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5339 // output cannot overwrite pending inputs
5340 assert(!vs_write_before_read(v, v1), "output overwrites input");
5341
5342 for (int i = 0; i < N; i++) {
5343 __ ushr(v[i], T, v1[i], shift);
5344 }
5345 }
5346
5347 template<int N>
5348 void vs_sshr(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5349 const VSeq<N>& v1, int shift) {
5350 // output must not be constant
5351 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5352 // output cannot overwrite pending inputs
5353 assert(!vs_write_before_read(v, v1), "output overwrites input");
5354 for (int i = 0; i < N; i++) {
5355 __ sshr(v[i], T, v1[i], shift);
5356 }
5357 }
5358
5359 template<int N>
5360 void vs_andr(const VSeq<N>& v, const VSeq<N>& v1, const VSeq<N>& v2) {
5361 // output must not be constant
5362 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5363 // output cannot overwrite pending inputs
5364 assert(!vs_write_before_read(v, v1), "output overwrites input");
5365 assert(!vs_write_before_read(v, v2), "output overwrites input");
5366 for (int i = 0; i < N; i++) {
5367 __ andr(v[i], __ T16B, v1[i], v2[i]);
5368 }
5369 }
5370
5371 template<int N>
5372 void vs_andr(const VSeq<N>& v, const VSeq<N>& v1, const FloatRegister v2) {
5373 // output must not be constant
5374 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5375 // output cannot overwrite pending inputs
5376 assert(!vs_write_before_read(v, v1), "output overwrites input");
5377 for (int i = 0; i < N; i++) {
5378 __ andr(v[i], __ T16B, v1[i], v2);
5379 }
5380 }
5381
5382 template<int N>
5383 void vs_eor(const VSeq<N>& v, const VSeq<N>& v1, const VSeq<N>& v2) {
5384 // output must not be constant
5385 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5386 // output cannot overwrite pending inputs
5387 assert(!vs_write_before_read(v, v1), "output overwrites input");
5388 assert(!vs_write_before_read(v, v2), "output overwrites input");
5389 for (int i = 0; i < N; i++) {
5390 __ eor(v[i], __ T16B, v1[i], v2[i]);
5391 }
5392 }
5393
5394 template<int N>
5395 void vs_orr(const VSeq<N>& v, const VSeq<N>& v1, const VSeq<N>& v2) {
5396 // output must not be constant
5397 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5398 // output cannot overwrite pending inputs
5399 assert(!vs_write_before_read(v, v1), "output overwrites input");
5400 assert(!vs_write_before_read(v, v2), "output overwrites input");
5401 for (int i = 0; i < N; i++) {
5402 __ orr(v[i], __ T16B, v1[i], v2[i]);
5403 }
5404 }
5405
5406 template<int N>
5407 void vs_notr(const VSeq<N>& v, const VSeq<N>& v1) {
5408 // output must not be constant
5409 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5410 // output cannot overwrite pending inputs
5411 assert(!vs_write_before_read(v, v1), "output overwrites input");
5412 for (int i = 0; i < N; i++) {
5413 __ notr(v[i], __ T16B, v1[i]);
5414 }
5415 }
5416
5417 template<int N>
5418 void vs_sqdmulh(const VSeq<N>& v, Assembler::SIMD_Arrangement T, const VSeq<N>& v1, const VSeq<N>& v2) {
5419 // output must not be constant
5420 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5421 // output cannot overwrite pending inputs
5422 assert(!vs_write_before_read(v, v1), "output overwrites input");
5423 assert(!vs_write_before_read(v, v2), "output overwrites input");
5424 for (int i = 0; i < N; i++) {
5425 __ sqdmulh(v[i], T, v1[i], v2[i]);
5426 }
5427 }
5428
5429 template<int N>
5430 void vs_mlsv(const VSeq<N>& v, Assembler::SIMD_Arrangement T, const VSeq<N>& v1, VSeq<N>& v2) {
5431 // output must not be constant
5432 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5433 // output cannot overwrite pending inputs
5434 assert(!vs_write_before_read(v, v1), "output overwrites input");
5435 assert(!vs_write_before_read(v, v2), "output overwrites input");
5436 for (int i = 0; i < N; i++) {
5437 __ mlsv(v[i], T, v1[i], v2[i]);
5438 }
5439 }
5440
5441 // load N/2 successive pairs of quadword values from memory in order
5442 // into N successive vector registers of the sequence via the
5443 // address supplied in base.
5444 template<int N>
5445 void vs_ldpq(const VSeq<N>& v, Register base) {
5446 static_assert(N > 0 && is_even(N), "sequence length must be even");
5447 for (int i = 0; i < N; i += 2) {
5448 __ ldpq(v[i], v[i+1], Address(base, 16 * i));
5449 }
5450 }
5451
5452 // load N/2 successive pairs of quadword values from memory in order
5453 // into N vector registers of the sequence via the address supplied
5454 // in base using post-increment addressing
5455 template<int N>
5456 void vs_ldpq_post(const VSeq<N>& v, Register base) {
5457 static_assert(N > 0 && is_even(N), "sequence length must be even");
5458 for (int i = 0; i < N; i += 2) {
5459 __ ldpq(v[i], v[i+1], __ post(base, 32));
5460 }
5461 }
5462
5463 // store N successive vector registers of the sequence into N/2
5464 // successive pairs of quadword memory locations via the address
5465 // supplied in base using post-increment addressing
5466 template<int N>
5467 void vs_stpq_post(const VSeq<N>& v, Register base) {
5468 static_assert(N > 0 && is_even(N), "sequence length must be even");
5469 for (int i = 0; i < N; i += 2) {
5470 __ stpq(v[i], v[i+1], __ post(base, 32));
5471 }
5472 }
5473
5474 // load N/2 pairs of quadword values from memory de-interleaved into
5475 // N vector registers 2 at a time via the address supplied in base
5476 // using post-increment addressing.
5477 template<int N>
5478 void vs_ld2_post(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base) {
5479 static_assert(N > 0 && is_even(N), "sequence length must be even");
5480 for (int i = 0; i < N; i += 2) {
5481 __ ld2(v[i], v[i+1], T, __ post(base, 32));
5482 }
5483 }
5484
5485 // store N vector registers interleaved into N/2 pairs of quadword
5486 // memory locations via the address supplied in base using
5487 // post-increment addressing.
5488 template<int N>
5489 void vs_st2_post(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base) {
5490 static_assert(N > 0 && is_even(N), "sequence length must be even");
5491 for (int i = 0; i < N; i += 2) {
5492 __ st2(v[i], v[i+1], T, __ post(base, 32));
5493 }
5494 }
5495
5496 // store two vector register sequences of length N
5497 // interleaved into N pairs of quadword memory locations
5498 // starting at the address supplied in dest using
5499 // post-increment addressing.
5500 template<int N>
5501 void vs_st1_interleaved(VSeq<N> A, VSeq<N> B, Register dest) {
5502 for (int i = 0; i < N; i++) {
5503 __ st1(A[i], __ T2D, __ post(dest, 16));
5504 __ st1(B[i], __ T2D, __ post(dest, 16));
5505 }
5506 }
5507
5508 // load N quadword values from memory de-interleaved into N vector
5509 // registers 3 elements at a time via the address supplied in base.
5510 template<int N>
5511 void vs_ld3(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base) {
5512 static_assert(N == ((N / 3) * 3), "sequence length must be multiple of 3");
5513 for (int i = 0; i < N; i += 3) {
5514 __ ld3(v[i], v[i+1], v[i+2], T, base);
5515 }
5516 }
5517
5518 // load N quadword values from memory de-interleaved into N vector
5519 // registers 3 elements at a time via the address supplied in base
5520 // using post-increment addressing.
5521 template<int N>
5522 void vs_ld3_post(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base) {
5523 static_assert(N == ((N / 3) * 3), "sequence length must be multiple of 3");
5524 for (int i = 0; i < N; i += 3) {
5525 __ ld3(v[i], v[i+1], v[i+2], T, __ post(base, 48));
5526 }
5527 }
5528
5529 // load N/2 pairs of quadword values from memory into N vector
5530 // registers via the address supplied in base with each pair indexed
5531 // using the start offset plus the corresponding entry in the
5532 // offsets array
5533 template<int N>
5534 void vs_ldpq_indexed(const VSeq<N>& v, Register base, int start, int (&offsets)[N/2]) {
5535 static_assert(N > 0 && is_even(N), "sequence length must be even");
5536 for (int i = 0; i < N/2; i++) {
5537 __ ldpq(v[2*i], v[2*i+1], Address(base, start + offsets[i]));
5538 }
5539 }
5540
5541 // store N vector registers into N/2 pairs of quadword memory
5542 // locations via the address supplied in base with each pair indexed
5543 // using the start offset plus the corresponding entry in the
5544 // offsets array
5545 template<int N>
5546 void vs_stpq_indexed(const VSeq<N>& v, Register base, int start, int offsets[N/2]) {
5547 for (int i = 0; i < N/2; i++) {
5548 __ stpq(v[2*i], v[2*i+1], Address(base, start + offsets[i]));
5549 }
5550 }
5551
5552 // load N single quadword values from memory into N vector registers
5553 // via the address supplied in base with each value indexed using
5554 // the start offset plus the corresponding entry in the offsets
5555 // array
5556 template<int N>
5557 void vs_ldr_indexed(const VSeq<N>& v, Assembler::SIMD_RegVariant T, Register base,
5558 int start, int (&offsets)[N]) {
5559 for (int i = 0; i < N; i++) {
5560 __ ldr(v[i], T, Address(base, start + offsets[i]));
5561 }
5562 }
5563
5564 // store N vector registers into N single quadword memory locations
5565 // via the address supplied in base with each value indexed using
5566 // the start offset plus the corresponding entry in the offsets
5567 // array
5568 template<int N>
5569 void vs_str_indexed(const VSeq<N>& v, Assembler::SIMD_RegVariant T, Register base,
5570 int start, int (&offsets)[N]) {
5571 for (int i = 0; i < N; i++) {
5572 __ str(v[i], T, Address(base, start + offsets[i]));
5573 }
5574 }
5575
5576 // load N/2 pairs of quadword values from memory de-interleaved into
5577 // N vector registers 2 at a time via the address supplied in base
5578 // with each pair indexed using the start offset plus the
5579 // corresponding entry in the offsets array
5580 template<int N>
5581 void vs_ld2_indexed(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base,
5582 Register tmp, int start, int (&offsets)[N/2]) {
5583 static_assert(N > 0 && is_even(N), "sequence length must be even");
5584 for (int i = 0; i < N/2; i++) {
5585 __ add(tmp, base, start + offsets[i]);
5586 __ ld2(v[2*i], v[2*i+1], T, tmp);
5587 }
5588 }
5589
5590 // store N vector registers 2 at a time interleaved into N/2 pairs
5591 // of quadword memory locations via the address supplied in base
5592 // with each pair indexed using the start offset plus the
5593 // corresponding entry in the offsets array
5594 template<int N>
5595 void vs_st2_indexed(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base,
5596 Register tmp, int start, int (&offsets)[N/2]) {
5597 static_assert(N > 0 && is_even(N), "sequence length must be even");
5598 for (int i = 0; i < N/2; i++) {
5599 __ add(tmp, base, start + offsets[i]);
5600 __ st2(v[2*i], v[2*i+1], T, tmp);
5601 }
5602 }
5603
5604 // Helper routines for various flavours of Montgomery multiply
5605
5606 // Perform 16 32-bit (4x4S) or 32 16-bit (4 x 8H) Montgomery
5607 // multiplications in parallel
5608 //
5609
5610 // See the montMul() method of the sun.security.provider.ML_DSA
5611 // class.
5612 //
5613 // Computes 4x4S results or 8x8H results
5614 // a = b * c * 2^MONT_R_BITS mod MONT_Q
5615 // Inputs: vb, vc - 4x4S or 4x8H vector register sequences
5616 // vq - 2x4S or 2x8H constants <MONT_Q, MONT_Q_INV_MOD_R>
5617 // Temps: vtmp - 4x4S or 4x8H vector sequence trashed after call
5618 // Outputs: va - 4x4S or 4x8H vector register sequences
5619 // vb, vc, vtmp and vq must all be disjoint
5620 // va must be disjoint from all other inputs/temps or must equal vc
5621 // va must have a non-zero delta i.e. it must not be a constant vseq.
5622 // n.b. MONT_R_BITS is 16 or 32, so the right shift by it is implicit.
5623 void vs_montmul4(const VSeq<4>& va, const VSeq<4>& vb, const VSeq<4>& vc,
5624 Assembler::SIMD_Arrangement T,
5625 const VSeq<4>& vtmp, const VSeq<2>& vq) {
5626 assert (T == __ T4S || T == __ T8H, "invalid arrangement for montmul");
5627 assert(vs_disjoint(vb, vc), "vb and vc overlap");
5628 assert(vs_disjoint(vb, vq), "vb and vq overlap");
5629 assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap");
5630
5631 assert(vs_disjoint(vc, vq), "vc and vq overlap");
5632 assert(vs_disjoint(vc, vtmp), "vc and vtmp overlap");
5633
5634 assert(vs_disjoint(vq, vtmp), "vq and vtmp overlap");
5635
5636 assert(vs_disjoint(va, vc) || vs_same(va, vc), "va and vc neither disjoint nor equal");
5637 assert(vs_disjoint(va, vb), "va and vb overlap");
5638 assert(vs_disjoint(va, vq), "va and vq overlap");
5639 assert(vs_disjoint(va, vtmp), "va and vtmp overlap");
5640 assert(!va.is_constant(), "output vector must identify 4 different registers");
5641
5642 // schedule 4 streams of instructions across the vector sequences
5643 for (int i = 0; i < 4; i++) {
5644 __ sqdmulh(vtmp[i], T, vb[i], vc[i]); // aHigh = hi32(2 * b * c)
5645 __ mulv(va[i], T, vb[i], vc[i]); // aLow = lo32(b * c)
5646 }
5647
5648 for (int i = 0; i < 4; i++) {
5649 __ mulv(va[i], T, va[i], vq[0]); // m = aLow * qinv
5650 }
5651
5652 for (int i = 0; i < 4; i++) {
5653 __ sqdmulh(va[i], T, va[i], vq[1]); // n = hi32(2 * m * q)
5654 }
5655
5656 for (int i = 0; i < 4; i++) {
5657 __ shsubv(va[i], T, vtmp[i], va[i]); // a = (aHigh - n) / 2
5658 }
5659 }
5660
5661 // Perform 8 32-bit (4x4S) or 16 16-bit (2 x 8H) Montgomery
5662 // multiplications in parallel
5663 //
5664
5665 // See the montMul() method of the sun.security.provider.ML_DSA
5666 // class.
5667 //
5668 // Computes 4x4S results or 8x8H results
5669 // a = b * c * 2^MONT_R_BITS mod MONT_Q
5670 // Inputs: vb, vc - 4x4S or 4x8H vector register sequences
5671 // vq - 2x4S or 2x8H constants <MONT_Q, MONT_Q_INV_MOD_R>
5672 // Temps: vtmp - 4x4S or 4x8H vector sequence trashed after call
5673 // Outputs: va - 4x4S or 4x8H vector register sequences
5674 // vb, vc, vtmp and vq must all be disjoint
5675 // va must be disjoint from all other inputs/temps or must equal vc
5676 // va must have a non-zero delta i.e. it must not be a constant vseq.
5677 // n.b. MONT_R_BITS is 16 or 32, so the right shift by it is implicit.
5678 void vs_montmul2(const VSeq<2>& va, const VSeq<2>& vb, const VSeq<2>& vc,
5679 Assembler::SIMD_Arrangement T,
5680 const VSeq<2>& vtmp, const VSeq<2>& vq) {
5681 assert (T == __ T4S || T == __ T8H, "invalid arrangement for montmul");
5682 assert(vs_disjoint(vb, vc), "vb and vc overlap");
5683 assert(vs_disjoint(vb, vq), "vb and vq overlap");
5684 assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap");
5685
5686 assert(vs_disjoint(vc, vq), "vc and vq overlap");
5687 assert(vs_disjoint(vc, vtmp), "vc and vtmp overlap");
5688
5689 assert(vs_disjoint(vq, vtmp), "vq and vtmp overlap");
5690
5691 assert(vs_disjoint(va, vc) || vs_same(va, vc), "va and vc neither disjoint nor equal");
5692 assert(vs_disjoint(va, vb), "va and vb overlap");
5693 assert(vs_disjoint(va, vq), "va and vq overlap");
5694 assert(vs_disjoint(va, vtmp), "va and vtmp overlap");
5695 assert(!va.is_constant(), "output vector must identify 2 different registers");
5696
5697 // schedule 2 streams of instructions across the vector sequences
5698 for (int i = 0; i < 2; i++) {
5699 __ sqdmulh(vtmp[i], T, vb[i], vc[i]); // aHigh = hi32(2 * b * c)
5700 __ mulv(va[i], T, vb[i], vc[i]); // aLow = lo32(b * c)
5701 }
5702
5703 for (int i = 0; i < 2; i++) {
5704 __ mulv(va[i], T, va[i], vq[0]); // m = aLow * qinv
5705 }
5706
5707 for (int i = 0; i < 2; i++) {
5708 __ sqdmulh(va[i], T, va[i], vq[1]); // n = hi32(2 * m * q)
5709 }
5710
5711 for (int i = 0; i < 2; i++) {
5712 __ shsubv(va[i], T, vtmp[i], va[i]); // a = (aHigh - n) / 2
5713 }
5714 }
5715
5716 // Perform 16 16-bit Montgomery multiplications in parallel.
5717 void kyber_montmul16(const VSeq<2>& va, const VSeq<2>& vb, const VSeq<2>& vc,
5718 const VSeq<2>& vtmp, const VSeq<2>& vq) {
5719 // Use the helper routine to schedule a 2x8H Montgomery multiply.
5720 // It will assert that the register use is valid
5721 vs_montmul2(va, vb, vc, __ T8H, vtmp, vq);
5722 }
5723
5724 // Perform 32 16-bit Montgomery multiplications in parallel.
5725 void kyber_montmul32(const VSeq<4>& va, const VSeq<4>& vb, const VSeq<4>& vc,
5726 const VSeq<4>& vtmp, const VSeq<2>& vq) {
5727 // Use the helper routine to schedule a 4x8H Montgomery multiply.
5728 // It will assert that the register use is valid
5729 vs_montmul4(va, vb, vc, __ T8H, vtmp, vq);
5730 }
5731
5732 // Perform 64 16-bit Montgomery multiplications in parallel.
5733 void kyber_montmul64(const VSeq<8>& va, const VSeq<8>& vb, const VSeq<8>& vc,
5734 const VSeq<4>& vtmp, const VSeq<2>& vq) {
5735 // Schedule two successive 4x8H multiplies via the montmul helper
5736 // on the front and back halves of va, vb and vc. The helper will
5737 // assert that the register use has no overlap conflicts on each
5738 // individual call but we also need to ensure that the necessary
5739 // disjoint/equality constraints are met across both calls.
5740
5741 // vb, vc, vtmp and vq must be disjoint. va must either be
5742 // disjoint from all other registers or equal vc
5743
5744 assert(vs_disjoint(vb, vc), "vb and vc overlap");
5745 assert(vs_disjoint(vb, vq), "vb and vq overlap");
5746 assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap");
5747
5748 assert(vs_disjoint(vc, vq), "vc and vq overlap");
5749 assert(vs_disjoint(vc, vtmp), "vc and vtmp overlap");
5750
5751 assert(vs_disjoint(vq, vtmp), "vq and vtmp overlap");
5752
5753 assert(vs_disjoint(va, vc) || vs_same(va, vc), "va and vc neither disjoint nor equal");
5754 assert(vs_disjoint(va, vb), "va and vb overlap");
5755 assert(vs_disjoint(va, vq), "va and vq overlap");
5756 assert(vs_disjoint(va, vtmp), "va and vtmp overlap");
5757
5758 // we multiply the front and back halves of each sequence 4 at a
5759 // time because
5760 //
5761 // 1) we are currently only able to get 4-way instruction
5762 // parallelism at best
5763 //
5764 // 2) we need registers for the constants in vq and temporary
5765 // scratch registers to hold intermediate results so vtmp can only
5766 // be a VSeq<4> which means we only have 4 scratch slots
5767
5768 vs_montmul4(vs_front(va), vs_front(vb), vs_front(vc), __ T8H, vtmp, vq);
5769 vs_montmul4(vs_back(va), vs_back(vb), vs_back(vc), __ T8H, vtmp, vq);
5770 }
5771
5772 void kyber_montmul32_sub_add(const VSeq<4>& va0, const VSeq<4>& va1,
5773 const VSeq<4>& vc,
5774 const VSeq<4>& vtmp,
5775 const VSeq<2>& vq) {
5776 // compute a = montmul(a1, c)
5777 kyber_montmul32(vc, va1, vc, vtmp, vq);
5778 // ouptut a1 = a0 - a
5779 vs_subv(va1, __ T8H, va0, vc);
5780 // and a0 = a0 + a
5781 vs_addv(va0, __ T8H, va0, vc);
5782 }
5783
5784 void kyber_sub_add_montmul32(const VSeq<4>& va0, const VSeq<4>& va1,
5785 const VSeq<4>& vb,
5786 const VSeq<4>& vtmp1,
5787 const VSeq<4>& vtmp2,
5788 const VSeq<2>& vq) {
5789 // compute c = a0 - a1
5790 vs_subv(vtmp1, __ T8H, va0, va1);
5791 // output a0 = a0 + a1
5792 vs_addv(va0, __ T8H, va0, va1);
5793 // output a1 = b montmul c
5794 kyber_montmul32(va1, vtmp1, vb, vtmp2, vq);
5795 }
5796
5797 void load64shorts(const VSeq<8>& v, Register shorts) {
5798 vs_ldpq_post(v, shorts);
5799 }
5800
5801 void load32shorts(const VSeq<4>& v, Register shorts) {
5802 vs_ldpq_post(v, shorts);
5803 }
5804
5805 void store64shorts(VSeq<8> v, Register tmpAddr) {
5806 vs_stpq_post(v, tmpAddr);
5807 }
5808
5809 // Kyber NTT function.
5810 // Implements
5811 // static int implKyberNtt(short[] poly, short[] ntt_zetas) {}
5812 //
5813 // coeffs (short[256]) = c_rarg0
5814 // ntt_zetas (short[256]) = c_rarg1
5815 address generate_kyberNtt() {
5816 StubId stub_id = StubId::stubgen_kyberNtt_id;
5817 int entry_count = StubInfo::entry_count(stub_id);
5818 assert(entry_count == 1, "sanity check");
5819 address start = load_archive_data(stub_id);
5820 if (start != nullptr) {
5821 return start;
5822 }
5823 __ align(CodeEntryAlignment);
5824 StubCodeMark mark(this, stub_id);
5825 start = __ pc();
5826 __ enter();
5827
5828 const Register coeffs = c_rarg0;
5829 const Register zetas = c_rarg1;
5830
5831 const Register kyberConsts = r10;
5832 const Register tmpAddr = r11;
5833
5834 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x8H inputs/outputs
5835 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
5836 VSeq<2> vq(30); // n.b. constants overlap vs3
5837
5838 __ lea(kyberConsts, ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
5839 // load the montmul constants
5840 vs_ldpq(vq, kyberConsts);
5841
5842 // Each level corresponds to an iteration of the outermost loop of the
5843 // Java method seilerNTT(int[] coeffs). There are some differences
5844 // from what is done in the seilerNTT() method, though:
5845 // 1. The computation is using 16-bit signed values, we do not convert them
5846 // to ints here.
5847 // 2. The zetas are delivered in a bigger array, 128 zetas are stored in
5848 // this array for each level, it is easier that way to fill up the vector
5849 // registers.
5850 // 3. In the seilerNTT() method we use R = 2^20 for the Montgomery
5851 // multiplications (this is because that way there should not be any
5852 // overflow during the inverse NTT computation), here we use R = 2^16 so
5853 // that we can use the 16-bit arithmetic in the vector unit.
5854 //
5855 // On each level, we fill up the vector registers in such a way that the
5856 // array elements that need to be multiplied by the zetas go into one
5857 // set of vector registers while the corresponding ones that don't need to
5858 // be multiplied, go into another set.
5859 // We can do 32 Montgomery multiplications in parallel, using 12 vector
5860 // registers interleaving the steps of 4 identical computations,
5861 // each done on 8 16-bit values per register.
5862
5863 // At levels 0-3 the coefficients multiplied by or added/subtracted
5864 // to the zetas occur in discrete blocks whose size is some multiple
5865 // of 32.
5866
5867 // level 0
5868 __ add(tmpAddr, coeffs, 256);
5869 load64shorts(vs1, tmpAddr);
5870 load64shorts(vs2, zetas);
5871 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5872 __ add(tmpAddr, coeffs, 0);
5873 load64shorts(vs1, tmpAddr);
5874 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5875 vs_addv(vs1, __ T8H, vs1, vs2);
5876 __ add(tmpAddr, coeffs, 0);
5877 vs_stpq_post(vs1, tmpAddr);
5878 __ add(tmpAddr, coeffs, 256);
5879 vs_stpq_post(vs3, tmpAddr);
5880 // restore montmul constants
5881 vs_ldpq(vq, kyberConsts);
5882 load64shorts(vs1, tmpAddr);
5883 load64shorts(vs2, zetas);
5884 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5885 __ add(tmpAddr, coeffs, 128);
5886 load64shorts(vs1, tmpAddr);
5887 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5888 vs_addv(vs1, __ T8H, vs1, vs2);
5889 __ add(tmpAddr, coeffs, 128);
5890 store64shorts(vs1, tmpAddr);
5891 __ add(tmpAddr, coeffs, 384);
5892 store64shorts(vs3, tmpAddr);
5893
5894 // level 1
5895 // restore montmul constants
5896 vs_ldpq(vq, kyberConsts);
5897 __ add(tmpAddr, coeffs, 128);
5898 load64shorts(vs1, tmpAddr);
5899 load64shorts(vs2, zetas);
5900 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5901 __ add(tmpAddr, coeffs, 0);
5902 load64shorts(vs1, tmpAddr);
5903 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5904 vs_addv(vs1, __ T8H, vs1, vs2);
5905 __ add(tmpAddr, coeffs, 0);
5906 store64shorts(vs1, tmpAddr);
5907 store64shorts(vs3, tmpAddr);
5908 vs_ldpq(vq, kyberConsts);
5909 __ add(tmpAddr, coeffs, 384);
5910 load64shorts(vs1, tmpAddr);
5911 load64shorts(vs2, zetas);
5912 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5913 __ add(tmpAddr, coeffs, 256);
5914 load64shorts(vs1, tmpAddr);
5915 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5916 vs_addv(vs1, __ T8H, vs1, vs2);
5917 __ add(tmpAddr, coeffs, 256);
5918 store64shorts(vs1, tmpAddr);
5919 store64shorts(vs3, tmpAddr);
5920
5921 // level 2
5922 vs_ldpq(vq, kyberConsts);
5923 int offsets1[4] = { 0, 32, 128, 160 };
5924 vs_ldpq_indexed(vs1, coeffs, 64, offsets1);
5925 load64shorts(vs2, zetas);
5926 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5927 vs_ldpq_indexed(vs1, coeffs, 0, offsets1);
5928 // kyber_subv_addv64();
5929 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5930 vs_addv(vs1, __ T8H, vs1, vs2);
5931 __ add(tmpAddr, coeffs, 0);
5932 vs_stpq_post(vs_front(vs1), tmpAddr);
5933 vs_stpq_post(vs_front(vs3), tmpAddr);
5934 vs_stpq_post(vs_back(vs1), tmpAddr);
5935 vs_stpq_post(vs_back(vs3), tmpAddr);
5936 vs_ldpq(vq, kyberConsts);
5937 vs_ldpq_indexed(vs1, tmpAddr, 64, offsets1);
5938 load64shorts(vs2, zetas);
5939 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5940 vs_ldpq_indexed(vs1, coeffs, 256, offsets1);
5941 // kyber_subv_addv64();
5942 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5943 vs_addv(vs1, __ T8H, vs1, vs2);
5944 __ add(tmpAddr, coeffs, 256);
5945 vs_stpq_post(vs_front(vs1), tmpAddr);
5946 vs_stpq_post(vs_front(vs3), tmpAddr);
5947 vs_stpq_post(vs_back(vs1), tmpAddr);
5948 vs_stpq_post(vs_back(vs3), tmpAddr);
5949
5950 // level 3
5951 vs_ldpq(vq, kyberConsts);
5952 int offsets2[4] = { 0, 64, 128, 192 };
5953 vs_ldpq_indexed(vs1, coeffs, 32, offsets2);
5954 load64shorts(vs2, zetas);
5955 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5956 vs_ldpq_indexed(vs1, coeffs, 0, offsets2);
5957 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5958 vs_addv(vs1, __ T8H, vs1, vs2);
5959 vs_stpq_indexed(vs1, coeffs, 0, offsets2);
5960 vs_stpq_indexed(vs3, coeffs, 32, offsets2);
5961
5962 vs_ldpq(vq, kyberConsts);
5963 vs_ldpq_indexed(vs1, coeffs, 256 + 32, offsets2);
5964 load64shorts(vs2, zetas);
5965 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5966 vs_ldpq_indexed(vs1, coeffs, 256, offsets2);
5967 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5968 vs_addv(vs1, __ T8H, vs1, vs2);
5969 vs_stpq_indexed(vs1, coeffs, 256, offsets2);
5970 vs_stpq_indexed(vs3, coeffs, 256 + 32, offsets2);
5971
5972 // level 4
5973 // At level 4 coefficients occur in 8 discrete blocks of size 16
5974 // so they are loaded by employing an ldr at 8 distinct offsets.
5975
5976 vs_ldpq(vq, kyberConsts);
5977 int offsets3[8] = { 0, 32, 64, 96, 128, 160, 192, 224 };
5978 vs_ldr_indexed(vs1, __ Q, coeffs, 16, offsets3);
5979 load64shorts(vs2, zetas);
5980 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5981 vs_ldr_indexed(vs1, __ Q, coeffs, 0, offsets3);
5982 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5983 vs_addv(vs1, __ T8H, vs1, vs2);
5984 vs_str_indexed(vs1, __ Q, coeffs, 0, offsets3);
5985 vs_str_indexed(vs3, __ Q, coeffs, 16, offsets3);
5986
5987 vs_ldpq(vq, kyberConsts);
5988 vs_ldr_indexed(vs1, __ Q, coeffs, 256 + 16, offsets3);
5989 load64shorts(vs2, zetas);
5990 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5991 vs_ldr_indexed(vs1, __ Q, coeffs, 256, offsets3);
5992 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5993 vs_addv(vs1, __ T8H, vs1, vs2);
5994 vs_str_indexed(vs1, __ Q, coeffs, 256, offsets3);
5995 vs_str_indexed(vs3, __ Q, coeffs, 256 + 16, offsets3);
5996
5997 // level 5
5998 // At level 5 related coefficients occur in discrete blocks of size 8 so
5999 // need to be loaded interleaved using an ld2 operation with arrangement 2D.
6000
6001 vs_ldpq(vq, kyberConsts);
6002 int offsets4[4] = { 0, 32, 64, 96 };
6003 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4);
6004 load32shorts(vs_front(vs2), zetas);
6005 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6006 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4);
6007 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4);
6008 load32shorts(vs_front(vs2), zetas);
6009 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6010 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4);
6011 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4);
6012 load32shorts(vs_front(vs2), zetas);
6013 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6014 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4);
6015
6016 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4);
6017 load32shorts(vs_front(vs2), zetas);
6018 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6019 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4);
6020
6021 // level 6
6022 // At level 6 related coefficients occur in discrete blocks of size 4 so
6023 // need to be loaded interleaved using an ld2 operation with arrangement 4S.
6024
6025 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4);
6026 load32shorts(vs_front(vs2), zetas);
6027 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6028 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4);
6029 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4);
6030 load32shorts(vs_front(vs2), zetas);
6031 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6032 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4);
6033
6034 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4);
6035 load32shorts(vs_front(vs2), zetas);
6036 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6037 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4);
6038
6039 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4);
6040 load32shorts(vs_front(vs2), zetas);
6041 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6042 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4);
6043
6044 __ leave(); // required for proper stackwalking of RuntimeStub frame
6045 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6046 __ ret(lr);
6047
6048 // record the stub entry and end
6049 store_archive_data(stub_id, start, __ pc());
6050
6051 return start;
6052 }
6053
6054 // Kyber Inverse NTT function
6055 // Implements
6056 // static int implKyberInverseNtt(short[] poly, short[] zetas) {}
6057 //
6058 // coeffs (short[256]) = c_rarg0
6059 // ntt_zetas (short[256]) = c_rarg1
6060 address generate_kyberInverseNtt() {
6061 StubId stub_id = StubId::stubgen_kyberInverseNtt_id;
6062 int entry_count = StubInfo::entry_count(stub_id);
6063 assert(entry_count == 1, "sanity check");
6064 address start = load_archive_data(stub_id);
6065 if (start != nullptr) {
6066 return start;
6067 }
6068 __ align(CodeEntryAlignment);
6069 StubCodeMark mark(this, stub_id);
6070 start = __ pc();
6071 __ enter();
6072
6073 const Register coeffs = c_rarg0;
6074 const Register zetas = c_rarg1;
6075
6076 const Register kyberConsts = r10;
6077 const Register tmpAddr = r11;
6078 const Register tmpAddr2 = c_rarg2;
6079
6080 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x8H inputs/outputs
6081 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
6082 VSeq<2> vq(30); // n.b. constants overlap vs3
6083
6084 __ lea(kyberConsts,
6085 ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
6086
6087 // level 0
6088 // At level 0 related coefficients occur in discrete blocks of size 4 so
6089 // need to be loaded interleaved using an ld2 operation with arrangement 4S.
6090
6091 vs_ldpq(vq, kyberConsts);
6092 int offsets4[4] = { 0, 32, 64, 96 };
6093 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4);
6094 load32shorts(vs_front(vs2), zetas);
6095 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6096 vs_front(vs2), vs_back(vs2), vtmp, vq);
6097 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4);
6098 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4);
6099 load32shorts(vs_front(vs2), zetas);
6100 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6101 vs_front(vs2), vs_back(vs2), vtmp, vq);
6102 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4);
6103 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4);
6104 load32shorts(vs_front(vs2), zetas);
6105 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6106 vs_front(vs2), vs_back(vs2), vtmp, vq);
6107 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4);
6108 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4);
6109 load32shorts(vs_front(vs2), zetas);
6110 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6111 vs_front(vs2), vs_back(vs2), vtmp, vq);
6112 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4);
6113
6114 // level 1
6115 // At level 1 related coefficients occur in discrete blocks of size 8 so
6116 // need to be loaded interleaved using an ld2 operation with arrangement 2D.
6117
6118 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4);
6119 load32shorts(vs_front(vs2), zetas);
6120 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6121 vs_front(vs2), vs_back(vs2), vtmp, vq);
6122 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4);
6123 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4);
6124 load32shorts(vs_front(vs2), zetas);
6125 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6126 vs_front(vs2), vs_back(vs2), vtmp, vq);
6127 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4);
6128
6129 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4);
6130 load32shorts(vs_front(vs2), zetas);
6131 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6132 vs_front(vs2), vs_back(vs2), vtmp, vq);
6133 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4);
6134 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4);
6135 load32shorts(vs_front(vs2), zetas);
6136 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6137 vs_front(vs2), vs_back(vs2), vtmp, vq);
6138 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4);
6139
6140 // level 2
6141 // At level 2 coefficients occur in 8 discrete blocks of size 16
6142 // so they are loaded by employing an ldr at 8 distinct offsets.
6143
6144 int offsets3[8] = { 0, 32, 64, 96, 128, 160, 192, 224 };
6145 vs_ldr_indexed(vs1, __ Q, coeffs, 0, offsets3);
6146 vs_ldr_indexed(vs2, __ Q, coeffs, 16, offsets3);
6147 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6148 vs_subv(vs1, __ T8H, vs1, vs2);
6149 vs_str_indexed(vs3, __ Q, coeffs, 0, offsets3);
6150 load64shorts(vs2, zetas);
6151 vs_ldpq(vq, kyberConsts);
6152 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6153 vs_str_indexed(vs2, __ Q, coeffs, 16, offsets3);
6154
6155 vs_ldr_indexed(vs1, __ Q, coeffs, 256, offsets3);
6156 vs_ldr_indexed(vs2, __ Q, coeffs, 256 + 16, offsets3);
6157 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6158 vs_subv(vs1, __ T8H, vs1, vs2);
6159 vs_str_indexed(vs3, __ Q, coeffs, 256, offsets3);
6160 load64shorts(vs2, zetas);
6161 vs_ldpq(vq, kyberConsts);
6162 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6163 vs_str_indexed(vs2, __ Q, coeffs, 256 + 16, offsets3);
6164
6165 // Barrett reduction at indexes where overflow may happen
6166
6167 // load q and the multiplier for the Barrett reduction
6168 __ add(tmpAddr, kyberConsts, 16);
6169 vs_ldpq(vq, tmpAddr);
6170
6171 VSeq<8> vq1 = VSeq<8>(vq[0], 0); // 2 constant 8 sequences
6172 VSeq<8> vq2 = VSeq<8>(vq[1], 0); // for above two kyber constants
6173 VSeq<8> vq3 = VSeq<8>(v29, 0); // 3rd sequence for const montmul
6174 vs_ldr_indexed(vs1, __ Q, coeffs, 0, offsets3);
6175 vs_sqdmulh(vs2, __ T8H, vs1, vq2);
6176 vs_sshr(vs2, __ T8H, vs2, 11);
6177 vs_mlsv(vs1, __ T8H, vs2, vq1);
6178 vs_str_indexed(vs1, __ Q, coeffs, 0, offsets3);
6179 vs_ldr_indexed(vs1, __ Q, coeffs, 256, offsets3);
6180 vs_sqdmulh(vs2, __ T8H, vs1, vq2);
6181 vs_sshr(vs2, __ T8H, vs2, 11);
6182 vs_mlsv(vs1, __ T8H, vs2, vq1);
6183 vs_str_indexed(vs1, __ Q, coeffs, 256, offsets3);
6184
6185 // level 3
6186 // From level 3 upwards coefficients occur in discrete blocks whose size is
6187 // some multiple of 32 so can be loaded using ldpq and suitable indexes.
6188
6189 int offsets2[4] = { 0, 64, 128, 192 };
6190 vs_ldpq_indexed(vs1, coeffs, 0, offsets2);
6191 vs_ldpq_indexed(vs2, coeffs, 32, offsets2);
6192 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6193 vs_subv(vs1, __ T8H, vs1, vs2);
6194 vs_stpq_indexed(vs3, coeffs, 0, offsets2);
6195 load64shorts(vs2, zetas);
6196 vs_ldpq(vq, kyberConsts);
6197 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6198 vs_stpq_indexed(vs2, coeffs, 32, offsets2);
6199
6200 vs_ldpq_indexed(vs1, coeffs, 256, offsets2);
6201 vs_ldpq_indexed(vs2, coeffs, 256 + 32, offsets2);
6202 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6203 vs_subv(vs1, __ T8H, vs1, vs2);
6204 vs_stpq_indexed(vs3, coeffs, 256, offsets2);
6205 load64shorts(vs2, zetas);
6206 vs_ldpq(vq, kyberConsts);
6207 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6208 vs_stpq_indexed(vs2, coeffs, 256 + 32, offsets2);
6209
6210 // level 4
6211
6212 int offsets1[4] = { 0, 32, 128, 160 };
6213 vs_ldpq_indexed(vs1, coeffs, 0, offsets1);
6214 vs_ldpq_indexed(vs2, coeffs, 64, offsets1);
6215 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6216 vs_subv(vs1, __ T8H, vs1, vs2);
6217 vs_stpq_indexed(vs3, coeffs, 0, offsets1);
6218 load64shorts(vs2, zetas);
6219 vs_ldpq(vq, kyberConsts);
6220 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6221 vs_stpq_indexed(vs2, coeffs, 64, offsets1);
6222
6223 vs_ldpq_indexed(vs1, coeffs, 256, offsets1);
6224 vs_ldpq_indexed(vs2, coeffs, 256 + 64, offsets1);
6225 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6226 vs_subv(vs1, __ T8H, vs1, vs2);
6227 vs_stpq_indexed(vs3, coeffs, 256, offsets1);
6228 load64shorts(vs2, zetas);
6229 vs_ldpq(vq, kyberConsts);
6230 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6231 vs_stpq_indexed(vs2, coeffs, 256 + 64, offsets1);
6232
6233 // level 5
6234
6235 __ add(tmpAddr, coeffs, 0);
6236 load64shorts(vs1, tmpAddr);
6237 __ add(tmpAddr, coeffs, 128);
6238 load64shorts(vs2, tmpAddr);
6239 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6240 vs_subv(vs1, __ T8H, vs1, vs2);
6241 __ add(tmpAddr, coeffs, 0);
6242 store64shorts(vs3, tmpAddr);
6243 load64shorts(vs2, zetas);
6244 vs_ldpq(vq, kyberConsts);
6245 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6246 __ add(tmpAddr, coeffs, 128);
6247 store64shorts(vs2, tmpAddr);
6248
6249 load64shorts(vs1, tmpAddr);
6250 __ add(tmpAddr, coeffs, 384);
6251 load64shorts(vs2, tmpAddr);
6252 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6253 vs_subv(vs1, __ T8H, vs1, vs2);
6254 __ add(tmpAddr, coeffs, 256);
6255 store64shorts(vs3, tmpAddr);
6256 load64shorts(vs2, zetas);
6257 vs_ldpq(vq, kyberConsts);
6258 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6259 __ add(tmpAddr, coeffs, 384);
6260 store64shorts(vs2, tmpAddr);
6261
6262 // Barrett reduction at indexes where overflow may happen
6263
6264 // load q and the multiplier for the Barrett reduction
6265 __ add(tmpAddr, kyberConsts, 16);
6266 vs_ldpq(vq, tmpAddr);
6267
6268 int offsets0[2] = { 0, 256 };
6269 vs_ldpq_indexed(vs_front(vs1), coeffs, 0, offsets0);
6270 vs_sqdmulh(vs2, __ T8H, vs1, vq2);
6271 vs_sshr(vs2, __ T8H, vs2, 11);
6272 vs_mlsv(vs1, __ T8H, vs2, vq1);
6273 vs_stpq_indexed(vs_front(vs1), coeffs, 0, offsets0);
6274
6275 // level 6
6276
6277 __ add(tmpAddr, coeffs, 0);
6278 load64shorts(vs1, tmpAddr);
6279 __ add(tmpAddr, coeffs, 256);
6280 load64shorts(vs2, tmpAddr);
6281 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6282 vs_subv(vs1, __ T8H, vs1, vs2);
6283 __ add(tmpAddr, coeffs, 0);
6284 store64shorts(vs3, tmpAddr);
6285 load64shorts(vs2, zetas);
6286 vs_ldpq(vq, kyberConsts);
6287 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6288 __ add(tmpAddr, coeffs, 256);
6289 store64shorts(vs2, tmpAddr);
6290
6291 __ add(tmpAddr, coeffs, 128);
6292 load64shorts(vs1, tmpAddr);
6293 __ add(tmpAddr, coeffs, 384);
6294 load64shorts(vs2, tmpAddr);
6295 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6296 vs_subv(vs1, __ T8H, vs1, vs2);
6297 __ add(tmpAddr, coeffs, 128);
6298 store64shorts(vs3, tmpAddr);
6299 load64shorts(vs2, zetas);
6300 vs_ldpq(vq, kyberConsts);
6301 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6302 __ add(tmpAddr, coeffs, 384);
6303 store64shorts(vs2, tmpAddr);
6304
6305 // multiply by 2^-n
6306
6307 // load toMont(2^-n mod q)
6308 __ add(tmpAddr, kyberConsts, 48);
6309 __ ldr(v29, __ Q, tmpAddr);
6310
6311 vs_ldpq(vq, kyberConsts);
6312 __ add(tmpAddr, coeffs, 0);
6313 load64shorts(vs1, tmpAddr);
6314 kyber_montmul64(vs2, vs1, vq3, vtmp, vq);
6315 __ add(tmpAddr, coeffs, 0);
6316 store64shorts(vs2, tmpAddr);
6317
6318 // now tmpAddr contains coeffs + 128 because store64shorts adjusted it so
6319 load64shorts(vs1, tmpAddr);
6320 kyber_montmul64(vs2, vs1, vq3, vtmp, vq);
6321 __ add(tmpAddr, coeffs, 128);
6322 store64shorts(vs2, tmpAddr);
6323
6324 // now tmpAddr contains coeffs + 256
6325 load64shorts(vs1, tmpAddr);
6326 kyber_montmul64(vs2, vs1, vq3, vtmp, vq);
6327 __ add(tmpAddr, coeffs, 256);
6328 store64shorts(vs2, tmpAddr);
6329
6330 // now tmpAddr contains coeffs + 384
6331 load64shorts(vs1, tmpAddr);
6332 kyber_montmul64(vs2, vs1, vq3, vtmp, vq);
6333 __ add(tmpAddr, coeffs, 384);
6334 store64shorts(vs2, tmpAddr);
6335
6336 __ leave(); // required for proper stackwalking of RuntimeStub frame
6337 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6338 __ ret(lr);
6339
6340 // record the stub entry and end
6341 store_archive_data(stub_id, start, __ pc());
6342
6343 return start;
6344 }
6345
6346 // Kyber multiply polynomials in the NTT domain.
6347 // Implements
6348 // static int implKyberNttMult(
6349 // short[] result, short[] ntta, short[] nttb, short[] zetas) {}
6350 //
6351 // The actual algorithm that is used here differs from the one in the Java
6352 // implementation, it uses Montgomery multiplications instead of Barrett
6353 // reduction, but the end result modulo MLKEM_Q is the same. This is the
6354 // Java equivalent of this intrinsic implementation:
6355 // static void implKyberNttMultJava(short[] result, short[] ntta, short[] nttb) {
6356 // for (int m = 0; m < ML_KEM_N / 2; m++) {
6357 // int a0 = ntta[2 * m];
6358 // int a1 = ntta[2 * m + 1];
6359 // int b0 = nttb[2 * m];
6360 // int b1 = nttb[2 * m + 1];
6361 // int r = montMul(a0, b0) +
6362 // montMul(montMul(a1, b1), MONT_ZETAS_FOR_NTT_MULT[m]);
6363 // result[2 * m] = (short) montMul(r, MONT_R_SQUARE_MOD_Q);
6364 // result[2 * m + 1] = (short) montMul(
6365 // (montMul(a0, b1) + montMul(a1, b0)), MONT_R_SQUARE_MOD_Q);
6366 // }
6367 // }
6368 //
6369 // result (short[256]) = c_rarg0
6370 // ntta (short[256]) = c_rarg1
6371 // nttb (short[256]) = c_rarg2
6372 // zetas (short[128]) = c_rarg3
6373 address generate_kyberNttMult() {
6374 StubId stub_id = StubId::stubgen_kyberNttMult_id;
6375 int entry_count = StubInfo::entry_count(stub_id);
6376 assert(entry_count == 1, "sanity check");
6377 address start = load_archive_data(stub_id);
6378 if (start != nullptr) {
6379 return start;
6380 }
6381 __ align(CodeEntryAlignment);
6382 StubCodeMark mark(this, stub_id);
6383 start = __ pc();
6384 __ enter();
6385
6386 const Register result = c_rarg0;
6387 const Register ntta = c_rarg1;
6388 const Register nttb = c_rarg2;
6389 const Register zetas = c_rarg3;
6390
6391 const Register kyberConsts = r10;
6392 const Register limit = r11;
6393
6394 VSeq<4> vs1(0), vs2(4); // 4 sets of 8x8H inputs/outputs/tmps
6395 VSeq<4> vs3(16), vs4(20);
6396 VSeq<2> vq(30); // pair of constants for montmul: q, qinv
6397 VSeq<2> vz(28); // pair of zetas
6398 VSeq<4> vc(27, 0); // constant sequence for montmul: montRSquareModQ
6399
6400 __ lea(kyberConsts,
6401 ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
6402
6403 Label kyberNttMult_loop;
6404
6405 __ add(limit, result, 512);
6406
6407 // load q and qinv
6408 vs_ldpq(vq, kyberConsts);
6409
6410 // load R^2 mod q (to convert back from Montgomery representation)
6411 __ add(kyberConsts, kyberConsts, 64);
6412 __ ldr(v27, __ Q, kyberConsts);
6413
6414 __ BIND(kyberNttMult_loop);
6415
6416 // load 16 zetas
6417 vs_ldpq_post(vz, zetas);
6418
6419 // load 2 sets of 32 coefficients from the two input arrays
6420 // interleaved as shorts. i.e. pairs of shorts adjacent in memory
6421 // are striped across pairs of vector registers
6422 vs_ld2_post(vs_front(vs1), __ T8H, ntta); // <a0, a1> x 8H
6423 vs_ld2_post(vs_back(vs1), __ T8H, nttb); // <b0, b1> x 8H
6424 vs_ld2_post(vs_front(vs4), __ T8H, ntta); // <a2, a3> x 8H
6425 vs_ld2_post(vs_back(vs4), __ T8H, nttb); // <b2, b3> x 8H
6426
6427 // compute 4 montmul cross-products for pairs (a0,a1) and (b0,b1)
6428 // i.e. montmul the first and second halves of vs1 in order and
6429 // then with one sequence reversed storing the two results in vs3
6430 //
6431 // vs3[0] <- montmul(a0, b0)
6432 // vs3[1] <- montmul(a1, b1)
6433 // vs3[2] <- montmul(a0, b1)
6434 // vs3[3] <- montmul(a1, b0)
6435 kyber_montmul16(vs_front(vs3), vs_front(vs1), vs_back(vs1), vs_front(vs2), vq);
6436 kyber_montmul16(vs_back(vs3),
6437 vs_front(vs1), vs_reverse(vs_back(vs1)), vs_back(vs2), vq);
6438
6439 // compute 4 montmul cross-products for pairs (a2,a3) and (b2,b3)
6440 // i.e. montmul the first and second halves of vs4 in order and
6441 // then with one sequence reversed storing the two results in vs1
6442 //
6443 // vs1[0] <- montmul(a2, b2)
6444 // vs1[1] <- montmul(a3, b3)
6445 // vs1[2] <- montmul(a2, b3)
6446 // vs1[3] <- montmul(a3, b2)
6447 kyber_montmul16(vs_front(vs1), vs_front(vs4), vs_back(vs4), vs_front(vs2), vq);
6448 kyber_montmul16(vs_back(vs1),
6449 vs_front(vs4), vs_reverse(vs_back(vs4)), vs_back(vs2), vq);
6450
6451 // montmul result 2 of each cross-product i.e. (a1*b1, a3*b3) by a zeta.
6452 // We can schedule two montmuls at a time if we use a suitable vector
6453 // sequence <vs3[1], vs1[1]>.
6454 int delta = vs1[1]->encoding() - vs3[1]->encoding();
6455 VSeq<2> vs5(vs3[1], delta);
6456
6457 // vs3[1] <- montmul(montmul(a1, b1), z0)
6458 // vs1[1] <- montmul(montmul(a3, b3), z1)
6459 kyber_montmul16(vs5, vz, vs5, vs_front(vs2), vq);
6460
6461 // add results in pairs storing in vs3
6462 // vs3[0] <- montmul(a0, b0) + montmul(montmul(a1, b1), z0);
6463 // vs3[1] <- montmul(a0, b1) + montmul(a1, b0);
6464 vs_addv(vs_front(vs3), __ T8H, vs_even(vs3), vs_odd(vs3));
6465
6466 // vs3[2] <- montmul(a2, b2) + montmul(montmul(a3, b3), z1);
6467 // vs3[3] <- montmul(a2, b3) + montmul(a3, b2);
6468 vs_addv(vs_back(vs3), __ T8H, vs_even(vs1), vs_odd(vs1));
6469
6470 // vs1 <- montmul(vs3, montRSquareModQ)
6471 kyber_montmul32(vs1, vs3, vc, vs2, vq);
6472
6473 // store back the two pairs of result vectors de-interleaved as 8H elements
6474 // i.e. storing each pairs of shorts striped across a register pair adjacent
6475 // in memory
6476 vs_st2_post(vs1, __ T8H, result);
6477
6478 __ cmp(result, limit);
6479 __ br(Assembler::NE, kyberNttMult_loop);
6480
6481 __ leave(); // required for proper stackwalking of RuntimeStub frame
6482 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6483 __ ret(lr);
6484
6485 // record the stub entry and end
6486 store_archive_data(stub_id, start, __ pc());
6487
6488 return start;
6489 }
6490
6491 // Kyber add 2 polynomials.
6492 // Implements
6493 // static int implKyberAddPoly(short[] result, short[] a, short[] b) {}
6494 //
6495 // result (short[256]) = c_rarg0
6496 // a (short[256]) = c_rarg1
6497 // b (short[256]) = c_rarg2
6498 address generate_kyberAddPoly_2() {
6499 StubId stub_id = StubId::stubgen_kyberAddPoly_2_id;
6500 int entry_count = StubInfo::entry_count(stub_id);
6501 assert(entry_count == 1, "sanity check");
6502 address start = load_archive_data(stub_id);
6503 if (start != nullptr) {
6504 return start;
6505 }
6506 __ align(CodeEntryAlignment);
6507 StubCodeMark mark(this, stub_id);
6508 start = __ pc();
6509 __ enter();
6510
6511 const Register result = c_rarg0;
6512 const Register a = c_rarg1;
6513 const Register b = c_rarg2;
6514
6515 const Register kyberConsts = r11;
6516
6517 // We sum 256 sets of values in total i.e. 32 x 8H quadwords.
6518 // So, we can load, add and store the data in 3 groups of 11,
6519 // 11 and 10 at a time i.e. we need to map sets of 10 or 11
6520 // registers. A further constraint is that the mapping needs
6521 // to skip callee saves. So, we allocate the register
6522 // sequences using two 8 sequences, two 2 sequences and two
6523 // single registers.
6524 VSeq<8> vs1_1(0);
6525 VSeq<2> vs1_2(16);
6526 FloatRegister vs1_3 = v28;
6527 VSeq<8> vs2_1(18);
6528 VSeq<2> vs2_2(26);
6529 FloatRegister vs2_3 = v29;
6530
6531 // two constant vector sequences
6532 VSeq<8> vc_1(31, 0);
6533 VSeq<2> vc_2(31, 0);
6534
6535 FloatRegister vc_3 = v31;
6536 __ lea(kyberConsts,
6537 ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
6538
6539 __ ldr(vc_3, __ Q, Address(kyberConsts, 16)); // q
6540 for (int i = 0; i < 3; i++) {
6541 // load 80 or 88 values from a into vs1_1/2/3
6542 vs_ldpq_post(vs1_1, a);
6543 vs_ldpq_post(vs1_2, a);
6544 if (i < 2) {
6545 __ ldr(vs1_3, __ Q, __ post(a, 16));
6546 }
6547 // load 80 or 88 values from b into vs2_1/2/3
6548 vs_ldpq_post(vs2_1, b);
6549 vs_ldpq_post(vs2_2, b);
6550 if (i < 2) {
6551 __ ldr(vs2_3, __ Q, __ post(b, 16));
6552 }
6553 // sum 80 or 88 values across vs1 and vs2 into vs1
6554 vs_addv(vs1_1, __ T8H, vs1_1, vs2_1);
6555 vs_addv(vs1_2, __ T8H, vs1_2, vs2_2);
6556 if (i < 2) {
6557 __ addv(vs1_3, __ T8H, vs1_3, vs2_3);
6558 }
6559 // add constant to all 80 or 88 results
6560 vs_addv(vs1_1, __ T8H, vs1_1, vc_1);
6561 vs_addv(vs1_2, __ T8H, vs1_2, vc_2);
6562 if (i < 2) {
6563 __ addv(vs1_3, __ T8H, vs1_3, vc_3);
6564 }
6565 // store 80 or 88 values
6566 vs_stpq_post(vs1_1, result);
6567 vs_stpq_post(vs1_2, result);
6568 if (i < 2) {
6569 __ str(vs1_3, __ Q, __ post(result, 16));
6570 }
6571 }
6572
6573 __ leave(); // required for proper stackwalking of RuntimeStub frame
6574 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6575 __ ret(lr);
6576
6577 // record the stub entry and end
6578 store_archive_data(stub_id, start, __ pc());
6579
6580 return start;
6581 }
6582
6583 // Kyber add 3 polynomials.
6584 // Implements
6585 // static int implKyberAddPoly(short[] result, short[] a, short[] b, short[] c) {}
6586 //
6587 // result (short[256]) = c_rarg0
6588 // a (short[256]) = c_rarg1
6589 // b (short[256]) = c_rarg2
6590 // c (short[256]) = c_rarg3
6591 address generate_kyberAddPoly_3() {
6592 StubId stub_id = StubId::stubgen_kyberAddPoly_3_id;
6593 int entry_count = StubInfo::entry_count(stub_id);
6594 assert(entry_count == 1, "sanity check");
6595 address start = load_archive_data(stub_id);
6596 if (start != nullptr) {
6597 return start;
6598 }
6599 __ align(CodeEntryAlignment);
6600 StubCodeMark mark(this, stub_id);
6601 start = __ pc();
6602 __ enter();
6603
6604 const Register result = c_rarg0;
6605 const Register a = c_rarg1;
6606 const Register b = c_rarg2;
6607 const Register c = c_rarg3;
6608
6609 const Register kyberConsts = r11;
6610
6611 // As above we sum 256 sets of values in total i.e. 32 x 8H
6612 // quadwords. So, we can load, add and store the data in 3
6613 // groups of 11, 11 and 10 at a time i.e. we need to map sets
6614 // of 10 or 11 registers. A further constraint is that the
6615 // mapping needs to skip callee saves. So, we allocate the
6616 // register sequences using two 8 sequences, two 2 sequences
6617 // and two single registers.
6618 VSeq<8> vs1_1(0);
6619 VSeq<2> vs1_2(16);
6620 FloatRegister vs1_3 = v28;
6621 VSeq<8> vs2_1(18);
6622 VSeq<2> vs2_2(26);
6623 FloatRegister vs2_3 = v29;
6624
6625 // two constant vector sequences
6626 VSeq<8> vc_1(31, 0);
6627 VSeq<2> vc_2(31, 0);
6628
6629 FloatRegister vc_3 = v31;
6630
6631 __ lea(kyberConsts,
6632 ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
6633
6634 __ ldr(vc_3, __ Q, Address(kyberConsts, 16)); // q
6635 for (int i = 0; i < 3; i++) {
6636 // load 80 or 88 values from a into vs1_1/2/3
6637 vs_ldpq_post(vs1_1, a);
6638 vs_ldpq_post(vs1_2, a);
6639 if (i < 2) {
6640 __ ldr(vs1_3, __ Q, __ post(a, 16));
6641 }
6642 // load 80 or 88 values from b into vs2_1/2/3
6643 vs_ldpq_post(vs2_1, b);
6644 vs_ldpq_post(vs2_2, b);
6645 if (i < 2) {
6646 __ ldr(vs2_3, __ Q, __ post(b, 16));
6647 }
6648 // sum 80 or 88 values across vs1 and vs2 into vs1
6649 vs_addv(vs1_1, __ T8H, vs1_1, vs2_1);
6650 vs_addv(vs1_2, __ T8H, vs1_2, vs2_2);
6651 if (i < 2) {
6652 __ addv(vs1_3, __ T8H, vs1_3, vs2_3);
6653 }
6654 // load 80 or 88 values from c into vs2_1/2/3
6655 vs_ldpq_post(vs2_1, c);
6656 vs_ldpq_post(vs2_2, c);
6657 if (i < 2) {
6658 __ ldr(vs2_3, __ Q, __ post(c, 16));
6659 }
6660 // sum 80 or 88 values across vs1 and vs2 into vs1
6661 vs_addv(vs1_1, __ T8H, vs1_1, vs2_1);
6662 vs_addv(vs1_2, __ T8H, vs1_2, vs2_2);
6663 if (i < 2) {
6664 __ addv(vs1_3, __ T8H, vs1_3, vs2_3);
6665 }
6666 // add constant to all 80 or 88 results
6667 vs_addv(vs1_1, __ T8H, vs1_1, vc_1);
6668 vs_addv(vs1_2, __ T8H, vs1_2, vc_2);
6669 if (i < 2) {
6670 __ addv(vs1_3, __ T8H, vs1_3, vc_3);
6671 }
6672 // store 80 or 88 values
6673 vs_stpq_post(vs1_1, result);
6674 vs_stpq_post(vs1_2, result);
6675 if (i < 2) {
6676 __ str(vs1_3, __ Q, __ post(result, 16));
6677 }
6678 }
6679
6680 __ leave(); // required for proper stackwalking of RuntimeStub frame
6681 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6682 __ ret(lr);
6683
6684 // record the stub entry and end
6685 store_archive_data(stub_id, start, __ pc());
6686
6687 return start;
6688 }
6689
6690 // Kyber parse XOF output to polynomial coefficient candidates
6691 // or decodePoly(12, ...).
6692 // Implements
6693 // static int implKyber12To16(
6694 // byte[] condensed, int index, short[] parsed, int parsedLength) {}
6695 //
6696 // we assume that parsed and condensed are allocated such that for
6697 // n = (parsedLength + 63) / 64
6698 // n blocks of 96 bytes of input can be processed, i.e.
6699 // index + n * 96 <= condensed.length and
6700 // n * 64 <= parsed.length
6701 //
6702 // condensed (byte[]) = c_rarg0
6703 // condensedIndex = c_rarg1
6704 // parsed (short[]) = c_rarg2
6705 // parsedLength = c_rarg3
6706 address generate_kyber12To16() {
6707 StubId stub_id = StubId::stubgen_kyber12To16_id;
6708 int entry_count = StubInfo::entry_count(stub_id);
6709 assert(entry_count == 1, "sanity check");
6710 address start = load_archive_data(stub_id);
6711 if (start != nullptr) {
6712 return start;
6713 }
6714 Label L_F00, L_loop;
6715
6716 __ align(CodeEntryAlignment);
6717 StubCodeMark mark(this, stub_id);
6718 start = __ pc();
6719 __ enter();
6720
6721 const Register condensed = c_rarg0;
6722 const Register condensedOffs = c_rarg1;
6723 const Register parsed = c_rarg2;
6724 const Register parsedLength = c_rarg3;
6725
6726 const Register tmpAddr = r11;
6727
6728 // Data is input 96 bytes at a time i.e. in groups of 6 x 16B
6729 // quadwords so we need a 6 vector sequence for the inputs.
6730 // Parsing produces 64 shorts, employing two 8 vector
6731 // sequences to store and combine the intermediate data.
6732 VSeq<6> vin(24);
6733 VSeq<8> va(0), vb(16);
6734
6735 __ adr(tmpAddr, L_F00);
6736 __ ldr(v31, __ Q, tmpAddr); // 8H times 0x0f00
6737 __ add(condensed, condensed, condensedOffs);
6738
6739 __ BIND(L_loop);
6740 // load 96 (6 x 16B) byte values
6741 vs_ld3_post(vin, __ T16B, condensed);
6742
6743 // The front half of sequence vin (vin[0], vin[1] and vin[2])
6744 // holds 48 (16x3) contiguous bytes from memory striped
6745 // horizontally across each of the 16 byte lanes. Equivalently,
6746 // that is 16 pairs of 12-bit integers. Likewise the back half
6747 // holds the next 48 bytes in the same arrangement.
6748
6749 // Each vector in the front half can also be viewed as a vertical
6750 // strip across the 16 pairs of 12 bit integers. Each byte in
6751 // vin[0] stores the low 8 bits of the first int in a pair. Each
6752 // byte in vin[1] stores the high 4 bits of the first int and the
6753 // low 4 bits of the second int. Each byte in vin[2] stores the
6754 // high 8 bits of the second int. Likewise the vectors in second
6755 // half.
6756
6757 // Converting the data to 16-bit shorts requires first of all
6758 // expanding each of the 6 x 16B vectors into 6 corresponding
6759 // pairs of 8H vectors. Mask, shift and add operations on the
6760 // resulting vector pairs can be used to combine 4 and 8 bit
6761 // parts of related 8H vector elements.
6762 //
6763 // The middle vectors (vin[2] and vin[5]) are actually expanded
6764 // twice, one copy manipulated to provide the lower 4 bits
6765 // belonging to the first short in a pair and another copy
6766 // manipulated to provide the higher 4 bits belonging to the
6767 // second short in a pair. This is why the vector sequences va
6768 // and vb are used to hold the expanded 8H elements are of length 8.
6769
6770 // Expand vin[0] into va[0:1], and vin[1] into va[2:3] and va[4:5]
6771 // n.b. target elements 2 and 3 duplicate elements 4 and 5
6772 __ ushll(va[0], __ T8H, vin[0], __ T8B, 0);
6773 __ ushll2(va[1], __ T8H, vin[0], __ T16B, 0);
6774 __ ushll(va[2], __ T8H, vin[1], __ T8B, 0);
6775 __ ushll2(va[3], __ T8H, vin[1], __ T16B, 0);
6776 __ ushll(va[4], __ T8H, vin[1], __ T8B, 0);
6777 __ ushll2(va[5], __ T8H, vin[1], __ T16B, 0);
6778
6779 // likewise expand vin[3] into vb[0:1], and vin[4] into vb[2:3]
6780 // and vb[4:5]
6781 __ ushll(vb[0], __ T8H, vin[3], __ T8B, 0);
6782 __ ushll2(vb[1], __ T8H, vin[3], __ T16B, 0);
6783 __ ushll(vb[2], __ T8H, vin[4], __ T8B, 0);
6784 __ ushll2(vb[3], __ T8H, vin[4], __ T16B, 0);
6785 __ ushll(vb[4], __ T8H, vin[4], __ T8B, 0);
6786 __ ushll2(vb[5], __ T8H, vin[4], __ T16B, 0);
6787
6788 // shift lo byte of copy 1 of the middle stripe into the high byte
6789 __ shl(va[2], __ T8H, va[2], 8);
6790 __ shl(va[3], __ T8H, va[3], 8);
6791 __ shl(vb[2], __ T8H, vb[2], 8);
6792 __ shl(vb[3], __ T8H, vb[3], 8);
6793
6794 // expand vin[2] into va[6:7] and vin[5] into vb[6:7] but this
6795 // time pre-shifted by 4 to ensure top bits of input 12-bit int
6796 // are in bit positions [4..11].
6797 __ ushll(va[6], __ T8H, vin[2], __ T8B, 4);
6798 __ ushll2(va[7], __ T8H, vin[2], __ T16B, 4);
6799 __ ushll(vb[6], __ T8H, vin[5], __ T8B, 4);
6800 __ ushll2(vb[7], __ T8H, vin[5], __ T16B, 4);
6801
6802 // mask hi 4 bits of the 1st 12-bit int in a pair from copy1 and
6803 // shift lo 4 bits of the 2nd 12-bit int in a pair to the bottom of
6804 // copy2
6805 __ andr(va[2], __ T16B, va[2], v31);
6806 __ andr(va[3], __ T16B, va[3], v31);
6807 __ ushr(va[4], __ T8H, va[4], 4);
6808 __ ushr(va[5], __ T8H, va[5], 4);
6809 __ andr(vb[2], __ T16B, vb[2], v31);
6810 __ andr(vb[3], __ T16B, vb[3], v31);
6811 __ ushr(vb[4], __ T8H, vb[4], 4);
6812 __ ushr(vb[5], __ T8H, vb[5], 4);
6813
6814 // sum hi 4 bits and lo 8 bits of the 1st 12-bit int in each pair and
6815 // hi 8 bits plus lo 4 bits of the 2nd 12-bit int in each pair
6816 // n.b. the ordering ensures: i) inputs are consumed before they
6817 // are overwritten ii) the order of 16-bit results across successive
6818 // pairs of vectors in va and then vb reflects the order of the
6819 // corresponding 12-bit inputs
6820 __ addv(va[0], __ T8H, va[0], va[2]);
6821 __ addv(va[2], __ T8H, va[1], va[3]);
6822 __ addv(va[1], __ T8H, va[4], va[6]);
6823 __ addv(va[3], __ T8H, va[5], va[7]);
6824 __ addv(vb[0], __ T8H, vb[0], vb[2]);
6825 __ addv(vb[2], __ T8H, vb[1], vb[3]);
6826 __ addv(vb[1], __ T8H, vb[4], vb[6]);
6827 __ addv(vb[3], __ T8H, vb[5], vb[7]);
6828
6829 // store 64 results interleaved as shorts
6830 vs_st2_post(vs_front(va), __ T8H, parsed);
6831 vs_st2_post(vs_front(vb), __ T8H, parsed);
6832
6833 __ sub(parsedLength, parsedLength, 64);
6834 __ cmp(parsedLength, (u1)0);
6835 __ br(Assembler::GT, L_loop);
6836
6837 __ leave(); // required for proper stackwalking of RuntimeStub frame
6838 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6839 __ ret(lr);
6840
6841 // bind label and generate constant data used by this stub
6842 __ BIND(L_F00);
6843 __ emit_int64(0x0f000f000f000f00);
6844 __ emit_int64(0x0f000f000f000f00);
6845
6846 // record the stub entry and end
6847 store_archive_data(stub_id, start, __ pc());
6848
6849 return start;
6850 }
6851
6852 // Kyber Barrett reduce function.
6853 // Implements
6854 // static int implKyberBarrettReduce(short[] coeffs) {}
6855 //
6856 // coeffs (short[256]) = c_rarg0
6857 address generate_kyberBarrettReduce() {
6858 StubId stub_id = StubId::stubgen_kyberBarrettReduce_id;
6859 int entry_count = StubInfo::entry_count(stub_id);
6860 assert(entry_count == 1, "sanity check");
6861 address start = load_archive_data(stub_id);
6862 if (start != nullptr) {
6863 return start;
6864 }
6865 __ align(CodeEntryAlignment);
6866 StubCodeMark mark(this, stub_id);
6867 start = __ pc();
6868 __ enter();
6869
6870 const Register coeffs = c_rarg0;
6871
6872 const Register kyberConsts = r10;
6873 const Register result = r11;
6874
6875 // As above we process 256 sets of values in total i.e. 32 x
6876 // 8H quadwords. So, we can load, add and store the data in 3
6877 // groups of 11, 11 and 10 at a time i.e. we need to map sets
6878 // of 10 or 11 registers. A further constraint is that the
6879 // mapping needs to skip callee saves. So, we allocate the
6880 // register sequences using two 8 sequences, two 2 sequences
6881 // and two single registers.
6882 VSeq<8> vs1_1(0);
6883 VSeq<2> vs1_2(16);
6884 FloatRegister vs1_3 = v28;
6885 VSeq<8> vs2_1(18);
6886 VSeq<2> vs2_2(26);
6887 FloatRegister vs2_3 = v29;
6888
6889 // we also need a pair of corresponding constant sequences
6890
6891 VSeq<8> vc1_1(30, 0);
6892 VSeq<2> vc1_2(30, 0);
6893 FloatRegister vc1_3 = v30; // for kyber_q
6894
6895 VSeq<8> vc2_1(31, 0);
6896 VSeq<2> vc2_2(31, 0);
6897 FloatRegister vc2_3 = v31; // for kyberBarrettMultiplier
6898
6899 __ add(result, coeffs, 0);
6900 __ lea(kyberConsts,
6901 ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
6902
6903 // load q and the multiplier for the Barrett reduction
6904 __ add(kyberConsts, kyberConsts, 16);
6905 __ ldpq(vc1_3, vc2_3, kyberConsts);
6906
6907 for (int i = 0; i < 3; i++) {
6908 // load 80 or 88 coefficients
6909 vs_ldpq_post(vs1_1, coeffs);
6910 vs_ldpq_post(vs1_2, coeffs);
6911 if (i < 2) {
6912 __ ldr(vs1_3, __ Q, __ post(coeffs, 16));
6913 }
6914
6915 // vs2 <- (2 * vs1 * kyberBarrettMultiplier) >> 16
6916 vs_sqdmulh(vs2_1, __ T8H, vs1_1, vc2_1);
6917 vs_sqdmulh(vs2_2, __ T8H, vs1_2, vc2_2);
6918 if (i < 2) {
6919 __ sqdmulh(vs2_3, __ T8H, vs1_3, vc2_3);
6920 }
6921
6922 // vs2 <- (vs1 * kyberBarrettMultiplier) >> 26
6923 vs_sshr(vs2_1, __ T8H, vs2_1, 11);
6924 vs_sshr(vs2_2, __ T8H, vs2_2, 11);
6925 if (i < 2) {
6926 __ sshr(vs2_3, __ T8H, vs2_3, 11);
6927 }
6928
6929 // vs1 <- vs1 - vs2 * kyber_q
6930 vs_mlsv(vs1_1, __ T8H, vs2_1, vc1_1);
6931 vs_mlsv(vs1_2, __ T8H, vs2_2, vc1_2);
6932 if (i < 2) {
6933 __ mlsv(vs1_3, __ T8H, vs2_3, vc1_3);
6934 }
6935
6936 vs_stpq_post(vs1_1, result);
6937 vs_stpq_post(vs1_2, result);
6938 if (i < 2) {
6939 __ str(vs1_3, __ Q, __ post(result, 16));
6940 }
6941 }
6942
6943 __ leave(); // required for proper stackwalking of RuntimeStub frame
6944 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6945 __ ret(lr);
6946
6947 // record the stub entry and end
6948 store_archive_data(stub_id, start, __ pc());
6949
6950 return start;
6951 }
6952
6953
6954 // Dilithium-specific montmul helper routines that generate parallel
6955 // code for, respectively, a single 4x4s vector sequence montmul or
6956 // two such multiplies in a row.
6957
6958 // Perform 16 32-bit Montgomery multiplications in parallel
6959 void dilithium_montmul16(const VSeq<4>& va, const VSeq<4>& vb, const VSeq<4>& vc,
6960 const VSeq<4>& vtmp, const VSeq<2>& vq) {
6961 // Use the helper routine to schedule a 4x4S Montgomery multiply.
6962 // It will assert that the register use is valid
6963 vs_montmul4(va, vb, vc, __ T4S, vtmp, vq);
6964 }
6965
6966 // Perform 2x16 32-bit Montgomery multiplications in parallel
6967 void dilithium_montmul32(const VSeq<8>& va, const VSeq<8>& vb, const VSeq<8>& vc,
6968 const VSeq<4>& vtmp, const VSeq<2>& vq) {
6969 // Schedule two successive 4x4S multiplies via the montmul helper
6970 // on the front and back halves of va, vb and vc. The helper will
6971 // assert that the register use has no overlap conflicts on each
6972 // individual call but we also need to ensure that the necessary
6973 // disjoint/equality constraints are met across both calls.
6974
6975 // vb, vc, vtmp and vq must be disjoint. va must either be
6976 // disjoint from all other registers or equal vc
6977
6978 assert(vs_disjoint(vb, vc), "vb and vc overlap");
6979 assert(vs_disjoint(vb, vq), "vb and vq overlap");
6980 assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap");
6981
6982 assert(vs_disjoint(vc, vq), "vc and vq overlap");
6983 assert(vs_disjoint(vc, vtmp), "vc and vtmp overlap");
6984
6985 assert(vs_disjoint(vq, vtmp), "vq and vtmp overlap");
6986
6987 assert(vs_disjoint(va, vc) || vs_same(va, vc), "va and vc neither disjoint nor equal");
6988 assert(vs_disjoint(va, vb), "va and vb overlap");
6989 assert(vs_disjoint(va, vq), "va and vq overlap");
6990 assert(vs_disjoint(va, vtmp), "va and vtmp overlap");
6991
6992 // We multiply the front and back halves of each sequence 4 at a
6993 // time because
6994 //
6995 // 1) we are currently only able to get 4-way instruction
6996 // parallelism at best
6997 //
6998 // 2) we need registers for the constants in vq and temporary
6999 // scratch registers to hold intermediate results so vtmp can only
7000 // be a VSeq<4> which means we only have 4 scratch slots.
7001
7002 vs_montmul4(vs_front(va), vs_front(vb), vs_front(vc), __ T4S, vtmp, vq);
7003 vs_montmul4(vs_back(va), vs_back(vb), vs_back(vc), __ T4S, vtmp, vq);
7004 }
7005
7006 // Perform combined montmul then add/sub on 4x4S vectors.
7007 void dilithium_montmul16_sub_add(
7008 const VSeq<4>& va0, const VSeq<4>& va1, const VSeq<4>& vc,
7009 const VSeq<4>& vtmp, const VSeq<2>& vq) {
7010 // compute a = montmul(a1, c)
7011 dilithium_montmul16(vc, va1, vc, vtmp, vq);
7012 // ouptut a1 = a0 - a
7013 vs_subv(va1, __ T4S, va0, vc);
7014 // and a0 = a0 + a
7015 vs_addv(va0, __ T4S, va0, vc);
7016 }
7017
7018 // Perform combined add/sub then montmul on 4x4S vectors.
7019 void dilithium_sub_add_montmul16(
7020 const VSeq<4>& va0, const VSeq<4>& va1, const VSeq<4>& vb,
7021 const VSeq<4>& vtmp1, const VSeq<4>& vtmp2, const VSeq<2>& vq) {
7022 // compute c = a0 - a1
7023 vs_subv(vtmp1, __ T4S, va0, va1);
7024 // output a0 = a0 + a1
7025 vs_addv(va0, __ T4S, va0, va1);
7026 // output a1 = b montmul c
7027 dilithium_montmul16(va1, vtmp1, vb, vtmp2, vq);
7028 }
7029
7030 // At these levels, the indices that correspond to the 'j's (and 'j+l's)
7031 // in the Java implementation come in sequences of at least 8, so we
7032 // can use ldpq to collect the corresponding data into pairs of vector
7033 // registers.
7034 // We collect the coefficients corresponding to the 'j+l' indexes into
7035 // the vector registers v0-v7, the zetas into the vector registers v16-v23
7036 // then we do the (Montgomery) multiplications by the zetas in parallel
7037 // into v16-v23, load the coeffs corresponding to the 'j' indexes into
7038 // v0-v7, then do the additions into v24-v31 and the subtractions into
7039 // v0-v7 and finally save the results back to the coeffs array.
7040 void dilithiumNttLevel0_4(const Register dilithiumConsts,
7041 const Register coeffs, const Register zetas) {
7042 int c1 = 0;
7043 int c2 = 512;
7044 int startIncr;
7045 // don't use callee save registers v8 - v15
7046 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7047 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7048 VSeq<2> vq(30); // n.b. constants overlap vs3
7049 int offsets[4] = { 0, 32, 64, 96 };
7050
7051 for (int level = 0; level < 5; level++) {
7052 int c1Start = c1;
7053 int c2Start = c2;
7054 if (level == 3) {
7055 offsets[1] = 32;
7056 offsets[2] = 128;
7057 offsets[3] = 160;
7058 } else if (level == 4) {
7059 offsets[1] = 64;
7060 offsets[2] = 128;
7061 offsets[3] = 192;
7062 }
7063
7064 // For levels 1 - 4 we simply load 2 x 4 adjacent values at a
7065 // time at 4 different offsets and multiply them in order by the
7066 // next set of input values. So we employ indexed load and store
7067 // pair instructions with arrangement 4S.
7068 for (int i = 0; i < 4; i++) {
7069 // reload q and qinv
7070 vs_ldpq(vq, dilithiumConsts); // qInv, q
7071 // load 8x4S coefficients via second start pos == c2
7072 vs_ldpq_indexed(vs1, coeffs, c2Start, offsets);
7073 // load next 8x4S inputs == b
7074 vs_ldpq_post(vs2, zetas);
7075 // compute a == c2 * b mod MONT_Q
7076 dilithium_montmul32(vs2, vs1, vs2, vtmp, vq);
7077 // load 8x4s coefficients via first start pos == c1
7078 vs_ldpq_indexed(vs1, coeffs, c1Start, offsets);
7079 // compute a1 = c1 + a
7080 vs_addv(vs3, __ T4S, vs1, vs2);
7081 // compute a2 = c1 - a
7082 vs_subv(vs1, __ T4S, vs1, vs2);
7083 // output a1 and a2
7084 vs_stpq_indexed(vs3, coeffs, c1Start, offsets);
7085 vs_stpq_indexed(vs1, coeffs, c2Start, offsets);
7086
7087 int k = 4 * level + i;
7088
7089 if (k > 7) {
7090 startIncr = 256;
7091 } else if (k == 5) {
7092 startIncr = 384;
7093 } else {
7094 startIncr = 128;
7095 }
7096
7097 c1Start += startIncr;
7098 c2Start += startIncr;
7099 }
7100
7101 c2 /= 2;
7102 }
7103 }
7104
7105 // Dilithium NTT function except for the final "normalization" to |coeff| < Q.
7106 // Implements the method
7107 // static int implDilithiumAlmostNtt(int[] coeffs, int zetas[]) {}
7108 // of the Java class sun.security.provider
7109 //
7110 // coeffs (int[256]) = c_rarg0
7111 // zetas (int[256]) = c_rarg1
7112 address generate_dilithiumAlmostNtt() {
7113 StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id;
7114 int entry_count = StubInfo::entry_count(stub_id);
7115 assert(entry_count == 1, "sanity check");
7116 address start = load_archive_data(stub_id);
7117 if (start != nullptr) {
7118 return start;
7119 }
7120 __ align(CodeEntryAlignment);
7121 StubCodeMark mark(this, stub_id);
7122 start = __ pc();
7123 __ enter();
7124
7125 const Register coeffs = c_rarg0;
7126 const Register zetas = c_rarg1;
7127
7128 const Register tmpAddr = r9;
7129 const Register dilithiumConsts = r10;
7130 const Register result = r11;
7131 // don't use callee save registers v8 - v15
7132 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7133 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7134 VSeq<2> vq(30); // n.b. constants overlap vs3
7135 int offsets[4] = { 0, 32, 64, 96};
7136 int offsets1[8] = { 16, 48, 80, 112, 144, 176, 208, 240 };
7137 int offsets2[8] = { 0, 32, 64, 96, 128, 160, 192, 224 };
7138 __ add(result, coeffs, 0);
7139 __ lea(dilithiumConsts,
7140 ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts));
7141
7142 // Each level represents one iteration of the outer for loop of the Java version.
7143
7144 // level 0-4
7145 dilithiumNttLevel0_4(dilithiumConsts, coeffs, zetas);
7146
7147 // level 5
7148
7149 // At level 5 the coefficients we need to combine with the zetas
7150 // are grouped in memory in blocks of size 4. So, for both sets of
7151 // coefficients we load 4 adjacent values at 8 different offsets
7152 // using an indexed ldr with register variant Q and multiply them
7153 // in sequence order by the next set of inputs. Likewise we store
7154 // the results using an indexed str with register variant Q.
7155 for (int i = 0; i < 1024; i += 256) {
7156 // reload constants q, qinv each iteration as they get clobbered later
7157 vs_ldpq(vq, dilithiumConsts); // qInv, q
7158 // load 32 (8x4S) coefficients via first offsets = c1
7159 vs_ldr_indexed(vs1, __ Q, coeffs, i, offsets1);
7160 // load next 32 (8x4S) inputs = b
7161 vs_ldpq_post(vs2, zetas);
7162 // a = b montul c1
7163 dilithium_montmul32(vs2, vs1, vs2, vtmp, vq);
7164 // load 32 (8x4S) coefficients via second offsets = c2
7165 vs_ldr_indexed(vs1, __ Q, coeffs, i, offsets2);
7166 // add/sub with result of multiply
7167 vs_addv(vs3, __ T4S, vs1, vs2); // a1 = a - c2
7168 vs_subv(vs1, __ T4S, vs1, vs2); // a0 = a + c1
7169 // write back new coefficients using same offsets
7170 vs_str_indexed(vs3, __ Q, coeffs, i, offsets2);
7171 vs_str_indexed(vs1, __ Q, coeffs, i, offsets1);
7172 }
7173
7174 // level 6
7175 // At level 6 the coefficients we need to combine with the zetas
7176 // are grouped in memory in pairs, the first two being montmul
7177 // inputs and the second add/sub inputs. We can still implement
7178 // the montmul+sub+add using 4-way parallelism but only if we
7179 // combine the coefficients with the zetas 16 at a time. We load 8
7180 // adjacent values at 4 different offsets using an ld2 load with
7181 // arrangement 2D. That interleaves the lower and upper halves of
7182 // each pair of quadwords into successive vector registers. We
7183 // then need to montmul the 4 even elements of the coefficients
7184 // register sequence by the zetas in order and then add/sub the 4
7185 // odd elements of the coefficients register sequence. We use an
7186 // equivalent st2 operation to store the results back into memory
7187 // de-interleaved.
7188 for (int i = 0; i < 1024; i += 128) {
7189 // reload constants q, qinv each iteration as they get clobbered later
7190 vs_ldpq(vq, dilithiumConsts); // qInv, q
7191 // load interleaved 16 (4x2D) coefficients via offsets
7192 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, i, offsets);
7193 // load next 16 (4x4S) inputs
7194 vs_ldpq_post(vs_front(vs2), zetas);
7195 // mont multiply odd elements of vs1 by vs2 and add/sub into odds/evens
7196 dilithium_montmul16_sub_add(vs_even(vs1), vs_odd(vs1),
7197 vs_front(vs2), vtmp, vq);
7198 // store interleaved 16 (4x2D) coefficients via offsets
7199 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, i, offsets);
7200 }
7201
7202 // level 7
7203 // At level 7 the coefficients we need to combine with the zetas
7204 // occur singly with montmul inputs alternating with add/sub
7205 // inputs. Once again we can use 4-way parallelism to combine 16
7206 // zetas at a time. However, we have to load 8 adjacent values at
7207 // 4 different offsets using an ld2 load with arrangement 4S. That
7208 // interleaves the odd words of each pair into one
7209 // coefficients vector register and the even words of the pair
7210 // into the next register. We then need to montmul the 4 even
7211 // elements of the coefficients register sequence by the zetas in
7212 // order and then add/sub the 4 odd elements of the coefficients
7213 // register sequence. We use an equivalent st2 operation to store
7214 // the results back into memory de-interleaved.
7215
7216 for (int i = 0; i < 1024; i += 128) {
7217 // reload constants q, qinv each iteration as they get clobbered later
7218 vs_ldpq(vq, dilithiumConsts); // qInv, q
7219 // load interleaved 16 (4x4S) coefficients via offsets
7220 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, i, offsets);
7221 // load next 16 (4x4S) inputs
7222 vs_ldpq_post(vs_front(vs2), zetas);
7223 // mont multiply odd elements of vs1 by vs2 and add/sub into odds/evens
7224 dilithium_montmul16_sub_add(vs_even(vs1), vs_odd(vs1),
7225 vs_front(vs2), vtmp, vq);
7226 // store interleaved 16 (4x4S) coefficients via offsets
7227 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, i, offsets);
7228 }
7229 __ leave(); // required for proper stackwalking of RuntimeStub frame
7230 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
7231 __ ret(lr);
7232
7233 // record the stub entry and end
7234 store_archive_data(stub_id, start, __ pc());
7235
7236 return start;
7237 }
7238
7239 // At these levels, the indices that correspond to the 'j's (and 'j+l's)
7240 // in the Java implementation come in sequences of at least 8, so we
7241 // can use ldpq to collect the corresponding data into pairs of vector
7242 // registers
7243 // We collect the coefficients that correspond to the 'j's into vs1
7244 // the coefficiets that correspond to the 'j+l's into vs2 then
7245 // do the additions into vs3 and the subtractions into vs1 then
7246 // save the result of the additions, load the zetas into vs2
7247 // do the (Montgomery) multiplications by zeta in parallel into vs2
7248 // finally save the results back to the coeffs array
7249 void dilithiumInverseNttLevel3_7(const Register dilithiumConsts,
7250 const Register coeffs, const Register zetas) {
7251 int c1 = 0;
7252 int c2 = 32;
7253 int startIncr;
7254 int offsets[4];
7255 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7256 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7257 VSeq<2> vq(30); // n.b. constants overlap vs3
7258
7259 offsets[0] = 0;
7260
7261 for (int level = 3; level < 8; level++) {
7262 int c1Start = c1;
7263 int c2Start = c2;
7264 if (level == 3) {
7265 offsets[1] = 64;
7266 offsets[2] = 128;
7267 offsets[3] = 192;
7268 } else if (level == 4) {
7269 offsets[1] = 32;
7270 offsets[2] = 128;
7271 offsets[3] = 160;
7272 } else {
7273 offsets[1] = 32;
7274 offsets[2] = 64;
7275 offsets[3] = 96;
7276 }
7277
7278 // For levels 3 - 7 we simply load 2 x 4 adjacent values at a
7279 // time at 4 different offsets and multiply them in order by the
7280 // next set of input values. So we employ indexed load and store
7281 // pair instructions with arrangement 4S.
7282 for (int i = 0; i < 4; i++) {
7283 // load v1 32 (8x4S) coefficients relative to first start index
7284 vs_ldpq_indexed(vs1, coeffs, c1Start, offsets);
7285 // load v2 32 (8x4S) coefficients relative to second start index
7286 vs_ldpq_indexed(vs2, coeffs, c2Start, offsets);
7287 // a0 = v1 + v2 -- n.b. clobbers vqs
7288 vs_addv(vs3, __ T4S, vs1, vs2);
7289 // a1 = v1 - v2
7290 vs_subv(vs1, __ T4S, vs1, vs2);
7291 // save a1 relative to first start index
7292 vs_stpq_indexed(vs3, coeffs, c1Start, offsets);
7293 // load constants q, qinv each iteration as they get clobbered above
7294 vs_ldpq(vq, dilithiumConsts); // qInv, q
7295 // load b next 32 (8x4S) inputs
7296 vs_ldpq_post(vs2, zetas);
7297 // a = a1 montmul b
7298 dilithium_montmul32(vs2, vs1, vs2, vtmp, vq);
7299 // save a relative to second start index
7300 vs_stpq_indexed(vs2, coeffs, c2Start, offsets);
7301
7302 int k = 4 * level + i;
7303
7304 if (k < 24) {
7305 startIncr = 256;
7306 } else if (k == 25) {
7307 startIncr = 384;
7308 } else {
7309 startIncr = 128;
7310 }
7311
7312 c1Start += startIncr;
7313 c2Start += startIncr;
7314 }
7315
7316 c2 *= 2;
7317 }
7318 }
7319
7320 // Dilithium Inverse NTT function except the final mod Q division by 2^256.
7321 // Implements the method
7322 // static int implDilithiumAlmostInverseNtt(int[] coeffs, int[] zetas) {} of
7323 // the sun.security.provider.ML_DSA class.
7324 //
7325 // coeffs (int[256]) = c_rarg0
7326 // zetas (int[256]) = c_rarg1
7327 address generate_dilithiumAlmostInverseNtt() {
7328 StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id;
7329 int entry_count = StubInfo::entry_count(stub_id);
7330 assert(entry_count == 1, "sanity check");
7331 address start = load_archive_data(stub_id);
7332 if (start != nullptr) {
7333 return start;
7334 }
7335 __ align(CodeEntryAlignment);
7336 StubCodeMark mark(this, stub_id);
7337 start = __ pc();
7338 __ enter();
7339
7340 const Register coeffs = c_rarg0;
7341 const Register zetas = c_rarg1;
7342
7343 const Register tmpAddr = r9;
7344 const Register dilithiumConsts = r10;
7345 const Register result = r11;
7346 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7347 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7348 VSeq<2> vq(30); // n.b. constants overlap vs3
7349 int offsets[4] = { 0, 32, 64, 96 };
7350 int offsets1[8] = { 0, 32, 64, 96, 128, 160, 192, 224 };
7351 int offsets2[8] = { 16, 48, 80, 112, 144, 176, 208, 240 };
7352
7353 __ add(result, coeffs, 0);
7354 __ lea(dilithiumConsts,
7355 ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts));
7356
7357 // Each level represents one iteration of the outer for loop of the Java version
7358
7359 // level 0
7360 // At level 0 we need to interleave adjacent quartets of
7361 // coefficients before we multiply and add/sub by the next 16
7362 // zetas just as we did for level 7 in the multiply code. So we
7363 // load and store the values using an ld2/st2 with arrangement 4S.
7364 for (int i = 0; i < 1024; i += 128) {
7365 // load constants q, qinv
7366 // n.b. this can be moved out of the loop as they do not get
7367 // clobbered by first two loops
7368 vs_ldpq(vq, dilithiumConsts); // qInv, q
7369 // a0/a1 load interleaved 32 (8x4S) coefficients
7370 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, i, offsets);
7371 // b load next 32 (8x4S) inputs
7372 vs_ldpq_post(vs_front(vs2), zetas);
7373 // compute in parallel (a0, a1) = (a0 + a1, (a0 - a1) montmul b)
7374 // n.b. second half of vs2 provides temporary register storage
7375 dilithium_sub_add_montmul16(vs_even(vs1), vs_odd(vs1),
7376 vs_front(vs2), vs_back(vs2), vtmp, vq);
7377 // a0/a1 store interleaved 32 (8x4S) coefficients
7378 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, i, offsets);
7379 }
7380
7381 // level 1
7382 // At level 1 we need to interleave pairs of adjacent pairs of
7383 // coefficients before we multiply by the next 16 zetas just as we
7384 // did for level 6 in the multiply code. So we load and store the
7385 // values an ld2/st2 with arrangement 2D.
7386 for (int i = 0; i < 1024; i += 128) {
7387 // a0/a1 load interleaved 32 (8x2D) coefficients
7388 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, i, offsets);
7389 // b load next 16 (4x4S) inputs
7390 vs_ldpq_post(vs_front(vs2), zetas);
7391 // compute in parallel (a0, a1) = (a0 + a1, (a0 - a1) montmul b)
7392 // n.b. second half of vs2 provides temporary register storage
7393 dilithium_sub_add_montmul16(vs_even(vs1), vs_odd(vs1),
7394 vs_front(vs2), vs_back(vs2), vtmp, vq);
7395 // a0/a1 store interleaved 32 (8x2D) coefficients
7396 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, i, offsets);
7397 }
7398
7399 // level 2
7400 // At level 2 coefficients come in blocks of 4. So, we load 4
7401 // adjacent coefficients at 8 distinct offsets for both the first
7402 // and second coefficient sequences, using an ldr with register
7403 // variant Q then combine them with next set of 32 zetas. Likewise
7404 // we store the results using an str with register variant Q.
7405 for (int i = 0; i < 1024; i += 256) {
7406 // c0 load 32 (8x4S) coefficients via first offsets
7407 vs_ldr_indexed(vs1, __ Q, coeffs, i, offsets1);
7408 // c1 load 32 (8x4S) coefficients via second offsets
7409 vs_ldr_indexed(vs2, __ Q, coeffs, i, offsets2);
7410 // a0 = c0 + c1 n.b. clobbers vq which overlaps vs3
7411 vs_addv(vs3, __ T4S, vs1, vs2);
7412 // c = c0 - c1
7413 vs_subv(vs1, __ T4S, vs1, vs2);
7414 // store a0 32 (8x4S) coefficients via first offsets
7415 vs_str_indexed(vs3, __ Q, coeffs, i, offsets1);
7416 // b load 32 (8x4S) next inputs
7417 vs_ldpq_post(vs2, zetas);
7418 // reload constants q, qinv -- they were clobbered earlier
7419 vs_ldpq(vq, dilithiumConsts); // qInv, q
7420 // compute a1 = b montmul c
7421 dilithium_montmul32(vs2, vs1, vs2, vtmp, vq);
7422 // store a1 32 (8x4S) coefficients via second offsets
7423 vs_str_indexed(vs2, __ Q, coeffs, i, offsets2);
7424 }
7425
7426 // level 3-7
7427 dilithiumInverseNttLevel3_7(dilithiumConsts, coeffs, zetas);
7428
7429 __ leave(); // required for proper stackwalking of RuntimeStub frame
7430 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
7431 __ ret(lr);
7432
7433 // record the stub entry and end
7434 store_archive_data(stub_id, start, __ pc());
7435
7436 return start;
7437 }
7438
7439 // Dilithium multiply polynomials in the NTT domain.
7440 // Straightforward implementation of the method
7441 // static int implDilithiumNttMult(
7442 // int[] product, int[] coeffs1, int[] coeffs2) {}
7443 // of the sun.security.provider.ML_DSA class.
7444 //
7445 // result (int[256]) = c_rarg0
7446 // poly1 (int[256]) = c_rarg1
7447 // poly2 (int[256]) = c_rarg2
7448 address generate_dilithiumNttMult() {
7449 StubId stub_id = StubId::stubgen_dilithiumNttMult_id;
7450 int entry_count = StubInfo::entry_count(stub_id);
7451 assert(entry_count == 1, "sanity check");
7452 address start = load_archive_data(stub_id);
7453 if (start != nullptr) {
7454 return start;
7455 }
7456 __ align(CodeEntryAlignment);
7457 StubCodeMark mark(this, stub_id);
7458 start = __ pc();
7459 __ enter();
7460
7461 Label L_loop;
7462
7463 const Register result = c_rarg0;
7464 const Register poly1 = c_rarg1;
7465 const Register poly2 = c_rarg2;
7466
7467 const Register dilithiumConsts = r10;
7468 const Register len = r11;
7469
7470 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7471 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7472 VSeq<2> vq(30); // n.b. constants overlap vs3
7473 VSeq<8> vrsquare(29, 0); // for montmul by constant RSQUARE
7474
7475 __ lea(dilithiumConsts,
7476 ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts));
7477
7478 // load constants q, qinv
7479 vs_ldpq(vq, dilithiumConsts); // qInv, q
7480 // load constant rSquare into v29
7481 __ ldr(v29, __ Q, Address(dilithiumConsts, 48)); // rSquare
7482
7483 __ mov(len, zr);
7484 __ add(len, len, 1024);
7485
7486 __ BIND(L_loop);
7487
7488 // b load 32 (8x4S) next inputs from poly1
7489 vs_ldpq_post(vs1, poly1);
7490 // c load 32 (8x4S) next inputs from poly2
7491 vs_ldpq_post(vs2, poly2);
7492 // compute a = b montmul c
7493 dilithium_montmul32(vs2, vs1, vs2, vtmp, vq);
7494 // compute a = rsquare montmul a
7495 dilithium_montmul32(vs2, vrsquare, vs2, vtmp, vq);
7496 // save a 32 (8x4S) results
7497 vs_stpq_post(vs2, result);
7498
7499 __ sub(len, len, 128);
7500 __ cmp(len, (u1)128);
7501 __ br(Assembler::GE, L_loop);
7502
7503 __ leave(); // required for proper stackwalking of RuntimeStub frame
7504 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
7505 __ ret(lr);
7506
7507 // record the stub entry and end
7508 store_archive_data(stub_id, start, __ pc());
7509
7510 return start;
7511 }
7512
7513 // Dilithium Montgomery multiply an array by a constant.
7514 // A straightforward implementation of the method
7515 // static int implDilithiumMontMulByConstant(int[] coeffs, int constant) {}
7516 // of the sun.security.provider.ML_DSA class
7517 //
7518 // coeffs (int[256]) = c_rarg0
7519 // constant (int) = c_rarg1
7520 address generate_dilithiumMontMulByConstant() {
7521 StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id;
7522 int entry_count = StubInfo::entry_count(stub_id);
7523 assert(entry_count == 1, "sanity check");
7524 address start = load_archive_data(stub_id);
7525 if (start != nullptr) {
7526 return start;
7527 }
7528 __ align(CodeEntryAlignment);
7529 StubCodeMark mark(this, stub_id);
7530 start = __ pc();
7531 __ enter();
7532
7533 Label L_loop;
7534
7535 const Register coeffs = c_rarg0;
7536 const Register constant = c_rarg1;
7537
7538 const Register dilithiumConsts = r10;
7539 const Register result = r11;
7540 const Register len = r12;
7541
7542 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7543 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7544 VSeq<2> vq(30); // n.b. constants overlap vs3
7545 VSeq<8> vconst(29, 0); // for montmul by constant
7546
7547 // results track inputs
7548 __ add(result, coeffs, 0);
7549 __ lea(dilithiumConsts,
7550 ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts));
7551
7552 // load constants q, qinv -- they do not get clobbered by first two loops
7553 vs_ldpq(vq, dilithiumConsts); // qInv, q
7554 // copy caller supplied constant across vconst
7555 __ dup(vconst[0], __ T4S, constant);
7556 __ mov(len, zr);
7557 __ add(len, len, 1024);
7558
7559 __ BIND(L_loop);
7560
7561 // load next 32 inputs
7562 vs_ldpq_post(vs2, coeffs);
7563 // mont mul by constant
7564 dilithium_montmul32(vs2, vconst, vs2, vtmp, vq);
7565 // write next 32 results
7566 vs_stpq_post(vs2, result);
7567
7568 __ sub(len, len, 128);
7569 __ cmp(len, (u1)128);
7570 __ br(Assembler::GE, L_loop);
7571
7572 __ leave(); // required for proper stackwalking of RuntimeStub frame
7573 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
7574 __ ret(lr);
7575
7576 // record the stub entry and end
7577 store_archive_data(stub_id, start, __ pc());
7578
7579 return start;
7580 }
7581
7582 // Dilithium decompose poly.
7583 // Implements the method
7584 // static int implDilithiumDecomposePoly(int[] input, int[] lowPart, int[] highPart,
7585 // int twoGamma2, int multiplier) {
7586 // of the sun.security.provider.ML_DSA class
7587 //
7588 // input (int[256]) = c_rarg0
7589 // lowPart (int[256]) = c_rarg1
7590 // highPart (int[256]) = c_rarg2
7591 // twoGamma2 (int) = c_rarg3
7592 // multiplier (int) = c_rarg4
7593 address generate_dilithiumDecomposePoly() {
7594 StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id;
7595 int entry_count = StubInfo::entry_count(stub_id);
7596 assert(entry_count == 1, "sanity check");
7597 address start = load_archive_data(stub_id);
7598 if (start != nullptr) {
7599 return start;
7600 }
7601 __ align(CodeEntryAlignment);
7602 StubCodeMark mark(this, stub_id);
7603 start = __ pc();
7604 Label L_loop;
7605
7606 const Register input = c_rarg0;
7607 const Register lowPart = c_rarg1;
7608 const Register highPart = c_rarg2;
7609 const Register twoGamma2 = c_rarg3;
7610 const Register multiplier = c_rarg4;
7611
7612 const Register len = r9;
7613 const Register dilithiumConsts = r10;
7614 const Register tmp = r11;
7615
7616 // 6 independent sets of 4x4s values
7617 VSeq<4> vs1(0), vs2(4), vs3(8);
7618 VSeq<4> vs4(12), vs5(16), vtmp(20);
7619
7620 // 7 constants for cross-multiplying
7621 VSeq<4> one(25, 0);
7622 VSeq<4> qminus1(26, 0);
7623 VSeq<4> g2(27, 0);
7624 VSeq<4> twog2(28, 0);
7625 VSeq<4> mult(29, 0);
7626 VSeq<4> q(30, 0);
7627 VSeq<4> qadd(31, 0);
7628
7629 __ enter();
7630
7631 __ lea(dilithiumConsts,
7632 ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts));
7633
7634 // save callee-saved registers
7635 __ stpd(v8, v9, __ pre(sp, -64));
7636 __ stpd(v10, v11, Address(sp, 16));
7637 __ stpd(v12, v13, Address(sp, 32));
7638 __ stpd(v14, v15, Address(sp, 48));
7639
7640 // populate constant registers
7641 __ mov(tmp, zr);
7642 __ add(tmp, tmp, 1);
7643 __ dup(one[0], __ T4S, tmp); // 1
7644 __ ldr(q[0], __ Q, Address(dilithiumConsts, 16)); // q
7645 __ ldr(qadd[0], __ Q, Address(dilithiumConsts, 64)); // addend for mod q reduce
7646 __ dup(twog2[0], __ T4S, twoGamma2); // 2 * gamma2
7647 __ dup(mult[0], __ T4S, multiplier); // multiplier for mod 2 * gamma reduce
7648 __ subv(qminus1[0], __ T4S, v30, v25); // q - 1
7649 __ sshr(g2[0], __ T4S, v28, 1); // gamma2
7650
7651 __ mov(len, zr);
7652 __ add(len, len, 1024);
7653
7654 __ BIND(L_loop);
7655
7656 // load next 4x4S inputs interleaved: rplus --> vs1
7657 __ ld4(vs1[0], vs1[1], vs1[2], vs1[3], __ T4S, __ post(input, 64));
7658
7659 // rplus = rplus - ((rplus + qadd) >> 23) * q
7660 vs_addv(vtmp, __ T4S, vs1, qadd);
7661 vs_sshr(vtmp, __ T4S, vtmp, 23);
7662 vs_mulv(vtmp, __ T4S, vtmp, q);
7663 vs_subv(vs1, __ T4S, vs1, vtmp);
7664
7665 // rplus = rplus + ((rplus >> 31) & dilithium_q);
7666 vs_sshr(vtmp, __ T4S, vs1, 31);
7667 vs_andr(vtmp, vtmp, q);
7668 vs_addv(vs1, __ T4S, vs1, vtmp);
7669
7670 // quotient --> vs2
7671 // int quotient = (rplus * multiplier) >> 22;
7672 vs_mulv(vtmp, __ T4S, vs1, mult);
7673 vs_sshr(vs2, __ T4S, vtmp, 22);
7674
7675 // r0 --> vs3
7676 // int r0 = rplus - quotient * twoGamma2;
7677 vs_mulv(vtmp, __ T4S, vs2, twog2);
7678 vs_subv(vs3, __ T4S, vs1, vtmp);
7679
7680 // mask --> vs4
7681 // int mask = (twoGamma2 - r0) >> 22;
7682 vs_subv(vtmp, __ T4S, twog2, vs3);
7683 vs_sshr(vs4, __ T4S, vtmp, 22);
7684
7685 // r0 -= (mask & twoGamma2);
7686 vs_andr(vtmp, vs4, twog2);
7687 vs_subv(vs3, __ T4S, vs3, vtmp);
7688
7689 // quotient += (mask & 1);
7690 vs_andr(vtmp, vs4, one);
7691 vs_addv(vs2, __ T4S, vs2, vtmp);
7692
7693 // mask = (twoGamma2 / 2 - r0) >> 31;
7694 vs_subv(vtmp, __ T4S, g2, vs3);
7695 vs_sshr(vs4, __ T4S, vtmp, 31);
7696
7697 // r0 -= (mask & twoGamma2);
7698 vs_andr(vtmp, vs4, twog2);
7699 vs_subv(vs3, __ T4S, vs3, vtmp);
7700
7701 // quotient += (mask & 1);
7702 vs_andr(vtmp, vs4, one);
7703 vs_addv(vs2, __ T4S, vs2, vtmp);
7704
7705 // r1 --> vs5
7706 // int r1 = rplus - r0 - (dilithium_q - 1);
7707 vs_subv(vtmp, __ T4S, vs1, vs3);
7708 vs_subv(vs5, __ T4S, vtmp, qminus1);
7709
7710 // r1 --> vs1 (overwriting rplus)
7711 // r1 = (r1 | (-r1)) >> 31; // 0 if rplus - r0 == (dilithium_q - 1), -1 otherwise
7712 vs_negr(vtmp, __ T4S, vs5);
7713 vs_orr(vtmp, vs5, vtmp);
7714 vs_sshr(vs1, __ T4S, vtmp, 31);
7715
7716 // r0 += ~r1;
7717 vs_notr(vtmp, vs1);
7718 vs_addv(vs3, __ T4S, vs3, vtmp);
7719
7720 // r1 = r1 & quotient;
7721 vs_andr(vs1, vs2, vs1);
7722
7723 // store results interleaved
7724 // lowPart[m] = r0;
7725 // highPart[m] = r1;
7726 __ st4(vs3[0], vs3[1], vs3[2], vs3[3], __ T4S, __ post(lowPart, 64));
7727 __ st4(vs1[0], vs1[1], vs1[2], vs1[3], __ T4S, __ post(highPart, 64));
7728
7729 __ sub(len, len, 64);
7730 __ cmp(len, (u1)64);
7731 __ br(Assembler::GE, L_loop);
7732
7733 // restore callee-saved vector registers
7734 __ ldpd(v14, v15, Address(sp, 48));
7735 __ ldpd(v12, v13, Address(sp, 32));
7736 __ ldpd(v10, v11, Address(sp, 16));
7737 __ ldpd(v8, v9, __ post(sp, 64));
7738
7739 __ leave(); // required for proper stackwalking of RuntimeStub frame
7740 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
7741 __ ret(lr);
7742
7743 // record the stub entry and end
7744 store_archive_data(stub_id, start, __ pc());
7745
7746 return start;
7747 }
7748
7749 static constexpr int montMulP256Shift1 = 12; // 64 - bits per limb
7750 static constexpr int montMulP256Shift2 = 52; // bits per limb
7751 // stack space needed for carry computation
7752 static constexpr int cDataSize = 6 * BytesPerLong;
7753 // stack space needed for data computed by the neon side
7754 static constexpr int mulDataSize = 16 * BytesPerLong;
7755
7756
7757 // Subroutine used by the 52 x 52 bit multiplication algorithm in
7758 // generate_intpoly_montgomeryMult_P256().
7759 // This function computes partial results of eight 52 x 52 bit multiplications,
7760 // where the multiplicands are stored as 64-bit values, specifically
7761 // (b_0, b_1, b_2, b_3) * (a_3, a_4). (The 4 calls to this function
7762 // together provide the results of these limb-multiplications.)
7763 // Calls to this function accept either the low 32 bits or high 20 bits
7764 // of each b_i packed into bs in ascending order. a_3 and a_4 are packed
7765 // into successive 64 bit elements of as. lane selects the low 32 or high
7766 // 20 bits of each a_j value. So four calls with the appropriate parameters
7767 // will produce the 64-bit low32 * low32, low32 * high20, high20 * low32,
7768 // high20 * high20 values in the output register sequences vs. The
7769 // 64-bit partial products are returned in vs in ascending order:
7770 // vs[0] = (b_0*a_3, b_1*a_3) . . . vs[3] = (b_2*a_4, b_3*a_4)
7771
7772 void neon_partial_mult_64(const VSeq<4>& vs, FloatRegister bs, FloatRegister as, int lane_lo) {
7773 __ umullv(vs[0], __ T2D, bs, __ T2S, as, __ S, lane_lo);
7774 __ umull2v(vs[1], __ T2D, bs, __ T4S, as, __ S, lane_lo);
7775 __ umullv(vs[2], __ T2D, bs, __ T2S, as, __ S, lane_lo + 2);
7776 __ umull2v(vs[3], __ T2D, bs, __ T4S, as, __ S, lane_lo + 2);
7777 }
7778
7779 // Subroutine used by the generate_intpoly_montgomeryMult_P256() function
7780 // to compute the result of a 52 x 52 bit multiplications where the
7781 // multiplicands, a and b are available as 64-bit values.
7782 // The result is going to two 64-bit registers lo (least significant 52 bits)
7783 // and hi (most significant 52 bits).
7784 void gpr_partial_mult_52(Register a, Register b, Register hi, Register lo,
7785 Register mask) {
7786 // compute 104-bit (40 + 64) full product
7787 __ umulh(hi, a, b);
7788 __ mul(lo, a, b);
7789 // combine 40 + 12 bits into hi result
7790 // on certain implementations of aarch64 (e.g. apple M1) replacing extr()
7791 // with the following equivalent instruction sequence the performance
7792 // improves slightly (despite it is two instructions longer and needs
7793 // an additional register)
7794 // __ lsl(hi, hi, montMulP256Shift1);
7795 // __ lsr(tmp, lo, montMulP256Shift2);
7796 // __ orr(hi, hi, tmp);
7797 __ extr(hi, hi, lo, montMulP256Shift2);
7798 // mask off 52 bits of lo result
7799 __ andr(lo, lo, mask);
7800 }
7801
7802 // This assembly follows the Java code in MontgomeryIntegerPolynomial256.mult()
7803 // quite closely. The main difference is that the computations done with the
7804 // last two limbs of `a` are done using Neon registers. This allows us to take
7805 // advantage of both the Neon registers and GPRs simultaneously.
7806 // It is also worth noting that since Neon does not support 64 bit
7807 // multiplication, we split each 64 bit value into lower and upper halves
7808 // and use the "schoolbook" multiplication algorithm.
7809 address generate_intpoly_montgomeryMult_P256() {
7810 assert(UseIntPolyIntrinsics, "what are we doing here?");
7811 StubId stub_id = StubId::stubgen_intpoly_montgomeryMult_P256_id;
7812 int entry_count = StubInfo::entry_count(stub_id);
7813 assert(entry_count == 1, "sanity check");
7814 address start = load_archive_data(stub_id);
7815 if (start != nullptr) {
7816 return start;
7817 }
7818 __ align(CodeEntryAlignment);
7819 StubCodeMark mark(this, stub_id);
7820 start = __ pc();
7821 __ enter();
7822
7823 // Registers that are used throughout entire routine
7824 const Register a = c_rarg0;
7825 const Register b = c_rarg1;
7826 const Register result = c_rarg2;
7827
7828 RegSet regs = RegSet::range(r0, r28) - rscratch1 - rscratch2
7829 - r16 - r17 - r18_tls - a - b - result;
7830
7831 auto common_regs = regs.begin();
7832 Register limb_mask = *common_regs++,
7833 c_ptr = *common_regs++,
7834 mod_0 = *common_regs++,
7835 mod_1 = *common_regs++,
7836 mod_3 = *common_regs++,
7837 mod_4 = *common_regs++,
7838 b_0 = *common_regs++,
7839 b_1 = *common_regs++,
7840 b_2 = *common_regs++,
7841 b_3 = *common_regs++,
7842 b_4 = *common_regs++;
7843
7844 FloatRegSet floatRegs = FloatRegSet::range(v0, v31)
7845 - FloatRegSet::range(v8, v15) // Caller saved vectors
7846 - FloatRegSet::range(v16, v31); // Manually-allocated vectors
7847
7848 auto common_vectors = floatRegs.begin();
7849 FloatRegister limb_mask_vec = *common_vectors++,
7850 b_lows = *common_vectors++,
7851 b_highs = *common_vectors++,
7852 a_vals = *common_vectors++;
7853
7854 // Push callee saved registers on to the stack
7855 RegSet callee_saved = RegSet::range(r19, r28);
7856 __ push(callee_saved, sp);
7857
7858 // Allocate space on the stack for carry values
7859 __ sub(sp, sp, cDataSize);
7860 __ mov(c_ptr, sp);
7861
7862 // Calculate (52-bit) limb masks for both gpr and vector registers
7863 __ mov(limb_mask, -UCONST64(1) >> montMulP256Shift1);
7864 __ dup(limb_mask_vec, __ T2D, limb_mask);
7865
7866 //Load input arrays and modulus
7867 Register a_ptr = *common_regs++, mod_ptr = *common_regs++;
7868 // skip 3 limbs so a_ptr addresses trailing pair {a3, a4}
7869 __ add(a_ptr, a, 3 * BytesPerLong);
7870 __ lea(mod_ptr, ExternalAddress((address)_modulus_P256));
7871 __ ldr(b_0, Address(b));
7872 __ ldr(b_1, Address(b, BytesPerLong));
7873 __ ldr(b_2, Address(b, 2 * BytesPerLong));
7874 __ ldr(b_3, Address(b, 3 * BytesPerLong));
7875 __ ldr(b_4, Address(b, 4 * BytesPerLong));
7876 __ ldr(mod_0, __ post(mod_ptr, BytesPerLong));
7877 __ ldr(mod_1, __ post(mod_ptr, BytesPerLong));
7878 __ ldr(mod_3, __ post(mod_ptr, BytesPerLong));
7879 __ ldr(mod_4, mod_ptr);
7880 __ ld1(a_vals, __ T2D, a_ptr);
7881 // use an interleaved load to group low 32 bits and high 20 bits
7882 // of 4 successive b values into two vector registers
7883 // n.b. these are the same inputs as the ones in b_0 ... b4
7884 __ ld2(b_lows, b_highs, __ T4S, b);
7885 common_regs = common_regs.remaining()
7886 + a_ptr + mod_ptr;
7887 a_ptr = mod_ptr = noreg;
7888
7889 //Regs used throughout the main "loop", which is partially unrolled here
7890 Register high = *common_regs++,
7891 low = *common_regs++,
7892 mul_ptr = *common_regs++,
7893 mod_high = *common_regs++,
7894 mod_low = *common_regs++,
7895 a_i = *common_regs++,
7896 c_i = *common_regs++,
7897 tmp = *common_regs++,
7898 n = *common_regs++;
7899
7900 // vector sequences used to compute and combine partial products of
7901 // b_i * a_j for i = {0,1,2,3} j = {3,4}
7902 VSeq<4> A(16);
7903 VSeq<4> B(20);
7904 VSeq<4> C(24);
7905 VSeq<4> D(28);
7906
7907
7908 // neon and gpr computations are interleaved to maximize parallelism
7909
7910 // allocate stack space for the neon results
7911 __ sub(sp, sp, mulDataSize);
7912 __ mov(mul_ptr, sp);
7913
7914 // cross-multiply low * low for limbs b0-b3 and a3-a4 in parallel
7915 neon_partial_mult_64(A, b_lows, a_vals, 0);
7916
7917 // Limb 0
7918 __ ldr(a_i, __ post(a, BytesPerLong));
7919 gpr_partial_mult_52(a_i, b_0, high, low, limb_mask);
7920 __ mov(n, low);
7921 // __ andr(n, low, limb_mask);
7922
7923 // cross-multiply high * low for limbs b0-b3 and a3-a4 in parallel
7924 neon_partial_mult_64(B, b_highs, a_vals, 0);
7925
7926 // Limb 0 modulus computation
7927 // n.b. modulus computation requires multiplying successive
7928 // limbs of the product by corresponding limbs of the p256
7929 // prime adding the result to the limb and folding this
7930 // partial result into a running 256-bit sum in c_i. Limbs
7931 // of c_i are stored via c_ptr once carries are included.
7932 // n.b. the mul + add is omitted for limb 2 since the
7933 // corresponding prime bits are zero.
7934 gpr_partial_mult_52(n, mod_0, mod_high, mod_low, limb_mask);
7935 __ add(low, low, mod_low);
7936 __ add(high, high, mod_high);
7937 __ lsr(c_i, low, montMulP256Shift2);
7938 __ add(c_i, c_i, high);
7939
7940 // cross-multiply low * high for limbs b0-b3 and a3-a4 in parallel
7941 neon_partial_mult_64(C, b_lows, a_vals, 1);
7942
7943 // Limb 1
7944 gpr_partial_mult_52(a_i, b_1, high, low, limb_mask);
7945
7946 // cross-multiply high * high for limbs b0-b3 and a3-a4 in parallel
7947 neon_partial_mult_64(D, b_highs, a_vals, 1);
7948
7949 gpr_partial_mult_52(n, mod_1, mod_high, mod_low, limb_mask);
7950 __ add(low, low, mod_low);
7951 __ add(high, high, mod_high);
7952 __ add(c_i, c_i, low);
7953 __ str(c_i, c_ptr);
7954 __ mov(c_i, high);
7955
7956 // combine neon 32-bit partial products, regrouping to produce
7957 // 8*52-bit low products in A and 8*52-bit high products in D
7958
7959 // add low*high/high*low intermediate products before regrouping
7960 vs_addv(B, __ T2D, B, C); // Store (B+C) in B
7961
7962 // Limb 2
7963 gpr_partial_mult_52(a_i, b_2, high, low, limb_mask);
7964 __ add(c_i, c_i, low);
7965 __ str(c_i, Address(c_ptr, 8));
7966 __ mov(c_i, high);
7967
7968 // shift high*high (40-bit) product up into 52-bits of output
7969 vs_shl(D, __ T2D, D, montMulP256Shift1);
7970
7971 // Limb 3
7972 gpr_partial_mult_52(a_i, b_3, high, low, limb_mask);
7973
7974 // shift high 32 (or 33) bits of intermediate products for addition to D
7975 vs_ushr(C, __ T2D, B, 32 - montMulP256Shift1); // Use C for ((B+C) >>> 20)
7976
7977 gpr_partial_mult_52(n, mod_3, mod_high, mod_low, limb_mask);
7978 __ add(low, low, mod_low);
7979 __ add(high, high, mod_high);
7980 __ add(c_i, c_i, low);
7981 __ str(c_i, Address(c_ptr, 2 * BytesPerLong));
7982 __ mov(c_i, high);
7983
7984 // shift low 32 bits of intermediate product up for masking and addition to A
7985 vs_shl(B, __ T2D, B, 32);
7986
7987 // Limb 4
7988 gpr_partial_mult_52(a_i, b_4, high, low, limb_mask);
7989
7990 // add high bits of intermediate product into D
7991 vs_addv(D, __ T2D, D, C);
7992
7993 gpr_partial_mult_52(n, mod_4, mod_high, mod_low, limb_mask);
7994 __ add(low, low, mod_low);
7995 __ add(high, high, mod_high);
7996 __ add(c_i, c_i, low);
7997 __ str(c_i, Address(c_ptr, 3 * BytesPerLong));
7998 __ str(high, Address(c_ptr, 4 * BytesPerLong));
7999
8000 // top 12 bits of 32*32 bit product in A need adding into high 52-bit output
8001 vs_ushr(C, __ T2D, A, 52); // C now holds (A >>> 52)
8002 // Only 20 of the 32 bits now in the top of B should be added into A
8003 vs_andr(B, B, limb_mask_vec);
8004 // reduce original 64-bit product to 52-bits
8005 vs_andr(A, A, limb_mask_vec);
8006 // add intermediate products to high 52-bit result in D
8007 vs_addv(D, __ T2D, D, C);
8008 // add 20/21 bits of intermediate product in top of B into low 52-bit result
8009 vs_addv(A, __ T2D, A, B);
8010 // save and then mask off any overflow bit from computing low 52-bit result
8011 vs_ushr(B, __ T2D, A, montMulP256Shift2);
8012 vs_andr(A, A, limb_mask_vec);
8013 // add any remaining carry into the high 52-bit result
8014 vs_addv(D, __ T2D, D, B);
8015
8016 // the write interleaves the 4 successive pairs of low and
8017 // high results: (l0, l1), (h0, h1), ... (l6, l7), (h6, h7)
8018 vs_st1_interleaved(A, D, mul_ptr);
8019
8020 // Free mul_ptr
8021 common_regs = common_regs.remaining() + mul_ptr;
8022 mul_ptr = noreg;
8023
8024 /////////////////////////
8025 // Loop 2 & 3
8026 /////////////////////////
8027
8028 for (int i = 0; i < 2; i++) {
8029 // Load a_i and increment by 8 bytes
8030 __ ldr(a_i, __ post(a, BytesPerLong));
8031 __ ldr(c_i, c_ptr); //Load prior c_i
8032
8033 // Limb 0
8034 gpr_partial_mult_52(a_i, b_0, high, low, limb_mask);
8035 __ add(low, low, c_i);
8036 __ ldr(c_i, Address(c_ptr, BytesPerLong));
8037 __ andr(n, low, limb_mask);
8038 gpr_partial_mult_52(n, mod_0, mod_high, mod_low, limb_mask);
8039 __ add(low, low, mod_low);
8040 __ add(high, high, mod_high);
8041 __ lsr(tmp, low, montMulP256Shift2);
8042 __ add(c_i, c_i, tmp);
8043 __ add(c_i, c_i, high);
8044
8045 // Limb 1
8046 gpr_partial_mult_52(a_i, b_1, high, low, limb_mask);
8047 gpr_partial_mult_52(n, mod_1, mod_high, mod_low, limb_mask);
8048 __ ldr(tmp, Address(c_ptr, 2 * BytesPerLong));
8049 __ add(low, low, mod_low);
8050 __ add(high, high, mod_high);
8051 __ add(c_i, c_i, low);
8052 __ str(c_i, c_ptr);
8053 __ add(c_i, tmp, high);
8054
8055 // Limb 2
8056 gpr_partial_mult_52(a_i, b_2, high, low, limb_mask);
8057 __ ldr(tmp, Address(c_ptr, 3 * BytesPerLong));
8058 __ add(c_i, c_i, low);
8059 __ str(c_i, Address(c_ptr, BytesPerLong));
8060 __ add(c_i, tmp, high);
8061
8062 // Limb 3
8063 gpr_partial_mult_52(a_i, b_3, high, low, limb_mask);
8064 gpr_partial_mult_52(n, mod_3, mod_high, mod_low, limb_mask);
8065 __ ldr(tmp, Address(c_ptr, 4 * BytesPerLong));
8066 __ add(low, low, mod_low);
8067 __ add(high, high, mod_high);
8068 __ add(c_i, c_i, low);
8069 __ str(c_i, Address(c_ptr, 2 * BytesPerLong));
8070 __ add(c_i, tmp, high);
8071
8072 // Limb 4
8073 gpr_partial_mult_52(a_i, b_4, high, low, limb_mask);
8074 gpr_partial_mult_52(n, mod_4, mod_high, mod_low, limb_mask);
8075 __ add(low, low, mod_low);
8076 __ add(high, high, mod_high);
8077 __ add(c_i, c_i, low);
8078 __ str(c_i, Address(c_ptr, 3 * BytesPerLong));
8079 __ str(high, Address(c_ptr, 4 * BytesPerLong));
8080 }
8081 // Reallocate regs b_0, b_1, b_2 and b_3
8082 common_regs = common_regs.remaining()
8083 + b_0 + b_1 + b_2 + b_3;
8084 b_0 = b_1 = b_2 = b_3 = noreg;
8085
8086 Register low_1 = *common_regs++;
8087 Register high_1 = *common_regs++;
8088
8089 //////////////////////////////
8090 // a[3]
8091 //////////////////////////////
8092
8093 // For a_3 and a_4 we have already computed the cross-products
8094 // with b_0 ... b_3 and stored them on the stack relative to
8095 // `mul_ptr` i.e. the current `sp`in the order
8096 // l(a_3 * b_0), l(a_3 * b_1), h(a_3 * b_0), h(a_3 * b_1),
8097 // l(a_3 * b_2), l(a_3 * b_3), h(a_3 * b_2), h(a_3 * b_3),
8098 // l(a_4 * b_0), l(a_4 * b_1), h(a_4 * b_0), h(a_4 * b_1),
8099 // l(a_4 * b_2), l(a_4 * b_3), h(a_4 * b_2), h(a_4 * b_3),
8100 // where l(x) is the low 52 bits of x and h(x) is the high 52 bits
8101
8102 __ ldr(low_1, Address(sp));
8103 __ ldr(high_1, Address(sp, 2 * BytesPerLong));
8104
8105 __ ldr(low, Address(sp, BytesPerLong));
8106 __ ldr(high, Address(sp, 3 * BytesPerLong));
8107 __ ldr(a_i, __ post(a, BytesPerLong));
8108 __ ldr(c_i, c_ptr);
8109
8110 // Limb 0
8111 __ add(low_1, low_1, c_i);
8112 __ ldr(c_i, Address(c_ptr, BytesPerLong));
8113 __ andr(n, low_1, limb_mask);
8114 gpr_partial_mult_52(n, mod_0, mod_high, mod_low, limb_mask);
8115 __ add(low_1, low_1, mod_low);
8116 __ add(high_1, high_1, mod_high);
8117 __ lsr(tmp, low_1, montMulP256Shift2);
8118 __ add(c_i, c_i, tmp);
8119 __ add(c_i, c_i, high_1);
8120
8121 // Limb 1
8122 __ ldr(low_1, Address(sp, 4 * BytesPerLong));
8123 __ ldr(high_1, Address(sp, 6 * BytesPerLong));
8124 gpr_partial_mult_52(n, mod_1, mod_high, mod_low, limb_mask);
8125 __ ldr(tmp, Address(c_ptr, 2 * BytesPerLong));
8126 __ andr(mod_low, mod_low, limb_mask);
8127 __ add(low, low, mod_low);
8128 __ add(high, high, mod_high);
8129 __ add(c_i, c_i, low);
8130 __ str(c_i, c_ptr);
8131 __ add(c_i, tmp, high);
8132
8133 // Limb 2
8134 __ ldr(low, Address(sp, 5 * BytesPerLong));
8135 __ ldr(high, Address(sp, 7 * BytesPerLong));
8136 __ ldr(tmp, Address(c_ptr, 3 * BytesPerLong));
8137 __ add(c_i, c_i, low_1);
8138 __ str(c_i, Address(c_ptr, BytesPerLong));
8139 __ add(c_i, tmp, high_1);
8140
8141 // Limb 3
8142 gpr_partial_mult_52(n, mod_3, mod_high, mod_low, limb_mask);
8143 __ ldr(tmp, Address(c_ptr, 4 * BytesPerLong));
8144 __ add(low, low, mod_low);
8145 __ add(high, high, mod_high);
8146 __ add(c_i, c_i, low);
8147 __ str(c_i, Address(c_ptr, 2 * BytesPerLong));
8148 __ add(c_i, tmp, high);
8149
8150 // Limb 4
8151 __ ldr(low, Address(sp, 8 * BytesPerLong));
8152 __ ldr(high, Address(sp, 10 * BytesPerLong));
8153 gpr_partial_mult_52(a_i, b_4, high_1, low_1, limb_mask);
8154 gpr_partial_mult_52(n, mod_4, mod_high, mod_low, limb_mask);
8155 __ add(low_1, low_1, mod_low);
8156 __ add(high_1, high_1, mod_high);
8157 __ add(c_i, c_i, low_1);
8158 __ str(c_i, Address(c_ptr, 3 * BytesPerLong));
8159 __ str(high_1, Address(c_ptr, 4 * BytesPerLong));
8160
8161 //////////////////////////////
8162 // a[4]
8163 //////////////////////////////
8164
8165 Register c5 = *common_regs++,
8166 c6 = *common_regs++,
8167 c7 = *common_regs++;
8168
8169 __ ldr(a_i, a);
8170 __ ldr(c_i, c_ptr);
8171
8172 // Limb 0
8173 __ ldr(low_1, Address(sp, 9 * BytesPerLong));
8174 __ ldr(high_1, Address(sp, 11 * BytesPerLong));
8175
8176 __ add(low, low, c_i);
8177 __ ldr(c_i, Address(c_ptr, BytesPerLong));
8178 __ andr(n, low, limb_mask);
8179 gpr_partial_mult_52(n, mod_0, mod_high, mod_low, limb_mask);
8180 __ add(low, low, mod_low);
8181 __ add(high, high, mod_high);
8182 __ lsr(tmp, low, montMulP256Shift2);
8183 __ add(c_i, c_i, tmp);
8184 __ add(c_i, c_i, high);
8185
8186 __ ldr(low, Address(sp, 12 * BytesPerLong));
8187 __ ldr(high, Address(sp, 14 * BytesPerLong));
8188 gpr_partial_mult_52(n, mod_1, mod_high, mod_low, limb_mask);
8189 __ add(low_1, low_1, mod_low);
8190 __ add(high_1, high_1, mod_high);
8191 __ add(c5, c_i, low_1);
8192 __ ldr(c_i, Address(c_ptr, 2 * BytesPerLong));
8193 __ lsr(tmp, c5, montMulP256Shift2);
8194 __ add(c_i, c_i, tmp);
8195 __ add(c_i, c_i, high_1);
8196
8197 // Limb 2
8198 __ ldr(low_1, Address(sp, 13 * BytesPerLong));
8199 __ ldr(high_1, Address(sp, 15 * BytesPerLong));
8200 __ add(c6, c_i, low);
8201 __ ldr(c_i, Address(c_ptr, 3 * BytesPerLong));
8202 __ lsr(tmp, c6, montMulP256Shift2);
8203 __ add(c_i, c_i, tmp);
8204 __ add(c_i, c_i, high);
8205
8206 // Limb 3
8207 gpr_partial_mult_52(n, mod_3, mod_high, mod_low, limb_mask);
8208 __ add(low_1, low_1, mod_low);
8209 __ add(high_1, high_1, mod_high);
8210 __ add(c7, c_i, low_1);
8211 __ ldr(c_i, Address(c_ptr, 4 * BytesPerLong));
8212 __ lsr(tmp, c7, montMulP256Shift2);
8213 __ add(c_i, c_i, tmp);
8214 __ add(c_i, c_i, high_1);
8215
8216 // Limb 4
8217 gpr_partial_mult_52(a_i, b_4, high, low, limb_mask);
8218 gpr_partial_mult_52(n, mod_4, mod_high, mod_low, limb_mask);
8219 __ add(low, low, mod_low);
8220 __ add(high, high, mod_high);
8221
8222 // Reallocate b_4
8223 common_regs = common_regs.remaining() + b_4;
8224 b_4 = noreg;
8225
8226 Register c8 = *common_regs++,
8227 c9 = *common_regs++;
8228
8229 __ add(c8, c_i, low);
8230 __ lsr(c9, c8, montMulP256Shift2);
8231 __ add(c9, c9, high);
8232
8233 __ andr(c5, c5, limb_mask);
8234 __ andr(c6, c6, limb_mask);
8235 __ andr(c7, c7, limb_mask);
8236 __ andr(c8, c8, limb_mask);
8237
8238 /////////////////////////////
8239 // Final carry propagate
8240 /////////////////////////////
8241
8242 // c0 = c5 - modulus[0];
8243 // c1 = c6 - modulus[1] + (c0 >> BITS_PER_LIMB);
8244 // c0 &= LIMB_MASK;
8245 // c2 = c7 + (c1 >> BITS_PER_LIMB);
8246 // c1 &= LIMB_MASK;
8247 // c3 = c8 - modulus[3] + (c2 >> BITS_PER_LIMB);
8248 // c2 &= LIMB_MASK;
8249 // c4 = c9 - modulus4] + (c3 >> BITS_PER_LIMB);
8250 // c3 &= LIMB_MASK;
8251
8252 // Free up all unused regs
8253 common_regs = common_regs.remaining()
8254 + c_ptr + low + high + mod_high
8255 + mod_low + a_i + c_i + n + low_1 + high_1;
8256 c_ptr = low = high = mod_high
8257 = mod_low = a_i = c_i = n = low_1 = high_1 = noreg;
8258
8259 Register c0 = *common_regs++,
8260 c1 = *common_regs++,
8261 c2 = *common_regs++,
8262 c3 = *common_regs++,
8263 c4 = *common_regs++;
8264
8265 __ sub(c0, c5, mod_0);
8266 __ sub(c1, c6, mod_1);
8267 __ sub(c3, c8, mod_3);
8268 __ sub(c4, c9, mod_4);
8269 __ add(c1, c1, c0, Assembler::ASR, montMulP256Shift2);
8270 __ andr(c0, c0, limb_mask);
8271 __ add(c2, c7, c1, Assembler::ASR, montMulP256Shift2);
8272 __ andr(c1, c1, limb_mask);
8273 __ add(c3, c3, c2, Assembler::ASR, montMulP256Shift2);
8274 __ andr(c2, c2, limb_mask);
8275 __ add(c4, c4, c3, Assembler::ASR, montMulP256Shift2);
8276 __ andr(c3, c3, limb_mask);
8277
8278 // Final write back
8279 // mask = c4 >> 63
8280 // r[0] = ((c5 & mask) | (c0 & ~mask));
8281 // r[1] = ((c6 & mask) | (c1 & ~mask));
8282 // r[2] = ((c7 & mask) | (c2 & ~mask));
8283 // r[3] = ((c8 & mask) | (c3 & ~mask));
8284 // r[4] = ((c9 & mask) | (c4 & ~mask));
8285
8286 common_regs = common_regs.remaining()
8287 + mod_0 + mod_1 + mod_3 + mod_4;
8288 mod_0 = mod_1 = mod_3 = mod_4 = noreg;
8289
8290 Register mask = *common_regs++;
8291 Register nmask = *common_regs++;
8292
8293 __ asr(mask, c4, 63);
8294 __ mvn(nmask, mask);
8295 __ andr(c5, c5, mask);
8296 __ andr(tmp, c0, nmask);
8297 __ orr(c5, c5, tmp);
8298 __ andr(c6, c6, mask);
8299 __ andr(tmp, c1, nmask);
8300 __ orr(c6, c6, tmp);
8301 __ andr(c7, c7, mask);
8302 __ andr(tmp, c2, nmask);
8303 __ orr(c7, c7, tmp);
8304 __ andr(c8, c8, mask);
8305 __ andr(tmp, c3, nmask);
8306 __ orr(c8, c8, tmp);
8307 __ andr(c9, c9, mask);
8308 __ andr(tmp, c4, nmask);
8309 __ orr(c9, c9, tmp);
8310
8311 __ str(c5, result);
8312 __ str(c6, Address(result, BytesPerLong));
8313 __ str(c7, Address(result, 2 * BytesPerLong));
8314 __ str(c8, Address(result, 3 * BytesPerLong));
8315 __ str(c9, Address(result, 4 * BytesPerLong));
8316
8317 // End intrinsic call
8318 __ add(sp, sp, cDataSize + mulDataSize);
8319 __ pop(callee_saved, sp);
8320 __ leave();
8321 __ mov(r0, zr); // return 0
8322 __ ret(lr);
8323
8324 // record the stub entry and end
8325 store_archive_data(stub_id, start, __ pc());
8326
8327 return start;
8328 }
8329
8330 address generate_intpoly_assign() {
8331 // KNOWN Lengths:
8332 // MontgomeryIntPolynP256: 5 = 4 + 1
8333 // IntegerPolynomial1305: 5 = 4 + 1
8334 // IntegerPolynomial25519: 10 = 8 + 2
8335 // IntegerPolynomialP256: 10 = 8 + 2
8336 // Curve25519OrderField: 10 = 8 + 2
8337 // Curve25519OrderField: 10 = 8 + 2
8338 // P256OrderField: 10 = 8 + 2
8339 // IntegerPolynomialP384: 14 = 8 + 4 + 2
8340 // P384OrderField: 14 = 8 + 4 + 2
8341 // IntegerPolynomial448: 16 = 8 + 8
8342 // Curve448OrderField: 16 = 8 + 8
8343 // Curve448OrderField: 16 = 8 + 8
8344 // IntegerPolynomialP521: 19 = 8 + 8 + 2 + 1
8345 // P521OrderField: 19 = 8 + 8 + 2 + 1
8346 // Special Cases 5, 10, 14, 16, 19
8347 assert(UseIntPolyIntrinsics, "what are we doing here?");
8348 StubId stub_id = StubId::stubgen_intpoly_assign_id;
8349 int entry_count = StubInfo::entry_count(stub_id);
8350 assert(entry_count == 1, "sanity check");
8351 address start = load_archive_data(stub_id);
8352 if (start != nullptr) {
8353 return start;
8354 }
8355
8356 __ align(CodeEntryAlignment);
8357 StubCodeMark mark(this, stub_id);
8358 start = __ pc();
8359 __ enter();
8360
8361 // Inputs
8362 const Register set = c_rarg0;
8363 const Register aLimbs = c_rarg1;
8364 const Register bLimbs = c_rarg2;
8365 const Register length = c_rarg3;
8366
8367 Label L_Length5, L_Length10, L_Length14, L_Length16, L_Length19, L_Default, L_Done;
8368
8369 /*
8370 int maskValue = -set;
8371 for (int i = 0; i < a.length; i++) {
8372 long dummyLimbs = maskValue & (a[i] ^ b[i]);
8373 a[i] = dummyLimbs ^ a[i];
8374 }
8375 */
8376 Register mask_scalar = r4;
8377 FloatRegister mask_vec = v0;
8378
8379 __ neg(mask_scalar, set);
8380 __ dup(mask_vec, __ T2D, mask_scalar);
8381
8382 __ cmp(length, (u1)5);
8383 __ br(Assembler::EQ, L_Length5);
8384 __ cmp(length, (u1)10);
8385 __ br(Assembler::EQ, L_Length10);
8386 __ cmp(length, (u1)14);
8387 __ br(Assembler::EQ, L_Length14);
8388 __ cmp(length, (u1)16);
8389 __ br(Assembler::EQ, L_Length16);
8390 __ cmp(length, (u1)19);
8391 __ br(Assembler::EQ, L_Length19);
8392 __ b(L_Default);
8393
8394
8395 // Length = 5
8396 // Use 5 GPRs (neon not faster with this few limbs)
8397 __ BIND(L_Length5);
8398 {
8399 Register a0 = r5;
8400 Register a1 = r6;
8401 Register a2 = r7;
8402 Register a3 = r10;
8403 Register a4 = r11;
8404 Register b0 = r12;
8405 Register b1 = r13;
8406 Register b2 = r14;
8407 Register b3 = r15;
8408 Register b4 = r19;
8409
8410 __ push(r19, sp);
8411
8412 __ ldr(a0, aLimbs);
8413 __ ldr(a1, Address(aLimbs, 1 * BytesPerLong));
8414 __ ldr(a2, Address(aLimbs, 2 * BytesPerLong));
8415 __ ldr(a3, Address(aLimbs, 3 * BytesPerLong));
8416 __ ldr(a4, Address(aLimbs, 4 * BytesPerLong));
8417
8418 __ ldr(b0, bLimbs);
8419 __ ldr(b1, Address(bLimbs, 1 * BytesPerLong));
8420 __ ldr(b2, Address(bLimbs, 2 * BytesPerLong));
8421 __ ldr(b3, Address(bLimbs, 3 * BytesPerLong));
8422 __ ldr(b4, Address(bLimbs, 4 * BytesPerLong));
8423
8424 __ eor(b0, b0, a0);
8425 __ eor(b1, b1, a1);
8426 __ eor(b2, b2, a2);
8427 __ eor(b3, b3, a3);
8428 __ eor(b4, b4, a4);
8429
8430 __ andr(b0, b0, mask_scalar);
8431 __ andr(b1, b1, mask_scalar);
8432 __ andr(b2, b2, mask_scalar);
8433 __ andr(b3, b3, mask_scalar);
8434 __ andr(b4, b4, mask_scalar);
8435
8436 __ eor(a0, a0, b0);
8437 __ eor(a1, a1, b1);
8438 __ eor(a2, a2, b2);
8439 __ eor(a3, a3, b3);
8440 __ eor(a4, a4, b4);
8441
8442 __ str(a0, aLimbs);
8443 __ str(a1, Address(aLimbs, 1 * BytesPerLong));
8444 __ str(a2, Address(aLimbs, 2 * BytesPerLong));
8445 __ str(a3, Address(aLimbs, 3 * BytesPerLong));
8446 __ str(a4, Address(aLimbs, 4 * BytesPerLong));
8447
8448 __ pop(r19, sp);
8449 __ b(L_Done);
8450 }
8451
8452 // Length = 10
8453 // Split into 4 neon regs and 2 GPRs
8454 __ BIND(L_Length10);
8455 {
8456 Register a9 = r10;
8457 Register a10 = r11;
8458 Register b9 = r12;
8459 Register b10 = r13;
8460
8461 VSeq<4> a_vec(16);
8462 VSeq<4> b_vec(20);
8463
8464 __ ldr(a9, Address(aLimbs, 8 * BytesPerLong));
8465 __ ldr(a10, Address(aLimbs, 9 * BytesPerLong));
8466 __ ldr(b9, Address(bLimbs, 8 * BytesPerLong));
8467 __ ldr(b10, Address(bLimbs, 9 * BytesPerLong));
8468
8469 vs_ldpq(a_vec, aLimbs);
8470
8471 __ eor(b9, b9, a9);
8472 __ eor(b10, b10, a10);
8473
8474 vs_ldpq(b_vec, bLimbs);
8475
8476 __ andr(b9, b9, mask_scalar);
8477 __ andr(b10, b10, mask_scalar);
8478
8479 vs_eor(b_vec, b_vec, a_vec);
8480
8481 __ eor(a9, a9, b9);
8482 __ eor(a10, a10, b10);
8483
8484 vs_andr(b_vec, b_vec, mask_vec);
8485
8486 __ str(a9, Address(aLimbs, 8 * BytesPerLong));
8487 __ str(a10, Address(aLimbs, 9 * BytesPerLong));
8488
8489 vs_eor(a_vec, a_vec, b_vec);
8490 vs_stpq_post(a_vec, aLimbs);
8491
8492 __ b(L_Done);
8493 }
8494
8495 // Length = 14
8496 // Split into 5 neon regs and 4 GPRs
8497 __ BIND(L_Length14);
8498 {
8499 Register a10 = r5;
8500 Register a11 = r6;
8501 Register a12 = r7;
8502 Register a13 = r8;
8503 Register b10 = r9;
8504 Register b11 = r10;
8505 Register b12 = r11;
8506 Register b13 = r12;
8507
8508 VSeq<5> a_vec(16);
8509 VSeq<5> b_vec(22);
8510
8511 int offsets[2] = { 0, 32 };
8512
8513 __ ldr(a10, Address(aLimbs, 10 * BytesPerLong));
8514 __ ldr(a11, Address(aLimbs, 11 * BytesPerLong));
8515 __ ldr(a12, Address(aLimbs, 12 * BytesPerLong));
8516 __ ldr(a13, Address(aLimbs, 13 * BytesPerLong));
8517
8518 __ ldr(b10, Address(bLimbs, 10 * BytesPerLong));
8519 __ ldr(b11, Address(bLimbs, 11 * BytesPerLong));
8520 __ ldr(b12, Address(bLimbs, 12 * BytesPerLong));
8521 __ ldr(b13, Address(bLimbs, 13 * BytesPerLong));
8522
8523 __ ld1(a_vec[0], __ T2D, aLimbs);
8524 vs_ldpq_indexed(vs_tail(a_vec), aLimbs, 16, offsets);
8525
8526 __ eor(b10, b10, a10);
8527 __ eor(b11, b11, a11);
8528 __ eor(b12, b12, a12);
8529 __ eor(b13, b13, a13);
8530
8531 __ ld1(b_vec[0], __ T2D, bLimbs);
8532 vs_ldpq_indexed(vs_tail(b_vec), bLimbs, 16, offsets);
8533
8534 __ andr(b10, b10, mask_scalar);
8535 __ andr(b11, b11, mask_scalar);
8536 __ andr(b12, b12, mask_scalar);
8537 __ andr(b13, b13, mask_scalar);
8538
8539 vs_eor(b_vec, b_vec, a_vec);
8540
8541 __ eor(a10, a10, b10);
8542 __ eor(a11, a11, b11);
8543 __ eor(a12, a12, b12);
8544 __ eor(a13, a13, b13);
8545
8546 vs_andr(b_vec, b_vec, mask_vec);
8547
8548 __ str(a10, Address(aLimbs, 10 * BytesPerLong));
8549 __ str(a11, Address(aLimbs, 11 * BytesPerLong));
8550 __ str(a12, Address(aLimbs, 12 * BytesPerLong));
8551 __ str(a13, Address(aLimbs, 13 * BytesPerLong));
8552
8553 vs_eor(a_vec, a_vec, b_vec);
8554
8555 __ st1(a_vec[0], __ T2D, aLimbs);
8556 vs_stpq_indexed(vs_tail(a_vec), aLimbs, 16, offsets);
8557
8558 __ b(L_Done);
8559 }
8560
8561 // Length = 16
8562 // Use 8 neon regs
8563 __ BIND(L_Length16);
8564 {
8565 VSeq<8> a_vec(16);
8566 VSeq<8> b_vec(24);
8567
8568 vs_ldpq(a_vec, aLimbs);
8569 vs_ldpq(b_vec, bLimbs);
8570 vs_eor(b_vec, b_vec, a_vec);
8571 vs_andr(b_vec, b_vec, mask_vec);
8572 vs_eor(a_vec, a_vec, b_vec);
8573 vs_stpq_post(a_vec, aLimbs);
8574
8575 __ b(L_Done);
8576 }
8577
8578 // Length = 19
8579 // Split into 8 neon regs and 3 GPRs
8580 __ BIND(L_Length19);
8581 {
8582 Register a17 = r10;
8583 Register a18 = r11;
8584 Register a19 = r12;
8585 Register b17 = r13;
8586 Register b18 = r14;
8587 Register b19 = r15;
8588
8589 VSeq<8> a_vec(16);
8590 VSeq<8> b_vec(24);
8591
8592 __ ldr(a17, Address(aLimbs, 16 * BytesPerLong));
8593 __ ldr(a18, Address(aLimbs, 17 * BytesPerLong));
8594 __ ldr(a19, Address(aLimbs, 18 * BytesPerLong));
8595 __ ldr(b17, Address(bLimbs, 16 * BytesPerLong));
8596 __ ldr(b18, Address(bLimbs, 17 * BytesPerLong));
8597 __ ldr(b19, Address(bLimbs, 18 * BytesPerLong));
8598
8599 vs_ldpq(a_vec, aLimbs);
8600
8601 __ eor(b17, b17, a17);
8602 __ eor(b18, b18, a18);
8603 __ eor(b19, b19, a19);
8604
8605 vs_ldpq(b_vec, bLimbs);
8606
8607 __ andr(b17, b17, mask_scalar);
8608 __ andr(b18, b18, mask_scalar);
8609 __ andr(b19, b19, mask_scalar);
8610
8611 vs_eor(b_vec, b_vec, a_vec);
8612
8613 __ eor(a17, a17, b17);
8614 __ eor(a18, a18, b18);
8615 __ eor(a19, a19, b19);
8616
8617 vs_andr(b_vec, b_vec, mask_vec);
8618
8619 __ str(a17, Address(aLimbs, 16 * BytesPerLong));
8620 __ str(a18, Address(aLimbs, 17 * BytesPerLong));
8621 __ str(a19, Address(aLimbs, 18 * BytesPerLong));
8622
8623 vs_eor(a_vec, a_vec, b_vec);
8624 vs_stpq_post(a_vec, aLimbs);
8625
8626 __ b(L_Done);
8627 }
8628
8629 __ BIND(L_Default);
8630 {
8631 Register ctr = r5;
8632 Register a_val = r6;
8633 Register b_val = r7;
8634
8635 __ mov(ctr, length); // length (the number of limbs) is never 0
8636
8637 Label default_loop;
8638 __ BIND(default_loop);
8639
8640 __ ldr(a_val, aLimbs);
8641 __ ldr(b_val, __ post(bLimbs, 8));
8642 __ eor(b_val, b_val, a_val);
8643 __ andr(b_val, b_val, mask_scalar);
8644 __ eor(a_val, a_val, b_val);
8645 __ str(a_val, __ post(aLimbs, 8));
8646 __ sub(ctr, ctr, 1);
8647 __ cmp(ctr, (u1)0);
8648 __ br(Assembler::NE, default_loop);
8649 }
8650
8651 __ BIND(L_Done);
8652 __ leave(); // required for proper stackwalking of RuntimeStub frame
8653 __ mov(r0, zr); // return 0
8654 __ ret(lr);
8655
8656 // record the stub entry and end
8657 store_archive_data(stub_id, start, __ pc());
8658
8659 return start;
8660 }
8661
8662 /**
8663 * Arithmetic polynomial multiplication in Curve25519. The algorithm mimics
8664 * the version in the IntegerPolynomial25519 class, including the use of all
8665 * columns (no folding method).
8666 *
8667 * Arguments:
8668 *
8669 * Inputs:
8670 * c_rarg0 - long[] aLimbs
8671 * c_rarg1 - long[] bLimbs
8672 *
8673 * Output:
8674 * c_rarg2 - long[] rLimbs result
8675 */
8676 address generate_intpoly_mult_25519() {
8677 StubId stub_id = StubId::stubgen_intpoly_mult_25519_id;
8678 int entry_count = StubInfo::entry_count(stub_id);
8679 assert(entry_count == 1, "sanity check");
8680 address start = load_archive_data(stub_id);
8681 if (start != nullptr) {
8682 return start;
8683 }
8684 __ align(CodeEntryAlignment);
8685 StubCodeMark mark(this, stub_id);
8686 start = __ pc();
8687 __ enter();
8688
8689 // Register Map
8690 const Register aLimbs = c_rarg0; // r0
8691 const Register bLimbs = c_rarg1; // r1
8692 const Register rLimbs = c_rarg2; // r2
8693
8694 Register c[] = {r3, r4, r5, r6, r7, r8, r9, r10, r11, r12};
8695 Register a = r13;
8696 Register b = r14;
8697 Register term = r15;
8698 Register low = r16;
8699 Register high = r17;
8700
8701 const int32_t limbs = 5;
8702 const int32_t bpl = 51;
8703 const int32_t rem = 64 - bpl;
8704 const int32_t TERM = 19;
8705 const int32_t columns = limbs * 2;
8706 const uint64_t mask = (uint64_t) -1 >> rem;
8707 const uint64_t CARRY_ADD = (uint64_t) 1 << (bpl - 1);
8708
8709 __ mov(term, TERM);
8710 for (int i = 0; i < columns; i++) {
8711 __ mov(c[i], zr);
8712 }
8713
8714 // Perform high/low multiplication with signed 5x51 bit limbs
8715 for (int i = 0; i < limbs; i++) {
8716 __ ldr(b, Address(bLimbs, i * 8));
8717 for (int j = 0; j < limbs; j++) {
8718 __ ldr(a, Address(aLimbs, j * 8));
8719 __ smulh(high, a, b);
8720 __ mul(low, a, b);
8721 __ extr(high, high, low, bpl);
8722 __ andr(low, low, mask);
8723 __ add(c[i + j], c[i + j], low);
8724 __ add(c[i + j + 1], c[i + j + 1], high);
8725 }
8726 }
8727
8728 for (int i = 0; i < limbs; i++) {
8729 __ mul(c[i + 5], c[i + 5], term);
8730 __ add(c[i], c[i], c[i + 5]);
8731 }
8732
8733 // Carry-add with reduction from high limb
8734 Register tmp = low;
8735 Register carry_add = high;
8736 __ mov(carry_add, CARRY_ADD);
8737
8738 // Limb 3
8739 __ add(tmp, c[3], carry_add);
8740 __ asr(tmp, tmp, bpl);
8741 __ add(c[4], c[4], tmp);
8742 __ lsl(tmp, tmp, bpl);
8743 __ sub(c[3], c[3], tmp);
8744
8745 // Limb 4
8746 __ add(tmp, c[4], carry_add);
8747 __ asr(tmp, tmp, bpl);
8748
8749 // Reduce high order limb and fold back into low order limb
8750 __ mul(term, tmp, term);
8751 __ add(c[0], c[0], term);
8752
8753 __ lsl(tmp, tmp, bpl);
8754 __ sub(c[4], c[4], tmp);
8755
8756 // Limbs 0 - 3
8757 for (int i = 0; i < (limbs - 1); i++) {
8758 __ add(tmp, c[i], carry_add);
8759 __ asr(tmp, tmp, bpl);
8760 __ add(c[i + 1], c[i + 1], tmp);
8761 __ lsl(tmp, tmp, bpl);
8762 __ sub(c[i], c[i], tmp);
8763 }
8764
8765 for (int i = 0; i < limbs; i++) {
8766 __ str(c[i], Address(rLimbs, i * 8));
8767 }
8768
8769 __ mov(r0, 0);
8770 __ leave(); // required for proper stackwalking of RuntimeStub frame
8771 __ ret(lr);
8772
8773 // record the stub entry and end
8774 store_archive_data(stub_id, start, __ pc());
8775
8776 return start;
8777 }
8778
8779 void bcax5(Register a0, Register a1, Register a2, Register a3, Register a4,
8780 Register tmp0, Register tmp1, Register tmp2) {
8781 __ bic(tmp0, a2, a1); // for a0
8782 __ bic(tmp1, a3, a2); // for a1
8783 __ bic(tmp2, a4, a3); // for a2
8784 __ eor(a2, a2, tmp2);
8785 __ bic(tmp2, a0, a4); // for a3
8786 __ eor(a3, a3, tmp2);
8787 __ bic(tmp2, a1, a0); // for a4
8788 __ eor(a0, a0, tmp0);
8789 __ eor(a1, a1, tmp1);
8790 __ eor(a4, a4, tmp2);
8791 }
8792
8793 void keccak_round_gpr(bool can_use_fp, bool can_use_r18, Register rc,
8794 Register a0, Register a1, Register a2, Register a3, Register a4,
8795 Register a5, Register a6, Register a7, Register a8, Register a9,
8796 Register a10, Register a11, Register a12, Register a13, Register a14,
8797 Register a15, Register a16, Register a17, Register a18, Register a19,
8798 Register a20, Register a21, Register a22, Register a23, Register a24,
8799 Register tmp0, Register tmp1, Register tmp2) {
8800 __ eor3(tmp1, a4, a9, a14);
8801 __ eor3(tmp0, tmp1, a19, a24); // tmp0 = a4^a9^a14^a19^a24 = c4
8802 __ eor3(tmp2, a1, a6, a11);
8803 __ eor3(tmp1, tmp2, a16, a21); // tmp1 = a1^a6^a11^a16^a21 = c1
8804 __ rax1(tmp2, tmp0, tmp1); // d0
8805 {
8806
8807 Register tmp3, tmp4;
8808 if (can_use_fp && can_use_r18) {
8809 tmp3 = rfp;
8810 tmp4 = r18_tls;
8811 } else {
8812 tmp3 = a4;
8813 tmp4 = a9;
8814 __ stp(tmp3, tmp4, __ pre(sp, -16));
8815 }
8816
8817 __ eor3(tmp3, a0, a5, a10);
8818 __ eor3(tmp4, tmp3, a15, a20); // tmp4 = a0^a5^a10^a15^a20 = c0
8819 __ eor(a0, a0, tmp2);
8820 __ eor(a5, a5, tmp2);
8821 __ eor(a10, a10, tmp2);
8822 __ eor(a15, a15, tmp2);
8823 __ eor(a20, a20, tmp2); // d0(tmp2)
8824 __ eor3(tmp3, a2, a7, a12);
8825 __ eor3(tmp2, tmp3, a17, a22); // tmp2 = a2^a7^a12^a17^a22 = c2
8826 __ rax1(tmp3, tmp4, tmp2); // d1
8827 __ eor(a1, a1, tmp3);
8828 __ eor(a6, a6, tmp3);
8829 __ eor(a11, a11, tmp3);
8830 __ eor(a16, a16, tmp3);
8831 __ eor(a21, a21, tmp3); // d1(tmp3)
8832 __ rax1(tmp3, tmp2, tmp0); // d3
8833 __ eor3(tmp2, a3, a8, a13);
8834 __ eor3(tmp0, tmp2, a18, a23); // tmp0 = a3^a8^a13^a18^a23 = c3
8835 __ eor(a3, a3, tmp3);
8836 __ eor(a8, a8, tmp3);
8837 __ eor(a13, a13, tmp3);
8838 __ eor(a18, a18, tmp3);
8839 __ eor(a23, a23, tmp3);
8840 __ rax1(tmp2, tmp1, tmp0); // d2
8841 __ eor(a2, a2, tmp2);
8842 __ eor(a7, a7, tmp2);
8843 __ eor(a12, a12, tmp2);
8844 __ rax1(tmp0, tmp0, tmp4); // d4
8845 if (!can_use_fp || !can_use_r18) {
8846 __ ldp(tmp3, tmp4, __ post(sp, 16));
8847 }
8848 __ eor(a17, a17, tmp2);
8849 __ eor(a22, a22, tmp2);
8850 __ eor(a4, a4, tmp0);
8851 __ eor(a9, a9, tmp0);
8852 __ eor(a14, a14, tmp0);
8853 __ eor(a19, a19, tmp0);
8854 __ eor(a24, a24, tmp0);
8855 }
8856
8857 __ rol(tmp0, a10, 3);
8858 __ rol(a10, a1, 1);
8859 __ rol(a1, a6, 44);
8860 __ rol(a6, a9, 20);
8861 __ rol(a9, a22, 61);
8862 __ rol(a22, a14, 39);
8863 __ rol(a14, a20, 18);
8864 __ rol(a20, a2, 62);
8865 __ rol(a2, a12, 43);
8866 __ rol(a12, a13, 25);
8867 __ rol(a13, a19, 8) ;
8868 __ rol(a19, a23, 56);
8869 __ rol(a23, a15, 41);
8870 __ rol(a15, a4, 27);
8871 __ rol(a4, a24, 14);
8872 __ rol(a24, a21, 2);
8873 __ rol(a21, a8, 55);
8874 __ rol(a8, a16, 45);
8875 __ rol(a16, a5, 36);
8876 __ rol(a5, a3, 28);
8877 __ rol(a3, a18, 21);
8878 __ rol(a18, a17, 15);
8879 __ rol(a17, a11, 10);
8880 __ rol(a11, a7, 6);
8881 __ mov(a7, tmp0);
8882
8883 bcax5(a0, a1, a2, a3, a4, tmp0, tmp1, tmp2);
8884 bcax5(a5, a6, a7, a8, a9, tmp0, tmp1, tmp2);
8885 bcax5(a10, a11, a12, a13, a14, tmp0, tmp1, tmp2);
8886 bcax5(a15, a16, a17, a18, a19, tmp0, tmp1, tmp2);
8887 bcax5(a20, a21, a22, a23, a24, tmp0, tmp1, tmp2);
8888
8889 __ ldr(tmp1, __ post(rc, 8));
8890 __ eor(a0, a0, tmp1);
8891
8892 }
8893
8894 // Arguments:
8895 //
8896 // Inputs:
8897 // c_rarg0 - byte[] source+offset
8898 // c_rarg1 - byte[] SHA.state
8899 // c_rarg2 - int block_size
8900 // c_rarg3 - int offset
8901 // c_rarg4 - int limit
8902 //
8903 address generate_sha3_implCompress_gpr(StubId stub_id) {
8904 bool multi_block;
8905 switch (stub_id) {
8906 case StubId::stubgen_sha3_implCompress_id:
8907 multi_block = false;
8908 break;
8909 case StubId::stubgen_sha3_implCompressMB_id:
8910 multi_block = true;
8911 break;
8912 default:
8913 ShouldNotReachHere();
8914 }
8915 int entry_count = StubInfo::entry_count(stub_id);
8916 assert(entry_count == 1, "sanity check");
8917 address start = load_archive_data(stub_id);
8918 if (start != nullptr) {
8919 return start;
8920 }
8921 __ align(CodeEntryAlignment);
8922 StubCodeMark mark(this, stub_id);
8923 start = __ pc();
8924
8925 Register buf = c_rarg0;
8926 Register state = c_rarg1;
8927 Register block_size = c_rarg2;
8928 Register ofs = c_rarg3;
8929 Register limit = c_rarg4;
8930
8931 // use r3.r17,r19..r28 to keep a0..a24.
8932 // a0..a24 are respective locals from SHA3.java
8933 Register a0 = r25,
8934 a1 = r26,
8935 a2 = r27,
8936 a3 = r3,
8937 a4 = r4,
8938 a5 = r5,
8939 a6 = r6,
8940 a7 = r7,
8941 a8 = rscratch1, // r8
8942 a9 = rscratch2, // r9
8943 a10 = r10,
8944 a11 = r11,
8945 a12 = r12,
8946 a13 = r13,
8947 a14 = r14,
8948 a15 = r15,
8949 a16 = r16,
8950 a17 = r17,
8951 a18 = r28,
8952 a19 = r19,
8953 a20 = r20,
8954 a21 = r21,
8955 a22 = r22,
8956 a23 = r23,
8957 a24 = r24;
8958
8959 Register tmp0 = block_size, tmp1 = buf, tmp2 = state, tmp3 = r30;
8960
8961 Label sha3_loop, rounds24_preloop, loop_body;
8962 Label sha3_512_or_sha3_384, shake128;
8963
8964 bool can_use_r18 = false;
8965 #ifndef R18_RESERVED
8966 can_use_r18 = true;
8967 #endif
8968 bool can_use_fp = !PreserveFramePointer;
8969
8970 __ enter();
8971
8972 // save almost all yet unsaved gpr registers on stack
8973 __ str(block_size, __ pre(sp, -128));
8974 if (multi_block) {
8975 __ stpw(ofs, limit, Address(sp, 8));
8976 }
8977 // 8 bytes at sp+16 will be used to keep buf
8978 __ stp(r19, r20, Address(sp, 32));
8979 __ stp(r21, r22, Address(sp, 48));
8980 __ stp(r23, r24, Address(sp, 64));
8981 __ stp(r25, r26, Address(sp, 80));
8982 __ stp(r27, r28, Address(sp, 96));
8983 if (can_use_r18 && can_use_fp) {
8984 __ stp(r18_tls, state, Address(sp, 112));
8985 } else {
8986 __ str(state, Address(sp, 112));
8987 }
8988
8989 // begin sha3 calculations: loading a0..a24 from state arrary
8990 __ ldp(a0, a1, state);
8991 __ ldp(a2, a3, Address(state, 16));
8992 __ ldp(a4, a5, Address(state, 32));
8993 __ ldp(a6, a7, Address(state, 48));
8994 __ ldp(a8, a9, Address(state, 64));
8995 __ ldp(a10, a11, Address(state, 80));
8996 __ ldp(a12, a13, Address(state, 96));
8997 __ ldp(a14, a15, Address(state, 112));
8998 __ ldp(a16, a17, Address(state, 128));
8999 __ ldp(a18, a19, Address(state, 144));
9000 __ ldp(a20, a21, Address(state, 160));
9001 __ ldp(a22, a23, Address(state, 176));
9002 __ ldr(a24, Address(state, 192));
9003
9004 __ BIND(sha3_loop);
9005
9006 // load input
9007 __ ldp(tmp3, tmp2, __ post(buf, 16));
9008 __ eor(a0, a0, tmp3);
9009 __ eor(a1, a1, tmp2);
9010 __ ldp(tmp3, tmp2, __ post(buf, 16));
9011 __ eor(a2, a2, tmp3);
9012 __ eor(a3, a3, tmp2);
9013 __ ldp(tmp3, tmp2, __ post(buf, 16));
9014 __ eor(a4, a4, tmp3);
9015 __ eor(a5, a5, tmp2);
9016 __ ldr(tmp3, __ post(buf, 8));
9017 __ eor(a6, a6, tmp3);
9018
9019 // block_size == 72, SHA3-512; block_size == 104, SHA3-384
9020 __ tbz(block_size, 7, sha3_512_or_sha3_384);
9021
9022 __ ldp(tmp3, tmp2, __ post(buf, 16));
9023 __ eor(a7, a7, tmp3);
9024 __ eor(a8, a8, tmp2);
9025 __ ldp(tmp3, tmp2, __ post(buf, 16));
9026 __ eor(a9, a9, tmp3);
9027 __ eor(a10, a10, tmp2);
9028 __ ldp(tmp3, tmp2, __ post(buf, 16));
9029 __ eor(a11, a11, tmp3);
9030 __ eor(a12, a12, tmp2);
9031 __ ldp(tmp3, tmp2, __ post(buf, 16));
9032 __ eor(a13, a13, tmp3);
9033 __ eor(a14, a14, tmp2);
9034 __ ldp(tmp3, tmp2, __ post(buf, 16));
9035 __ eor(a15, a15, tmp3);
9036 __ eor(a16, a16, tmp2);
9037
9038 // block_size == 136, bit4 == 0 and bit5 == 0, SHA3-256 or SHAKE256
9039 __ andw(tmp2, block_size, 48);
9040 __ cbzw(tmp2, rounds24_preloop);
9041 __ tbnz(block_size, 5, shake128);
9042 // block_size == 144, bit5 == 0, SHA3-244
9043 __ ldr(tmp3, __ post(buf, 8));
9044 __ eor(a17, a17, tmp3);
9045 __ b(rounds24_preloop);
9046
9047 __ BIND(shake128);
9048 __ ldp(tmp3, tmp2, __ post(buf, 16));
9049 __ eor(a17, a17, tmp3);
9050 __ eor(a18, a18, tmp2);
9051 __ ldp(tmp3, tmp2, __ post(buf, 16));
9052 __ eor(a19, a19, tmp3);
9053 __ eor(a20, a20, tmp2);
9054 __ b(rounds24_preloop); // block_size == 168, SHAKE128
9055
9056 __ BIND(sha3_512_or_sha3_384);
9057 __ ldp(tmp3, tmp2, __ post(buf, 16));
9058 __ eor(a7, a7, tmp3);
9059 __ eor(a8, a8, tmp2);
9060 __ tbz(block_size, 5, rounds24_preloop); // SHA3-512
9061
9062 // SHA3-384
9063 __ ldp(tmp3, tmp2, __ post(buf, 16));
9064 __ eor(a9, a9, tmp3);
9065 __ eor(a10, a10, tmp2);
9066 __ ldp(tmp3, tmp2, __ post(buf, 16));
9067 __ eor(a11, a11, tmp3);
9068 __ eor(a12, a12, tmp2);
9069
9070 __ BIND(rounds24_preloop);
9071 __ fmovs(v0, 24.0); // float loop counter,
9072 __ fmovs(v1, 1.0); // exact representation
9073
9074 __ str(buf, Address(sp, 16));
9075 __ lea(tmp3, ExternalAddress((address) _sha3_round_consts));
9076
9077 __ BIND(loop_body);
9078 keccak_round_gpr(can_use_fp, can_use_r18, tmp3,
9079 a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12,
9080 a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24,
9081 tmp0, tmp1, tmp2);
9082 __ fsubs(v0, v0, v1);
9083 __ fcmps(v0, 0.0);
9084 __ br(__ NE, loop_body);
9085
9086 if (multi_block) {
9087 __ ldrw(block_size, sp); // block_size
9088 __ ldpw(tmp2, tmp1, Address(sp, 8)); // offset, limit
9089 __ addw(tmp2, tmp2, block_size);
9090 __ cmpw(tmp2, tmp1);
9091 __ strw(tmp2, Address(sp, 8)); // store offset in case we're jumping
9092 __ ldr(buf, Address(sp, 16)); // restore buf in case we're jumping
9093 __ br(Assembler::LE, sha3_loop);
9094 __ movw(c_rarg0, tmp2); // return offset
9095 }
9096 if (can_use_fp && can_use_r18) {
9097 __ ldp(r18_tls, state, Address(sp, 112));
9098 } else {
9099 __ ldr(state, Address(sp, 112));
9100 }
9101 // save calculated sha3 state
9102 __ stp(a0, a1, Address(state));
9103 __ stp(a2, a3, Address(state, 16));
9104 __ stp(a4, a5, Address(state, 32));
9105 __ stp(a6, a7, Address(state, 48));
9106 __ stp(a8, a9, Address(state, 64));
9107 __ stp(a10, a11, Address(state, 80));
9108 __ stp(a12, a13, Address(state, 96));
9109 __ stp(a14, a15, Address(state, 112));
9110 __ stp(a16, a17, Address(state, 128));
9111 __ stp(a18, a19, Address(state, 144));
9112 __ stp(a20, a21, Address(state, 160));
9113 __ stp(a22, a23, Address(state, 176));
9114 __ str(a24, Address(state, 192));
9115
9116 // restore required registers from stack
9117 __ ldp(r19, r20, Address(sp, 32));
9118 __ ldp(r21, r22, Address(sp, 48));
9119 __ ldp(r23, r24, Address(sp, 64));
9120 __ ldp(r25, r26, Address(sp, 80));
9121 __ ldp(r27, r28, Address(sp, 96));
9122 if (can_use_fp && can_use_r18) {
9123 __ add(rfp, sp, 128); // leave() will copy rfp to sp below
9124 } // else no need to recalculate rfp, since it wasn't changed
9125
9126 __ leave();
9127
9128 __ ret(lr);
9129
9130 // record the stub entry and end
9131 store_archive_data(stub_id, start, __ pc());
9132
9133 return start;
9134 }
9135
9136 /**
9137 * Arguments:
9138 *
9139 * Inputs:
9140 * c_rarg0 - int crc
9141 * c_rarg1 - byte* buf
9142 * c_rarg2 - int length
9143 *
9144 * Output:
9145 * rax - int crc result
9146 */
9147 address generate_updateBytesCRC32() {
9148 assert(UseCRC32Intrinsics, "what are we doing here?");
9149 StubId stub_id = StubId::stubgen_updateBytesCRC32_id;
9150 int entry_count = StubInfo::entry_count(stub_id);
9151 assert(entry_count == 1, "sanity check");
9152 address start = load_archive_data(stub_id);
9153 if (start != nullptr) {
9154 return start;
9155 }
9156 __ align(CodeEntryAlignment);
9157 StubCodeMark mark(this, stub_id);
9158
9159 start = __ pc();
9160
9161 const Register crc = c_rarg0; // crc
9162 const Register buf = c_rarg1; // source java byte array address
9163 const Register len = c_rarg2; // length
9164 const Register table0 = c_rarg3; // crc_table address
9165 const Register table1 = c_rarg4;
9166 const Register table2 = c_rarg5;
9167 const Register table3 = c_rarg6;
9168 const Register tmp3 = c_rarg7;
9169
9170 BLOCK_COMMENT("Entry:");
9171 __ enter(); // required for proper stackwalking of RuntimeStub frame
9172
9173 __ kernel_crc32(crc, buf, len,
9174 table0, table1, table2, table3, rscratch1, rscratch2, tmp3);
9175
9176 __ leave(); // required for proper stackwalking of RuntimeStub frame
9177 __ ret(lr);
9178
9179 // record the stub entry and end
9180 store_archive_data(stub_id, start, __ pc());
9181
9182 return start;
9183 }
9184
9185 /**
9186 * Arguments:
9187 *
9188 * Inputs:
9189 * c_rarg0 - int crc
9190 * c_rarg1 - byte* buf
9191 * c_rarg2 - int length
9192 * c_rarg3 - int* table
9193 *
9194 * Output:
9195 * r0 - int crc result
9196 */
9197 address generate_updateBytesCRC32C() {
9198 assert(UseCRC32CIntrinsics, "what are we doing here?");
9199 StubId stub_id = StubId::stubgen_updateBytesCRC32C_id;
9200 int entry_count = StubInfo::entry_count(stub_id);
9201 assert(entry_count == 1, "sanity check");
9202 address start = load_archive_data(stub_id);
9203 if (start != nullptr) {
9204 return start;
9205 }
9206 __ align(CodeEntryAlignment);
9207 StubCodeMark mark(this, stub_id);
9208
9209 start = __ pc();
9210
9211 const Register crc = c_rarg0; // crc
9212 const Register buf = c_rarg1; // source java byte array address
9213 const Register len = c_rarg2; // length
9214 const Register table0 = c_rarg3; // crc_table address
9215 const Register table1 = c_rarg4;
9216 const Register table2 = c_rarg5;
9217 const Register table3 = c_rarg6;
9218 const Register tmp3 = c_rarg7;
9219
9220 BLOCK_COMMENT("Entry:");
9221 __ enter(); // required for proper stackwalking of RuntimeStub frame
9222
9223 __ kernel_crc32c(crc, buf, len,
9224 table0, table1, table2, table3, rscratch1, rscratch2, tmp3);
9225
9226 __ leave(); // required for proper stackwalking of RuntimeStub frame
9227 __ ret(lr);
9228
9229 // record the stub entry and end
9230 store_archive_data(stub_id, start, __ pc());
9231
9232 return start;
9233 }
9234
9235 /***
9236 * Arguments:
9237 *
9238 * Inputs:
9239 * c_rarg0 - int adler
9240 * c_rarg1 - byte* buff
9241 * c_rarg2 - int len
9242 *
9243 * Output:
9244 * c_rarg0 - int adler result
9245 */
9246 address generate_updateBytesAdler32() {
9247 StubId stub_id = StubId::stubgen_updateBytesAdler32_id;
9248 int entry_count = StubInfo::entry_count(stub_id);
9249 assert(entry_count == 1, "sanity check");
9250 address start = load_archive_data(stub_id);
9251 if (start != nullptr) {
9252 return start;
9253 }
9254 __ align(CodeEntryAlignment);
9255 StubCodeMark mark(this, stub_id);
9256 start = __ pc();
9257
9258 Label L_simple_by1_loop, L_nmax, L_nmax_loop, L_by16, L_by16_loop, L_by1_loop, L_do_mod, L_combine, L_by1;
9259
9260 // Aliases
9261 Register adler = c_rarg0;
9262 Register s1 = c_rarg0;
9263 Register s2 = c_rarg3;
9264 Register buff = c_rarg1;
9265 Register len = c_rarg2;
9266 Register nmax = r4;
9267 Register base = r5;
9268 Register count = r6;
9269 Register temp0 = rscratch1;
9270 Register temp1 = rscratch2;
9271 FloatRegister vbytes = v0;
9272 FloatRegister vs1acc = v1;
9273 FloatRegister vs2acc = v2;
9274 FloatRegister vtable = v3;
9275
9276 // Max number of bytes we can process before having to take the mod
9277 // 0x15B0 is 5552 in decimal, the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
9278 uint64_t BASE = 0xfff1;
9279 uint64_t NMAX = 0x15B0;
9280
9281 __ mov(base, BASE);
9282 __ mov(nmax, NMAX);
9283
9284 // Load accumulation coefficients for the upper 16 bits
9285 __ lea(temp0, ExternalAddress((address) StubRoutines::aarch64::_adler_table));
9286 __ ld1(vtable, __ T16B, Address(temp0));
9287
9288 // s1 is initialized to the lower 16 bits of adler
9289 // s2 is initialized to the upper 16 bits of adler
9290 __ ubfx(s2, adler, 16, 16); // s2 = ((adler >> 16) & 0xffff)
9291 __ uxth(s1, adler); // s1 = (adler & 0xffff)
9292
9293 // The pipelined loop needs at least 16 elements for 1 iteration
9294 // It does check this, but it is more effective to skip to the cleanup loop
9295 __ cmp(len, (u1)16);
9296 __ br(Assembler::HS, L_nmax);
9297 __ cbz(len, L_combine);
9298
9299 __ bind(L_simple_by1_loop);
9300 __ ldrb(temp0, Address(__ post(buff, 1)));
9301 __ add(s1, s1, temp0);
9302 __ add(s2, s2, s1);
9303 __ subs(len, len, 1);
9304 __ br(Assembler::HI, L_simple_by1_loop);
9305
9306 // s1 = s1 % BASE
9307 __ subs(temp0, s1, base);
9308 __ csel(s1, temp0, s1, Assembler::HS);
9309
9310 // s2 = s2 % BASE
9311 __ lsr(temp0, s2, 16);
9312 __ lsl(temp1, temp0, 4);
9313 __ sub(temp1, temp1, temp0);
9314 __ add(s2, temp1, s2, ext::uxth);
9315
9316 __ subs(temp0, s2, base);
9317 __ csel(s2, temp0, s2, Assembler::HS);
9318
9319 __ b(L_combine);
9320
9321 __ bind(L_nmax);
9322 __ subs(len, len, nmax);
9323 __ sub(count, nmax, 16);
9324 __ br(Assembler::LO, L_by16);
9325
9326 __ bind(L_nmax_loop);
9327
9328 generate_updateBytesAdler32_accum(s1, s2, buff, temp0, temp1,
9329 vbytes, vs1acc, vs2acc, vtable);
9330
9331 __ subs(count, count, 16);
9332 __ br(Assembler::HS, L_nmax_loop);
9333
9334 // s1 = s1 % BASE
9335 __ lsr(temp0, s1, 16);
9336 __ lsl(temp1, temp0, 4);
9337 __ sub(temp1, temp1, temp0);
9338 __ add(temp1, temp1, s1, ext::uxth);
9339
9340 __ lsr(temp0, temp1, 16);
9341 __ lsl(s1, temp0, 4);
9342 __ sub(s1, s1, temp0);
9343 __ add(s1, s1, temp1, ext:: uxth);
9344
9345 __ subs(temp0, s1, base);
9346 __ csel(s1, temp0, s1, Assembler::HS);
9347
9348 // s2 = s2 % BASE
9349 __ lsr(temp0, s2, 16);
9350 __ lsl(temp1, temp0, 4);
9351 __ sub(temp1, temp1, temp0);
9352 __ add(temp1, temp1, s2, ext::uxth);
9353
9354 __ lsr(temp0, temp1, 16);
9355 __ lsl(s2, temp0, 4);
9356 __ sub(s2, s2, temp0);
9357 __ add(s2, s2, temp1, ext:: uxth);
9358
9359 __ subs(temp0, s2, base);
9360 __ csel(s2, temp0, s2, Assembler::HS);
9361
9362 __ subs(len, len, nmax);
9363 __ sub(count, nmax, 16);
9364 __ br(Assembler::HS, L_nmax_loop);
9365
9366 __ bind(L_by16);
9367 __ adds(len, len, count);
9368 __ br(Assembler::LO, L_by1);
9369
9370 __ bind(L_by16_loop);
9371
9372 generate_updateBytesAdler32_accum(s1, s2, buff, temp0, temp1,
9373 vbytes, vs1acc, vs2acc, vtable);
9374
9375 __ subs(len, len, 16);
9376 __ br(Assembler::HS, L_by16_loop);
9377
9378 __ bind(L_by1);
9379 __ adds(len, len, 15);
9380 __ br(Assembler::LO, L_do_mod);
9381
9382 __ bind(L_by1_loop);
9383 __ ldrb(temp0, Address(__ post(buff, 1)));
9384 __ add(s1, temp0, s1);
9385 __ add(s2, s2, s1);
9386 __ subs(len, len, 1);
9387 __ br(Assembler::HS, L_by1_loop);
9388
9389 __ bind(L_do_mod);
9390 // s1 = s1 % BASE
9391 __ lsr(temp0, s1, 16);
9392 __ lsl(temp1, temp0, 4);
9393 __ sub(temp1, temp1, temp0);
9394 __ add(temp1, temp1, s1, ext::uxth);
9395
9396 __ lsr(temp0, temp1, 16);
9397 __ lsl(s1, temp0, 4);
9398 __ sub(s1, s1, temp0);
9399 __ add(s1, s1, temp1, ext:: uxth);
9400
9401 __ subs(temp0, s1, base);
9402 __ csel(s1, temp0, s1, Assembler::HS);
9403
9404 // s2 = s2 % BASE
9405 __ lsr(temp0, s2, 16);
9406 __ lsl(temp1, temp0, 4);
9407 __ sub(temp1, temp1, temp0);
9408 __ add(temp1, temp1, s2, ext::uxth);
9409
9410 __ lsr(temp0, temp1, 16);
9411 __ lsl(s2, temp0, 4);
9412 __ sub(s2, s2, temp0);
9413 __ add(s2, s2, temp1, ext:: uxth);
9414
9415 __ subs(temp0, s2, base);
9416 __ csel(s2, temp0, s2, Assembler::HS);
9417
9418 // Combine lower bits and higher bits
9419 __ bind(L_combine);
9420 __ orr(s1, s1, s2, Assembler::LSL, 16); // adler = s1 | (s2 << 16)
9421
9422 __ ret(lr);
9423
9424 // record the stub entry and end
9425 store_archive_data(stub_id, start, __ pc());
9426
9427 return start;
9428 }
9429
9430 void generate_updateBytesAdler32_accum(Register s1, Register s2, Register buff,
9431 Register temp0, Register temp1, FloatRegister vbytes,
9432 FloatRegister vs1acc, FloatRegister vs2acc, FloatRegister vtable) {
9433 // Below is a vectorized implementation of updating s1 and s2 for 16 bytes.
9434 // We use b1, b2, ..., b16 to denote the 16 bytes loaded in each iteration.
9435 // In non-vectorized code, we update s1 and s2 as:
9436 // s1 <- s1 + b1
9437 // s2 <- s2 + s1
9438 // s1 <- s1 + b2
9439 // s2 <- s2 + b1
9440 // ...
9441 // s1 <- s1 + b16
9442 // s2 <- s2 + s1
9443 // Putting above assignments together, we have:
9444 // s1_new = s1 + b1 + b2 + ... + b16
9445 // s2_new = s2 + (s1 + b1) + (s1 + b1 + b2) + ... + (s1 + b1 + b2 + ... + b16)
9446 // = s2 + s1 * 16 + (b1 * 16 + b2 * 15 + ... + b16 * 1)
9447 // = s2 + s1 * 16 + (b1, b2, ... b16) dot (16, 15, ... 1)
9448 __ ld1(vbytes, __ T16B, Address(__ post(buff, 16)));
9449
9450 // s2 = s2 + s1 * 16
9451 __ add(s2, s2, s1, Assembler::LSL, 4);
9452
9453 // vs1acc = b1 + b2 + b3 + ... + b16
9454 // vs2acc = (b1 * 16) + (b2 * 15) + (b3 * 14) + ... + (b16 * 1)
9455 __ umullv(vs2acc, __ T8B, vtable, vbytes);
9456 __ umlalv(vs2acc, __ T16B, vtable, vbytes);
9457 __ uaddlv(vs1acc, __ T16B, vbytes);
9458 __ uaddlv(vs2acc, __ T8H, vs2acc);
9459
9460 // s1 = s1 + vs1acc, s2 = s2 + vs2acc
9461 __ fmovd(temp0, vs1acc);
9462 __ fmovd(temp1, vs2acc);
9463 __ add(s1, s1, temp0);
9464 __ add(s2, s2, temp1);
9465 }
9466
9467 /**
9468 * Arguments:
9469 *
9470 * Input:
9471 * c_rarg0 - x address
9472 * c_rarg1 - x length
9473 * c_rarg2 - y address
9474 * c_rarg3 - y length
9475 * c_rarg4 - z address
9476 */
9477 address generate_multiplyToLen() {
9478 StubId stub_id = StubId::stubgen_multiplyToLen_id;
9479 int entry_count = StubInfo::entry_count(stub_id);
9480 assert(entry_count == 1, "sanity check");
9481 address start = load_archive_data(stub_id);
9482 if (start != nullptr) {
9483 return start;
9484 }
9485 __ align(CodeEntryAlignment);
9486 StubCodeMark mark(this, stub_id);
9487
9488 start = __ pc();
9489 const Register x = r0;
9490 const Register xlen = r1;
9491 const Register y = r2;
9492 const Register ylen = r3;
9493 const Register z = r4;
9494
9495 const Register tmp0 = r5;
9496 const Register tmp1 = r10;
9497 const Register tmp2 = r11;
9498 const Register tmp3 = r12;
9499 const Register tmp4 = r13;
9500 const Register tmp5 = r14;
9501 const Register tmp6 = r15;
9502 const Register tmp7 = r16;
9503
9504 BLOCK_COMMENT("Entry:");
9505 __ enter(); // required for proper stackwalking of RuntimeStub frame
9506 __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
9507 __ leave(); // required for proper stackwalking of RuntimeStub frame
9508 __ ret(lr);
9509
9510 // record the stub entry and end
9511 store_archive_data(stub_id, start, __ pc());
9512
9513 return start;
9514 }
9515
9516 address generate_squareToLen() {
9517 // squareToLen algorithm for sizes 1..127 described in java code works
9518 // faster than multiply_to_len on some CPUs and slower on others, but
9519 // multiply_to_len shows a bit better overall results
9520 StubId stub_id = StubId::stubgen_squareToLen_id;
9521 int entry_count = StubInfo::entry_count(stub_id);
9522 assert(entry_count == 1, "sanity check");
9523 address start = load_archive_data(stub_id);
9524 if (start != nullptr) {
9525 return start;
9526 }
9527 __ align(CodeEntryAlignment);
9528 StubCodeMark mark(this, stub_id);
9529 start = __ pc();
9530
9531 const Register x = r0;
9532 const Register xlen = r1;
9533 const Register z = r2;
9534 const Register y = r4; // == x
9535 const Register ylen = r5; // == xlen
9536
9537 const Register tmp0 = r3;
9538 const Register tmp1 = r10;
9539 const Register tmp2 = r11;
9540 const Register tmp3 = r12;
9541 const Register tmp4 = r13;
9542 const Register tmp5 = r14;
9543 const Register tmp6 = r15;
9544 const Register tmp7 = r16;
9545
9546 RegSet spilled_regs = RegSet::of(y, ylen);
9547 BLOCK_COMMENT("Entry:");
9548 __ enter();
9549 __ push(spilled_regs, sp);
9550 __ mov(y, x);
9551 __ mov(ylen, xlen);
9552 __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
9553 __ pop(spilled_regs, sp);
9554 __ leave();
9555 __ ret(lr);
9556
9557 // record the stub entry and end
9558 store_archive_data(stub_id, start, __ pc());
9559
9560 return start;
9561 }
9562
9563 address generate_mulAdd() {
9564 StubId stub_id = StubId::stubgen_mulAdd_id;
9565 int entry_count = StubInfo::entry_count(stub_id);
9566 assert(entry_count == 1, "sanity check");
9567 address start = load_archive_data(stub_id);
9568 if (start != nullptr) {
9569 return start;
9570 }
9571 __ align(CodeEntryAlignment);
9572 StubCodeMark mark(this, stub_id);
9573
9574 start = __ pc();
9575
9576 const Register out = r0;
9577 const Register in = r1;
9578 const Register offset = r2;
9579 const Register len = r3;
9580 const Register k = r4;
9581
9582 BLOCK_COMMENT("Entry:");
9583 __ enter();
9584 __ mul_add(out, in, offset, len, k);
9585 __ leave();
9586 __ ret(lr);
9587
9588 // record the stub entry and end
9589 store_archive_data(stub_id, start, __ pc());
9590
9591 return start;
9592 }
9593
9594 // Arguments:
9595 //
9596 // Input:
9597 // c_rarg0 - newArr address
9598 // c_rarg1 - oldArr address
9599 // c_rarg2 - newIdx
9600 // c_rarg3 - shiftCount
9601 // c_rarg4 - numIter
9602 //
9603 address generate_bigIntegerRightShift() {
9604 StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id;
9605 int entry_count = StubInfo::entry_count(stub_id);
9606 assert(entry_count == 1, "sanity check");
9607 address start = load_archive_data(stub_id);
9608 if (start != nullptr) {
9609 return start;
9610 }
9611 __ align(CodeEntryAlignment);
9612 StubCodeMark mark(this, stub_id);
9613 start = __ pc();
9614
9615 Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit;
9616
9617 Register newArr = c_rarg0;
9618 Register oldArr = c_rarg1;
9619 Register newIdx = c_rarg2;
9620 Register shiftCount = c_rarg3;
9621 Register numIter = c_rarg4;
9622 Register idx = numIter;
9623
9624 Register newArrCur = rscratch1;
9625 Register shiftRevCount = rscratch2;
9626 Register oldArrCur = r13;
9627 Register oldArrNext = r14;
9628
9629 FloatRegister oldElem0 = v0;
9630 FloatRegister oldElem1 = v1;
9631 FloatRegister newElem = v2;
9632 FloatRegister shiftVCount = v3;
9633 FloatRegister shiftVRevCount = v4;
9634
9635 __ cbz(idx, Exit);
9636
9637 __ add(newArr, newArr, newIdx, Assembler::LSL, 2);
9638
9639 // left shift count
9640 __ movw(shiftRevCount, 32);
9641 __ subw(shiftRevCount, shiftRevCount, shiftCount);
9642
9643 // numIter too small to allow a 4-words SIMD loop, rolling back
9644 __ cmp(numIter, (u1)4);
9645 __ br(Assembler::LT, ShiftThree);
9646
9647 __ dup(shiftVCount, __ T4S, shiftCount);
9648 __ dup(shiftVRevCount, __ T4S, shiftRevCount);
9649 __ negr(shiftVCount, __ T4S, shiftVCount);
9650
9651 __ BIND(ShiftSIMDLoop);
9652
9653 // Calculate the load addresses
9654 __ sub(idx, idx, 4);
9655 __ add(oldArrNext, oldArr, idx, Assembler::LSL, 2);
9656 __ add(newArrCur, newArr, idx, Assembler::LSL, 2);
9657 __ add(oldArrCur, oldArrNext, 4);
9658
9659 // Load 4 words and process
9660 __ ld1(oldElem0, __ T4S, Address(oldArrCur));
9661 __ ld1(oldElem1, __ T4S, Address(oldArrNext));
9662 __ ushl(oldElem0, __ T4S, oldElem0, shiftVCount);
9663 __ ushl(oldElem1, __ T4S, oldElem1, shiftVRevCount);
9664 __ orr(newElem, __ T16B, oldElem0, oldElem1);
9665 __ st1(newElem, __ T4S, Address(newArrCur));
9666
9667 __ cmp(idx, (u1)4);
9668 __ br(Assembler::LT, ShiftTwoLoop);
9669 __ b(ShiftSIMDLoop);
9670
9671 __ BIND(ShiftTwoLoop);
9672 __ cbz(idx, Exit);
9673 __ cmp(idx, (u1)1);
9674 __ br(Assembler::EQ, ShiftOne);
9675
9676 // Calculate the load addresses
9677 __ sub(idx, idx, 2);
9678 __ add(oldArrNext, oldArr, idx, Assembler::LSL, 2);
9679 __ add(newArrCur, newArr, idx, Assembler::LSL, 2);
9680 __ add(oldArrCur, oldArrNext, 4);
9681
9682 // Load 2 words and process
9683 __ ld1(oldElem0, __ T2S, Address(oldArrCur));
9684 __ ld1(oldElem1, __ T2S, Address(oldArrNext));
9685 __ ushl(oldElem0, __ T2S, oldElem0, shiftVCount);
9686 __ ushl(oldElem1, __ T2S, oldElem1, shiftVRevCount);
9687 __ orr(newElem, __ T8B, oldElem0, oldElem1);
9688 __ st1(newElem, __ T2S, Address(newArrCur));
9689 __ b(ShiftTwoLoop);
9690
9691 __ BIND(ShiftThree);
9692 __ tbz(idx, 1, ShiftOne);
9693 __ tbz(idx, 0, ShiftTwo);
9694 __ ldrw(r10, Address(oldArr, 12));
9695 __ ldrw(r11, Address(oldArr, 8));
9696 __ lsrvw(r10, r10, shiftCount);
9697 __ lslvw(r11, r11, shiftRevCount);
9698 __ orrw(r12, r10, r11);
9699 __ strw(r12, Address(newArr, 8));
9700
9701 __ BIND(ShiftTwo);
9702 __ ldrw(r10, Address(oldArr, 8));
9703 __ ldrw(r11, Address(oldArr, 4));
9704 __ lsrvw(r10, r10, shiftCount);
9705 __ lslvw(r11, r11, shiftRevCount);
9706 __ orrw(r12, r10, r11);
9707 __ strw(r12, Address(newArr, 4));
9708
9709 __ BIND(ShiftOne);
9710 __ ldrw(r10, Address(oldArr, 4));
9711 __ ldrw(r11, Address(oldArr));
9712 __ lsrvw(r10, r10, shiftCount);
9713 __ lslvw(r11, r11, shiftRevCount);
9714 __ orrw(r12, r10, r11);
9715 __ strw(r12, Address(newArr));
9716
9717 __ BIND(Exit);
9718 __ ret(lr);
9719
9720 // record the stub entry and end
9721 store_archive_data(stub_id, start, __ pc());
9722
9723 return start;
9724 }
9725
9726 // Arguments:
9727 //
9728 // Input:
9729 // c_rarg0 - newArr address
9730 // c_rarg1 - oldArr address
9731 // c_rarg2 - newIdx
9732 // c_rarg3 - shiftCount
9733 // c_rarg4 - numIter
9734 //
9735 address generate_bigIntegerLeftShift() {
9736 StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id;
9737 int entry_count = StubInfo::entry_count(stub_id);
9738 assert(entry_count == 1, "sanity check");
9739 address start = load_archive_data(stub_id);
9740 if (start != nullptr) {
9741 return start;
9742 }
9743 __ align(CodeEntryAlignment);
9744 StubCodeMark mark(this, stub_id);
9745 start = __ pc();
9746
9747 Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit;
9748
9749 Register newArr = c_rarg0;
9750 Register oldArr = c_rarg1;
9751 Register newIdx = c_rarg2;
9752 Register shiftCount = c_rarg3;
9753 Register numIter = c_rarg4;
9754
9755 Register shiftRevCount = rscratch1;
9756 Register oldArrNext = rscratch2;
9757
9758 FloatRegister oldElem0 = v0;
9759 FloatRegister oldElem1 = v1;
9760 FloatRegister newElem = v2;
9761 FloatRegister shiftVCount = v3;
9762 FloatRegister shiftVRevCount = v4;
9763
9764 __ cbz(numIter, Exit);
9765
9766 __ add(oldArrNext, oldArr, 4);
9767 __ add(newArr, newArr, newIdx, Assembler::LSL, 2);
9768
9769 // right shift count
9770 __ movw(shiftRevCount, 32);
9771 __ subw(shiftRevCount, shiftRevCount, shiftCount);
9772
9773 // numIter too small to allow a 4-words SIMD loop, rolling back
9774 __ cmp(numIter, (u1)4);
9775 __ br(Assembler::LT, ShiftThree);
9776
9777 __ dup(shiftVCount, __ T4S, shiftCount);
9778 __ dup(shiftVRevCount, __ T4S, shiftRevCount);
9779 __ negr(shiftVRevCount, __ T4S, shiftVRevCount);
9780
9781 __ BIND(ShiftSIMDLoop);
9782
9783 // load 4 words and process
9784 __ ld1(oldElem0, __ T4S, __ post(oldArr, 16));
9785 __ ld1(oldElem1, __ T4S, __ post(oldArrNext, 16));
9786 __ ushl(oldElem0, __ T4S, oldElem0, shiftVCount);
9787 __ ushl(oldElem1, __ T4S, oldElem1, shiftVRevCount);
9788 __ orr(newElem, __ T16B, oldElem0, oldElem1);
9789 __ st1(newElem, __ T4S, __ post(newArr, 16));
9790 __ sub(numIter, numIter, 4);
9791
9792 __ cmp(numIter, (u1)4);
9793 __ br(Assembler::LT, ShiftTwoLoop);
9794 __ b(ShiftSIMDLoop);
9795
9796 __ BIND(ShiftTwoLoop);
9797 __ cbz(numIter, Exit);
9798 __ cmp(numIter, (u1)1);
9799 __ br(Assembler::EQ, ShiftOne);
9800
9801 // load 2 words and process
9802 __ ld1(oldElem0, __ T2S, __ post(oldArr, 8));
9803 __ ld1(oldElem1, __ T2S, __ post(oldArrNext, 8));
9804 __ ushl(oldElem0, __ T2S, oldElem0, shiftVCount);
9805 __ ushl(oldElem1, __ T2S, oldElem1, shiftVRevCount);
9806 __ orr(newElem, __ T8B, oldElem0, oldElem1);
9807 __ st1(newElem, __ T2S, __ post(newArr, 8));
9808 __ sub(numIter, numIter, 2);
9809 __ b(ShiftTwoLoop);
9810
9811 __ BIND(ShiftThree);
9812 __ ldrw(r10, __ post(oldArr, 4));
9813 __ ldrw(r11, __ post(oldArrNext, 4));
9814 __ lslvw(r10, r10, shiftCount);
9815 __ lsrvw(r11, r11, shiftRevCount);
9816 __ orrw(r12, r10, r11);
9817 __ strw(r12, __ post(newArr, 4));
9818 __ tbz(numIter, 1, Exit);
9819 __ tbz(numIter, 0, ShiftOne);
9820
9821 __ BIND(ShiftTwo);
9822 __ ldrw(r10, __ post(oldArr, 4));
9823 __ ldrw(r11, __ post(oldArrNext, 4));
9824 __ lslvw(r10, r10, shiftCount);
9825 __ lsrvw(r11, r11, shiftRevCount);
9826 __ orrw(r12, r10, r11);
9827 __ strw(r12, __ post(newArr, 4));
9828
9829 __ BIND(ShiftOne);
9830 __ ldrw(r10, Address(oldArr));
9831 __ ldrw(r11, Address(oldArrNext));
9832 __ lslvw(r10, r10, shiftCount);
9833 __ lsrvw(r11, r11, shiftRevCount);
9834 __ orrw(r12, r10, r11);
9835 __ strw(r12, Address(newArr));
9836
9837 __ BIND(Exit);
9838 __ ret(lr);
9839
9840 // record the stub entry and end
9841 store_archive_data(stub_id, start, __ pc());
9842
9843 return start;
9844 }
9845
9846 address generate_count_positives(address &count_positives_long) {
9847 StubId stub_id = StubId::stubgen_count_positives_id;
9848 GrowableArray<address> entries;
9849 int entry_count = StubInfo::entry_count(stub_id);
9850 // We have an extra entry for count_positives_long.
9851 assert(entry_count == 2, "sanity check");
9852 address start = load_archive_data(stub_id, &entries);
9853 if (start != nullptr) {
9854 assert(entries.length() == 1,
9855 "unexpected extra entry count %d", entries.length());
9856 count_positives_long = entries.at(0);
9857 return start;
9858 }
9859 const u1 large_loop_size = 64;
9860 const uint64_t UPPER_BIT_MASK=0x8080808080808080;
9861 int dcache_line = VM_Version::dcache_line_size();
9862
9863 Register ary1 = r1, len = r2, result = r0;
9864
9865 __ align(CodeEntryAlignment);
9866 StubCodeMark mark(this, stub_id);
9867
9868 address entry = __ pc();
9869
9870 __ enter();
9871 // precondition: a copy of len is already in result
9872 // __ mov(result, len);
9873
9874 Label RET_ADJUST, RET_ADJUST_16, RET_ADJUST_LONG, RET_NO_POP, RET_LEN, ALIGNED, LOOP16, CHECK_16,
9875 LARGE_LOOP, POST_LOOP16, LEN_OVER_15, LEN_OVER_8, POST_LOOP16_LOAD_TAIL;
9876
9877 __ cmp(len, (u1)15);
9878 __ br(Assembler::GT, LEN_OVER_15);
9879 // The only case when execution falls into this code is when pointer is near
9880 // the end of memory page and we have to avoid reading next page
9881 __ add(ary1, ary1, len);
9882 __ subs(len, len, 8);
9883 __ br(Assembler::GT, LEN_OVER_8);
9884 __ ldr(rscratch2, Address(ary1, -8));
9885 __ sub(rscratch1, zr, len, __ LSL, 3); // LSL 3 is to get bits from bytes.
9886 __ lsrv(rscratch2, rscratch2, rscratch1);
9887 __ tst(rscratch2, UPPER_BIT_MASK);
9888 __ csel(result, zr, result, Assembler::NE);
9889 __ leave();
9890 __ ret(lr);
9891 __ bind(LEN_OVER_8);
9892 __ ldp(rscratch1, rscratch2, Address(ary1, -16));
9893 __ sub(len, len, 8); // no data dep., then sub can be executed while loading
9894 __ tst(rscratch2, UPPER_BIT_MASK);
9895 __ br(Assembler::NE, RET_NO_POP);
9896 __ sub(rscratch2, zr, len, __ LSL, 3); // LSL 3 is to get bits from bytes
9897 __ lsrv(rscratch1, rscratch1, rscratch2);
9898 __ tst(rscratch1, UPPER_BIT_MASK);
9899 __ bind(RET_NO_POP);
9900 __ csel(result, zr, result, Assembler::NE);
9901 __ leave();
9902 __ ret(lr);
9903
9904 Register tmp1 = r3, tmp2 = r4, tmp3 = r5, tmp4 = r6, tmp5 = r7, tmp6 = r10;
9905 const RegSet spilled_regs = RegSet::range(tmp1, tmp5) + tmp6;
9906
9907 count_positives_long = __ pc(); // 2nd entry point
9908 entries.append(count_positives_long);
9909
9910 __ enter();
9911
9912 __ bind(LEN_OVER_15);
9913 __ push(spilled_regs, sp);
9914 __ andr(rscratch2, ary1, 15); // check pointer for 16-byte alignment
9915 __ cbz(rscratch2, ALIGNED);
9916 __ ldp(tmp6, tmp1, Address(ary1));
9917 __ mov(tmp5, 16);
9918 __ sub(rscratch1, tmp5, rscratch2); // amount of bytes until aligned address
9919 __ add(ary1, ary1, rscratch1);
9920 __ orr(tmp6, tmp6, tmp1);
9921 __ tst(tmp6, UPPER_BIT_MASK);
9922 __ br(Assembler::NE, RET_ADJUST);
9923 __ sub(len, len, rscratch1);
9924
9925 __ bind(ALIGNED);
9926 __ cmp(len, large_loop_size);
9927 __ br(Assembler::LT, CHECK_16);
9928 // Perform 16-byte load as early return in pre-loop to handle situation
9929 // when initially aligned large array has negative values at starting bytes,
9930 // so LARGE_LOOP would do 4 reads instead of 1 (in worst case), which is
9931 // slower. Cases with negative bytes further ahead won't be affected that
9932 // much. In fact, it'll be faster due to early loads, less instructions and
9933 // less branches in LARGE_LOOP.
9934 __ ldp(tmp6, tmp1, Address(__ post(ary1, 16)));
9935 __ sub(len, len, 16);
9936 __ orr(tmp6, tmp6, tmp1);
9937 __ tst(tmp6, UPPER_BIT_MASK);
9938 __ br(Assembler::NE, RET_ADJUST_16);
9939 __ cmp(len, large_loop_size);
9940 __ br(Assembler::LT, CHECK_16);
9941
9942 if (SoftwarePrefetchHintDistance >= 0
9943 && SoftwarePrefetchHintDistance >= dcache_line) {
9944 // initial prefetch
9945 __ prfm(Address(ary1, SoftwarePrefetchHintDistance - dcache_line));
9946 }
9947 __ bind(LARGE_LOOP);
9948 if (SoftwarePrefetchHintDistance >= 0) {
9949 __ prfm(Address(ary1, SoftwarePrefetchHintDistance));
9950 }
9951 // Issue load instructions first, since it can save few CPU/MEM cycles, also
9952 // instead of 4 triples of "orr(...), addr(...);cbnz(...);" (for each ldp)
9953 // better generate 7 * orr(...) + 1 andr(...) + 1 cbnz(...) which saves 3
9954 // instructions per cycle and have less branches, but this approach disables
9955 // early return, thus, all 64 bytes are loaded and checked every time.
9956 __ ldp(tmp2, tmp3, Address(ary1));
9957 __ ldp(tmp4, tmp5, Address(ary1, 16));
9958 __ ldp(rscratch1, rscratch2, Address(ary1, 32));
9959 __ ldp(tmp6, tmp1, Address(ary1, 48));
9960 __ add(ary1, ary1, large_loop_size);
9961 __ sub(len, len, large_loop_size);
9962 __ orr(tmp2, tmp2, tmp3);
9963 __ orr(tmp4, tmp4, tmp5);
9964 __ orr(rscratch1, rscratch1, rscratch2);
9965 __ orr(tmp6, tmp6, tmp1);
9966 __ orr(tmp2, tmp2, tmp4);
9967 __ orr(rscratch1, rscratch1, tmp6);
9968 __ orr(tmp2, tmp2, rscratch1);
9969 __ tst(tmp2, UPPER_BIT_MASK);
9970 __ br(Assembler::NE, RET_ADJUST_LONG);
9971 __ cmp(len, large_loop_size);
9972 __ br(Assembler::GE, LARGE_LOOP);
9973
9974 __ bind(CHECK_16); // small 16-byte load pre-loop
9975 __ cmp(len, (u1)16);
9976 __ br(Assembler::LT, POST_LOOP16);
9977
9978 __ bind(LOOP16); // small 16-byte load loop
9979 __ ldp(tmp2, tmp3, Address(__ post(ary1, 16)));
9980 __ sub(len, len, 16);
9981 __ orr(tmp2, tmp2, tmp3);
9982 __ tst(tmp2, UPPER_BIT_MASK);
9983 __ br(Assembler::NE, RET_ADJUST_16);
9984 __ cmp(len, (u1)16);
9985 __ br(Assembler::GE, LOOP16); // 16-byte load loop end
9986
9987 __ bind(POST_LOOP16); // 16-byte aligned, so we can read unconditionally
9988 __ cmp(len, (u1)8);
9989 __ br(Assembler::LE, POST_LOOP16_LOAD_TAIL);
9990 __ ldr(tmp3, Address(__ post(ary1, 8)));
9991 __ tst(tmp3, UPPER_BIT_MASK);
9992 __ br(Assembler::NE, RET_ADJUST);
9993 __ sub(len, len, 8);
9994
9995 __ bind(POST_LOOP16_LOAD_TAIL);
9996 __ cbz(len, RET_LEN); // Can't shift left by 64 when len==0
9997 __ ldr(tmp1, Address(ary1));
9998 __ mov(tmp2, 64);
9999 __ sub(tmp4, tmp2, len, __ LSL, 3);
10000 __ lslv(tmp1, tmp1, tmp4);
10001 __ tst(tmp1, UPPER_BIT_MASK);
10002 __ br(Assembler::NE, RET_ADJUST);
10003 // Fallthrough
10004
10005 __ bind(RET_LEN);
10006 __ pop(spilled_regs, sp);
10007 __ leave();
10008 __ ret(lr);
10009
10010 // difference result - len is the count of guaranteed to be
10011 // positive bytes
10012
10013 __ bind(RET_ADJUST_LONG);
10014 __ add(len, len, (u1)(large_loop_size - 16));
10015 __ bind(RET_ADJUST_16);
10016 __ add(len, len, 16);
10017 __ bind(RET_ADJUST);
10018 __ pop(spilled_regs, sp);
10019 __ leave();
10020 __ sub(result, result, len);
10021 __ ret(lr);
10022
10023 // record the stub entry and end plus the extra entry
10024 store_archive_data(stub_id, entry, __ pc(), &entries);
10025
10026 return entry;
10027 }
10028
10029 void generate_large_array_equals_loop_nonsimd(int loopThreshold,
10030 bool usePrefetch, Label &NOT_EQUAL) {
10031 Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
10032 tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11,
10033 tmp7 = r12, tmp8 = r13;
10034 Label LOOP;
10035
10036 __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize)));
10037 __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize)));
10038 __ bind(LOOP);
10039 if (usePrefetch) {
10040 __ prfm(Address(a1, SoftwarePrefetchHintDistance));
10041 __ prfm(Address(a2, SoftwarePrefetchHintDistance));
10042 }
10043 __ ldp(tmp5, tmp7, Address(__ post(a1, 2 * wordSize)));
10044 __ eor(tmp1, tmp1, tmp2);
10045 __ eor(tmp3, tmp3, tmp4);
10046 __ ldp(tmp6, tmp8, Address(__ post(a2, 2 * wordSize)));
10047 __ orr(tmp1, tmp1, tmp3);
10048 __ cbnz(tmp1, NOT_EQUAL);
10049 __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize)));
10050 __ eor(tmp5, tmp5, tmp6);
10051 __ eor(tmp7, tmp7, tmp8);
10052 __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize)));
10053 __ orr(tmp5, tmp5, tmp7);
10054 __ cbnz(tmp5, NOT_EQUAL);
10055 __ ldp(tmp5, tmp7, Address(__ post(a1, 2 * wordSize)));
10056 __ eor(tmp1, tmp1, tmp2);
10057 __ eor(tmp3, tmp3, tmp4);
10058 __ ldp(tmp6, tmp8, Address(__ post(a2, 2 * wordSize)));
10059 __ orr(tmp1, tmp1, tmp3);
10060 __ cbnz(tmp1, NOT_EQUAL);
10061 __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize)));
10062 __ eor(tmp5, tmp5, tmp6);
10063 __ sub(cnt1, cnt1, 8 * wordSize);
10064 __ eor(tmp7, tmp7, tmp8);
10065 __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize)));
10066 // tmp6 is not used. MacroAssembler::subs is used here (rather than
10067 // cmp) because subs allows an unlimited range of immediate operand.
10068 __ subs(tmp6, cnt1, loopThreshold);
10069 __ orr(tmp5, tmp5, tmp7);
10070 __ cbnz(tmp5, NOT_EQUAL);
10071 __ br(__ GE, LOOP);
10072 // post-loop
10073 __ eor(tmp1, tmp1, tmp2);
10074 __ eor(tmp3, tmp3, tmp4);
10075 __ orr(tmp1, tmp1, tmp3);
10076 __ sub(cnt1, cnt1, 2 * wordSize);
10077 __ cbnz(tmp1, NOT_EQUAL);
10078 }
10079
10080 void generate_large_array_equals_loop_simd(int loopThreshold,
10081 bool usePrefetch, Label &NOT_EQUAL) {
10082 Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
10083 tmp2 = rscratch2;
10084 Label LOOP;
10085
10086 __ bind(LOOP);
10087 if (usePrefetch) {
10088 __ prfm(Address(a1, SoftwarePrefetchHintDistance));
10089 __ prfm(Address(a2, SoftwarePrefetchHintDistance));
10090 }
10091 __ ld1(v0, v1, v2, v3, __ T2D, Address(__ post(a1, 4 * 2 * wordSize)));
10092 __ sub(cnt1, cnt1, 8 * wordSize);
10093 __ ld1(v4, v5, v6, v7, __ T2D, Address(__ post(a2, 4 * 2 * wordSize)));
10094 __ subs(tmp1, cnt1, loopThreshold);
10095 __ eor(v0, __ T16B, v0, v4);
10096 __ eor(v1, __ T16B, v1, v5);
10097 __ eor(v2, __ T16B, v2, v6);
10098 __ eor(v3, __ T16B, v3, v7);
10099 __ orr(v0, __ T16B, v0, v1);
10100 __ orr(v1, __ T16B, v2, v3);
10101 __ orr(v0, __ T16B, v0, v1);
10102 __ umov(tmp1, v0, __ D, 0);
10103 __ umov(tmp2, v0, __ D, 1);
10104 __ orr(tmp1, tmp1, tmp2);
10105 __ cbnz(tmp1, NOT_EQUAL);
10106 __ br(__ GE, LOOP);
10107 }
10108
10109 // a1 = r1 - array1 address
10110 // a2 = r2 - array2 address
10111 // result = r0 - return value. Already contains "false"
10112 // cnt1 = r10 - amount of elements left to check, reduced by wordSize
10113 // r3-r5 are reserved temporary registers
10114 // Clobbers: v0-v7 when UseSIMDForArrayEquals, rscratch1, rscratch2
10115 address generate_large_array_equals() {
10116 StubId stub_id = StubId::stubgen_large_array_equals_id;
10117 int entry_count = StubInfo::entry_count(stub_id);
10118 assert(entry_count == 1, "sanity check");
10119 address start = load_archive_data(stub_id);
10120 if (start != nullptr) {
10121 return start;
10122 }
10123 Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
10124 tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11,
10125 tmp7 = r12, tmp8 = r13;
10126 Label TAIL, NOT_EQUAL, EQUAL, NOT_EQUAL_NO_POP, NO_PREFETCH_LARGE_LOOP,
10127 SMALL_LOOP, POST_LOOP;
10128 const int PRE_LOOP_SIZE = UseSIMDForArrayEquals ? 0 : 16;
10129 // calculate if at least 32 prefetched bytes are used
10130 int prefetchLoopThreshold = SoftwarePrefetchHintDistance + 32;
10131 int nonPrefetchLoopThreshold = (64 + PRE_LOOP_SIZE);
10132 RegSet spilled_regs = RegSet::range(tmp6, tmp8);
10133 assert_different_registers(a1, a2, result, cnt1, tmp1, tmp2, tmp3, tmp4,
10134 tmp5, tmp6, tmp7, tmp8);
10135
10136 __ align(CodeEntryAlignment);
10137
10138 StubCodeMark mark(this, stub_id);
10139
10140 address entry = __ pc();
10141 __ enter();
10142 __ sub(cnt1, cnt1, wordSize); // first 8 bytes were loaded outside of stub
10143 // also advance pointers to use post-increment instead of pre-increment
10144 __ add(a1, a1, wordSize);
10145 __ add(a2, a2, wordSize);
10146 if (AvoidUnalignedAccesses) {
10147 // both implementations (SIMD/nonSIMD) are using relatively large load
10148 // instructions (ld1/ldp), which has huge penalty (up to x2 exec time)
10149 // on some CPUs in case of address is not at least 16-byte aligned.
10150 // Arrays are 8-byte aligned currently, so, we can make additional 8-byte
10151 // load if needed at least for 1st address and make if 16-byte aligned.
10152 Label ALIGNED16;
10153 __ tbz(a1, 3, ALIGNED16);
10154 __ ldr(tmp1, Address(__ post(a1, wordSize)));
10155 __ ldr(tmp2, Address(__ post(a2, wordSize)));
10156 __ sub(cnt1, cnt1, wordSize);
10157 __ eor(tmp1, tmp1, tmp2);
10158 __ cbnz(tmp1, NOT_EQUAL_NO_POP);
10159 __ bind(ALIGNED16);
10160 }
10161 if (UseSIMDForArrayEquals) {
10162 if (SoftwarePrefetchHintDistance >= 0) {
10163 __ subs(tmp1, cnt1, prefetchLoopThreshold);
10164 __ br(__ LE, NO_PREFETCH_LARGE_LOOP);
10165 generate_large_array_equals_loop_simd(prefetchLoopThreshold,
10166 /* prfm = */ true, NOT_EQUAL);
10167 __ subs(zr, cnt1, nonPrefetchLoopThreshold);
10168 __ br(__ LT, TAIL);
10169 }
10170 __ bind(NO_PREFETCH_LARGE_LOOP);
10171 generate_large_array_equals_loop_simd(nonPrefetchLoopThreshold,
10172 /* prfm = */ false, NOT_EQUAL);
10173 } else {
10174 __ push(spilled_regs, sp);
10175 if (SoftwarePrefetchHintDistance >= 0) {
10176 __ subs(tmp1, cnt1, prefetchLoopThreshold);
10177 __ br(__ LE, NO_PREFETCH_LARGE_LOOP);
10178 generate_large_array_equals_loop_nonsimd(prefetchLoopThreshold,
10179 /* prfm = */ true, NOT_EQUAL);
10180 __ subs(zr, cnt1, nonPrefetchLoopThreshold);
10181 __ br(__ LT, TAIL);
10182 }
10183 __ bind(NO_PREFETCH_LARGE_LOOP);
10184 generate_large_array_equals_loop_nonsimd(nonPrefetchLoopThreshold,
10185 /* prfm = */ false, NOT_EQUAL);
10186 }
10187 __ bind(TAIL);
10188 __ cbz(cnt1, EQUAL);
10189 __ subs(cnt1, cnt1, wordSize);
10190 __ br(__ LE, POST_LOOP);
10191 __ bind(SMALL_LOOP);
10192 __ ldr(tmp1, Address(__ post(a1, wordSize)));
10193 __ ldr(tmp2, Address(__ post(a2, wordSize)));
10194 __ subs(cnt1, cnt1, wordSize);
10195 __ eor(tmp1, tmp1, tmp2);
10196 __ cbnz(tmp1, NOT_EQUAL);
10197 __ br(__ GT, SMALL_LOOP);
10198 __ bind(POST_LOOP);
10199 __ ldr(tmp1, Address(a1, cnt1));
10200 __ ldr(tmp2, Address(a2, cnt1));
10201 __ eor(tmp1, tmp1, tmp2);
10202 __ cbnz(tmp1, NOT_EQUAL);
10203 __ bind(EQUAL);
10204 __ mov(result, true);
10205 __ bind(NOT_EQUAL);
10206 if (!UseSIMDForArrayEquals) {
10207 __ pop(spilled_regs, sp);
10208 }
10209 __ bind(NOT_EQUAL_NO_POP);
10210 __ leave();
10211 __ ret(lr);
10212
10213 // record the stub entry and end
10214 store_archive_data(stub_id, entry, __ pc());
10215
10216 return entry;
10217 }
10218
10219 // result = r0 - return value. Contains initial hashcode value on entry.
10220 // ary = r1 - array address
10221 // cnt = r2 - elements count
10222 // Clobbers: v0-v13, rscratch1, rscratch2
10223 address generate_large_arrays_hashcode(BasicType eltype) {
10224 StubId stub_id;
10225 switch (eltype) {
10226 case T_BOOLEAN:
10227 stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id;
10228 break;
10229 case T_BYTE:
10230 stub_id = StubId::stubgen_large_arrays_hashcode_byte_id;
10231 break;
10232 case T_CHAR:
10233 stub_id = StubId::stubgen_large_arrays_hashcode_char_id;
10234 break;
10235 case T_SHORT:
10236 stub_id = StubId::stubgen_large_arrays_hashcode_short_id;
10237 break;
10238 case T_INT:
10239 stub_id = StubId::stubgen_large_arrays_hashcode_int_id;
10240 break;
10241 default:
10242 stub_id = StubId::NO_STUBID;
10243 ShouldNotReachHere();
10244 };
10245 int entry_count = StubInfo::entry_count(stub_id);
10246 assert(entry_count == 1, "sanity check");
10247 address start = load_archive_data(stub_id);
10248 if (start != nullptr) {
10249 return start;
10250 }
10251 const Register result = r0, ary = r1, cnt = r2;
10252 const FloatRegister vdata0 = v3, vdata1 = v2, vdata2 = v1, vdata3 = v0;
10253 const FloatRegister vmul0 = v4, vmul1 = v5, vmul2 = v6, vmul3 = v7;
10254 const FloatRegister vpow = v12; // powers of 31: <31^3, ..., 31^0>
10255 const FloatRegister vpowm = v13;
10256
10257 ARRAYS_HASHCODE_REGISTERS;
10258
10259 Label SMALL_LOOP, LARGE_LOOP_PREHEADER, LARGE_LOOP, TAIL, TAIL_SHORTCUT, BR_BASE;
10260
10261 unsigned int vf; // vectorization factor
10262 bool multiply_by_halves;
10263 Assembler::SIMD_Arrangement load_arrangement;
10264 switch (eltype) {
10265 case T_BOOLEAN:
10266 case T_BYTE:
10267 load_arrangement = Assembler::T8B;
10268 multiply_by_halves = true;
10269 vf = 8;
10270 break;
10271 case T_CHAR:
10272 case T_SHORT:
10273 load_arrangement = Assembler::T8H;
10274 multiply_by_halves = true;
10275 vf = 8;
10276 break;
10277 case T_INT:
10278 load_arrangement = Assembler::T4S;
10279 multiply_by_halves = false;
10280 vf = 4;
10281 break;
10282 default:
10283 ShouldNotReachHere();
10284 }
10285
10286 // Unroll factor
10287 const unsigned uf = 4;
10288
10289 // Effective vectorization factor
10290 const unsigned evf = vf * uf;
10291
10292 __ align(CodeEntryAlignment);
10293
10294 StubCodeMark mark(this, stub_id);
10295
10296 address entry = __ pc();
10297 __ enter();
10298
10299 // Put 0-3'th powers of 31 into a single SIMD register together. The register will be used in
10300 // the SMALL and LARGE LOOPS' epilogues. The initialization is hoisted here and the register's
10301 // value shouldn't change throughout both loops.
10302 __ movw(rscratch1, intpow(31U, 3));
10303 __ mov(vpow, Assembler::S, 0, rscratch1);
10304 __ movw(rscratch1, intpow(31U, 2));
10305 __ mov(vpow, Assembler::S, 1, rscratch1);
10306 __ movw(rscratch1, intpow(31U, 1));
10307 __ mov(vpow, Assembler::S, 2, rscratch1);
10308 __ movw(rscratch1, intpow(31U, 0));
10309 __ mov(vpow, Assembler::S, 3, rscratch1);
10310
10311 __ mov(vmul0, Assembler::T16B, 0);
10312 __ mov(vmul0, Assembler::S, 3, result);
10313
10314 __ andr(rscratch2, cnt, (uf - 1) * vf);
10315 __ cbz(rscratch2, LARGE_LOOP_PREHEADER);
10316
10317 __ movw(rscratch1, intpow(31U, multiply_by_halves ? vf / 2 : vf));
10318 __ mov(vpowm, Assembler::S, 0, rscratch1);
10319
10320 // SMALL LOOP
10321 __ bind(SMALL_LOOP);
10322
10323 __ ld1(vdata0, load_arrangement, Address(__ post(ary, vf * type2aelembytes(eltype))));
10324 __ mulvs(vmul0, Assembler::T4S, vmul0, vpowm, 0);
10325 __ subsw(rscratch2, rscratch2, vf);
10326
10327 if (load_arrangement == Assembler::T8B) {
10328 // Extend 8B to 8H to be able to use vector multiply
10329 // instructions
10330 assert(load_arrangement == Assembler::T8B, "expected to extend 8B to 8H");
10331 if (is_signed_subword_type(eltype)) {
10332 __ sxtl(vdata0, Assembler::T8H, vdata0, load_arrangement);
10333 } else {
10334 __ uxtl(vdata0, Assembler::T8H, vdata0, load_arrangement);
10335 }
10336 }
10337
10338 switch (load_arrangement) {
10339 case Assembler::T4S:
10340 __ addv(vmul0, load_arrangement, vmul0, vdata0);
10341 break;
10342 case Assembler::T8B:
10343 case Assembler::T8H:
10344 assert(is_subword_type(eltype), "subword type expected");
10345 if (is_signed_subword_type(eltype)) {
10346 __ saddwv(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T4H);
10347 } else {
10348 __ uaddwv(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T4H);
10349 }
10350 break;
10351 default:
10352 __ should_not_reach_here();
10353 }
10354
10355 // Process the upper half of a vector
10356 if (load_arrangement == Assembler::T8B || load_arrangement == Assembler::T8H) {
10357 __ mulvs(vmul0, Assembler::T4S, vmul0, vpowm, 0);
10358 if (is_signed_subword_type(eltype)) {
10359 __ saddwv2(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T8H);
10360 } else {
10361 __ uaddwv2(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T8H);
10362 }
10363 }
10364
10365 __ br(Assembler::HI, SMALL_LOOP);
10366
10367 // SMALL LOOP'S EPILOQUE
10368 __ lsr(rscratch2, cnt, exact_log2(evf));
10369 __ cbnz(rscratch2, LARGE_LOOP_PREHEADER);
10370
10371 __ mulv(vmul0, Assembler::T4S, vmul0, vpow);
10372 __ addv(vmul0, Assembler::T4S, vmul0);
10373 __ umov(result, vmul0, Assembler::S, 0);
10374
10375 // TAIL
10376 __ bind(TAIL);
10377
10378 // The andr performs cnt % vf. The subtract shifted by 3 offsets past vf - 1 - (cnt % vf) pairs
10379 // of load + madd insns i.e. it only executes cnt % vf load + madd pairs.
10380 assert(is_power_of_2(vf), "can't use this value to calculate the jump target PC");
10381 __ andr(rscratch2, cnt, vf - 1);
10382 __ bind(TAIL_SHORTCUT);
10383 __ adr(rscratch1, BR_BASE);
10384 // For Cortex-A53 offset is 4 because 2 nops are generated.
10385 __ sub(rscratch1, rscratch1, rscratch2, ext::uxtw, VM_Version::supports_a53mac() ? 4 : 3);
10386 __ movw(rscratch2, 0x1f);
10387 __ br(rscratch1);
10388
10389 for (size_t i = 0; i < vf - 1; ++i) {
10390 __ load(rscratch1, Address(__ post(ary, type2aelembytes(eltype))),
10391 eltype);
10392 __ maddw(result, result, rscratch2, rscratch1);
10393 // maddw generates an extra nop for Cortex-A53 (see maddw definition in macroAssembler).
10394 // Generate 2nd nop to have 4 instructions per iteration.
10395 if (VM_Version::supports_a53mac()) {
10396 __ nop();
10397 }
10398 }
10399 __ bind(BR_BASE);
10400
10401 __ leave();
10402 __ ret(lr);
10403
10404 // LARGE LOOP
10405 __ bind(LARGE_LOOP_PREHEADER);
10406
10407 __ lsr(rscratch2, cnt, exact_log2(evf));
10408
10409 if (multiply_by_halves) {
10410 // 31^4 - multiplier between lower and upper parts of a register
10411 __ movw(rscratch1, intpow(31U, vf / 2));
10412 __ mov(vpowm, Assembler::S, 1, rscratch1);
10413 // 31^28 - remainder of the iteraion multiplier, 28 = 32 - 4
10414 __ movw(rscratch1, intpow(31U, evf - vf / 2));
10415 __ mov(vpowm, Assembler::S, 0, rscratch1);
10416 } else {
10417 // 31^16
10418 __ movw(rscratch1, intpow(31U, evf));
10419 __ mov(vpowm, Assembler::S, 0, rscratch1);
10420 }
10421
10422 __ mov(vmul3, Assembler::T16B, 0);
10423 __ mov(vmul2, Assembler::T16B, 0);
10424 __ mov(vmul1, Assembler::T16B, 0);
10425
10426 __ bind(LARGE_LOOP);
10427
10428 __ mulvs(vmul3, Assembler::T4S, vmul3, vpowm, 0);
10429 __ mulvs(vmul2, Assembler::T4S, vmul2, vpowm, 0);
10430 __ mulvs(vmul1, Assembler::T4S, vmul1, vpowm, 0);
10431 __ mulvs(vmul0, Assembler::T4S, vmul0, vpowm, 0);
10432
10433 __ ld1(vdata3, vdata2, vdata1, vdata0, load_arrangement,
10434 Address(__ post(ary, evf * type2aelembytes(eltype))));
10435
10436 if (load_arrangement == Assembler::T8B) {
10437 // Extend 8B to 8H to be able to use vector multiply
10438 // instructions
10439 assert(load_arrangement == Assembler::T8B, "expected to extend 8B to 8H");
10440 if (is_signed_subword_type(eltype)) {
10441 __ sxtl(vdata3, Assembler::T8H, vdata3, load_arrangement);
10442 __ sxtl(vdata2, Assembler::T8H, vdata2, load_arrangement);
10443 __ sxtl(vdata1, Assembler::T8H, vdata1, load_arrangement);
10444 __ sxtl(vdata0, Assembler::T8H, vdata0, load_arrangement);
10445 } else {
10446 __ uxtl(vdata3, Assembler::T8H, vdata3, load_arrangement);
10447 __ uxtl(vdata2, Assembler::T8H, vdata2, load_arrangement);
10448 __ uxtl(vdata1, Assembler::T8H, vdata1, load_arrangement);
10449 __ uxtl(vdata0, Assembler::T8H, vdata0, load_arrangement);
10450 }
10451 }
10452
10453 switch (load_arrangement) {
10454 case Assembler::T4S:
10455 __ addv(vmul3, load_arrangement, vmul3, vdata3);
10456 __ addv(vmul2, load_arrangement, vmul2, vdata2);
10457 __ addv(vmul1, load_arrangement, vmul1, vdata1);
10458 __ addv(vmul0, load_arrangement, vmul0, vdata0);
10459 break;
10460 case Assembler::T8B:
10461 case Assembler::T8H:
10462 assert(is_subword_type(eltype), "subword type expected");
10463 if (is_signed_subword_type(eltype)) {
10464 __ saddwv(vmul3, vmul3, Assembler::T4S, vdata3, Assembler::T4H);
10465 __ saddwv(vmul2, vmul2, Assembler::T4S, vdata2, Assembler::T4H);
10466 __ saddwv(vmul1, vmul1, Assembler::T4S, vdata1, Assembler::T4H);
10467 __ saddwv(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T4H);
10468 } else {
10469 __ uaddwv(vmul3, vmul3, Assembler::T4S, vdata3, Assembler::T4H);
10470 __ uaddwv(vmul2, vmul2, Assembler::T4S, vdata2, Assembler::T4H);
10471 __ uaddwv(vmul1, vmul1, Assembler::T4S, vdata1, Assembler::T4H);
10472 __ uaddwv(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T4H);
10473 }
10474 break;
10475 default:
10476 __ should_not_reach_here();
10477 }
10478
10479 // Process the upper half of a vector
10480 if (load_arrangement == Assembler::T8B || load_arrangement == Assembler::T8H) {
10481 __ mulvs(vmul3, Assembler::T4S, vmul3, vpowm, 1);
10482 __ mulvs(vmul2, Assembler::T4S, vmul2, vpowm, 1);
10483 __ mulvs(vmul1, Assembler::T4S, vmul1, vpowm, 1);
10484 __ mulvs(vmul0, Assembler::T4S, vmul0, vpowm, 1);
10485 if (is_signed_subword_type(eltype)) {
10486 __ saddwv2(vmul3, vmul3, Assembler::T4S, vdata3, Assembler::T8H);
10487 __ saddwv2(vmul2, vmul2, Assembler::T4S, vdata2, Assembler::T8H);
10488 __ saddwv2(vmul1, vmul1, Assembler::T4S, vdata1, Assembler::T8H);
10489 __ saddwv2(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T8H);
10490 } else {
10491 __ uaddwv2(vmul3, vmul3, Assembler::T4S, vdata3, Assembler::T8H);
10492 __ uaddwv2(vmul2, vmul2, Assembler::T4S, vdata2, Assembler::T8H);
10493 __ uaddwv2(vmul1, vmul1, Assembler::T4S, vdata1, Assembler::T8H);
10494 __ uaddwv2(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T8H);
10495 }
10496 }
10497
10498 __ subsw(rscratch2, rscratch2, 1);
10499 __ br(Assembler::HI, LARGE_LOOP);
10500
10501 __ mulv(vmul3, Assembler::T4S, vmul3, vpow);
10502 __ addv(vmul3, Assembler::T4S, vmul3);
10503 __ umov(result, vmul3, Assembler::S, 0);
10504
10505 __ mov(rscratch2, intpow(31U, vf));
10506
10507 __ mulv(vmul2, Assembler::T4S, vmul2, vpow);
10508 __ addv(vmul2, Assembler::T4S, vmul2);
10509 __ umov(rscratch1, vmul2, Assembler::S, 0);
10510 __ maddw(result, result, rscratch2, rscratch1);
10511
10512 __ mulv(vmul1, Assembler::T4S, vmul1, vpow);
10513 __ addv(vmul1, Assembler::T4S, vmul1);
10514 __ umov(rscratch1, vmul1, Assembler::S, 0);
10515 __ maddw(result, result, rscratch2, rscratch1);
10516
10517 __ mulv(vmul0, Assembler::T4S, vmul0, vpow);
10518 __ addv(vmul0, Assembler::T4S, vmul0);
10519 __ umov(rscratch1, vmul0, Assembler::S, 0);
10520 __ maddw(result, result, rscratch2, rscratch1);
10521
10522 __ andr(rscratch2, cnt, vf - 1);
10523 __ cbnz(rscratch2, TAIL_SHORTCUT);
10524
10525 __ leave();
10526 __ ret(lr);
10527
10528 // record the stub entry and end
10529 store_archive_data(stub_id, entry, __ pc());
10530
10531 return entry;
10532 }
10533
10534 address generate_dsin_dcos(bool isCos) {
10535 StubId stub_id = (isCos ? StubId::stubgen_dcos_id : StubId::stubgen_dsin_id);
10536 int entry_count = StubInfo::entry_count(stub_id);
10537 assert(entry_count == 1, "sanity check");
10538 address start = load_archive_data(stub_id);
10539 if (start != nullptr) {
10540 return start;
10541 }
10542 __ align(CodeEntryAlignment);
10543 StubCodeMark mark(this, stub_id);
10544 start = __ pc();
10545 __ generate_dsin_dcos(isCos, (address)StubRoutines::aarch64::_npio2_hw,
10546 (address)StubRoutines::aarch64::_two_over_pi,
10547 (address)StubRoutines::aarch64::_pio2,
10548 (address)StubRoutines::aarch64::_dsin_coef,
10549 (address)StubRoutines::aarch64::_dcos_coef);
10550
10551 // record the stub entry and end
10552 store_archive_data(stub_id, start, __ pc());
10553
10554 return start;
10555 }
10556
10557 // code for comparing 16 characters of strings with Latin1 and Utf16 encoding
10558 void compare_string_16_x_LU(Register tmpL, Register tmpU, Label &DIFF1,
10559 Label &DIFF2) {
10560 Register cnt1 = r2, tmp2 = r11, tmp3 = r12;
10561 FloatRegister vtmp = v1, vtmpZ = v0, vtmp3 = v2;
10562
10563 __ ldrq(vtmp, Address(__ post(tmp2, 16)));
10564 __ ldr(tmpU, Address(__ post(cnt1, 8)));
10565 __ zip1(vtmp3, __ T16B, vtmp, vtmpZ);
10566 // now we have 32 bytes of characters (converted to U) in vtmp:vtmp3
10567
10568 __ fmovd(tmpL, vtmp3);
10569 __ eor(rscratch2, tmp3, tmpL);
10570 __ cbnz(rscratch2, DIFF2);
10571
10572 __ ldr(tmp3, Address(__ post(cnt1, 8)));
10573 __ umov(tmpL, vtmp3, __ D, 1);
10574 __ eor(rscratch2, tmpU, tmpL);
10575 __ cbnz(rscratch2, DIFF1);
10576
10577 __ zip2(vtmp, __ T16B, vtmp, vtmpZ);
10578 __ ldr(tmpU, Address(__ post(cnt1, 8)));
10579 __ fmovd(tmpL, vtmp);
10580 __ eor(rscratch2, tmp3, tmpL);
10581 __ cbnz(rscratch2, DIFF2);
10582
10583 __ ldr(tmp3, Address(__ post(cnt1, 8)));
10584 __ umov(tmpL, vtmp, __ D, 1);
10585 __ eor(rscratch2, tmpU, tmpL);
10586 __ cbnz(rscratch2, DIFF1);
10587 }
10588
10589 // r0 = result
10590 // r1 = str1
10591 // r2 = cnt1
10592 // r3 = str2
10593 // r4 = cnt2
10594 // r10 = tmp1
10595 // r11 = tmp2
10596 address generate_compare_long_string_different_encoding(bool isLU) {
10597 StubId stub_id = (isLU ? StubId::stubgen_compare_long_string_LU_id : StubId::stubgen_compare_long_string_UL_id);
10598 int entry_count = StubInfo::entry_count(stub_id);
10599 assert(entry_count == 1, "sanity check");
10600 address start = load_archive_data(stub_id);
10601 if (start != nullptr) {
10602 return start;
10603 }
10604 __ align(CodeEntryAlignment);
10605 StubCodeMark mark(this, stub_id);
10606 address entry = __ pc();
10607 Label SMALL_LOOP, TAIL, TAIL_LOAD_16, LOAD_LAST, DIFF1, DIFF2,
10608 DONE, CALCULATE_DIFFERENCE, LARGE_LOOP_PREFETCH, NO_PREFETCH,
10609 LARGE_LOOP_PREFETCH_REPEAT1, LARGE_LOOP_PREFETCH_REPEAT2;
10610 Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
10611 tmp1 = r10, tmp2 = r11, tmp3 = r12, tmp4 = r14;
10612 FloatRegister vtmpZ = v0, vtmp = v1, vtmp3 = v2;
10613 RegSet spilled_regs = RegSet::of(tmp3, tmp4);
10614
10615 int prefetchLoopExitCondition = MAX2(64, SoftwarePrefetchHintDistance/2);
10616
10617 __ eor(vtmpZ, __ T16B, vtmpZ, vtmpZ);
10618 // cnt2 == amount of characters left to compare
10619 // Check already loaded first 4 symbols(vtmp and tmp2(LU)/tmp1(UL))
10620 __ zip1(vtmp, __ T8B, vtmp, vtmpZ);
10621 __ add(str1, str1, isLU ? wordSize/2 : wordSize);
10622 __ add(str2, str2, isLU ? wordSize : wordSize/2);
10623 __ fmovd(isLU ? tmp1 : tmp2, vtmp);
10624 __ subw(cnt2, cnt2, 8); // Already loaded 4 symbols. Last 4 is special case.
10625 __ eor(rscratch2, tmp1, tmp2);
10626 __ mov(rscratch1, tmp2);
10627 __ cbnz(rscratch2, CALCULATE_DIFFERENCE);
10628 Register tmpU = isLU ? rscratch1 : tmp1, // where to keep U for comparison
10629 tmpL = isLU ? tmp1 : rscratch1; // where to keep L for comparison
10630 __ push(spilled_regs, sp);
10631 __ mov(tmp2, isLU ? str1 : str2); // init the pointer to L next load
10632 __ mov(cnt1, isLU ? str2 : str1); // init the pointer to U next load
10633
10634 __ ldr(tmp3, Address(__ post(cnt1, 8)));
10635
10636 if (SoftwarePrefetchHintDistance >= 0) {
10637 __ subs(rscratch2, cnt2, prefetchLoopExitCondition);
10638 __ br(__ LT, NO_PREFETCH);
10639 __ bind(LARGE_LOOP_PREFETCH);
10640 __ prfm(Address(tmp2, SoftwarePrefetchHintDistance));
10641 __ mov(tmp4, 2);
10642 __ prfm(Address(cnt1, SoftwarePrefetchHintDistance));
10643 __ bind(LARGE_LOOP_PREFETCH_REPEAT1);
10644 compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2);
10645 __ subs(tmp4, tmp4, 1);
10646 __ br(__ GT, LARGE_LOOP_PREFETCH_REPEAT1);
10647 __ prfm(Address(cnt1, SoftwarePrefetchHintDistance));
10648 __ mov(tmp4, 2);
10649 __ bind(LARGE_LOOP_PREFETCH_REPEAT2);
10650 compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2);
10651 __ subs(tmp4, tmp4, 1);
10652 __ br(__ GT, LARGE_LOOP_PREFETCH_REPEAT2);
10653 __ sub(cnt2, cnt2, 64);
10654 __ subs(rscratch2, cnt2, prefetchLoopExitCondition);
10655 __ br(__ GE, LARGE_LOOP_PREFETCH);
10656 }
10657 __ cbz(cnt2, LOAD_LAST); // no characters left except last load
10658 __ bind(NO_PREFETCH);
10659 __ subs(cnt2, cnt2, 16);
10660 __ br(__ LT, TAIL);
10661 __ align(OptoLoopAlignment);
10662 __ bind(SMALL_LOOP); // smaller loop
10663 __ subs(cnt2, cnt2, 16);
10664 compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2);
10665 __ br(__ GE, SMALL_LOOP);
10666 __ cmn(cnt2, (u1)16);
10667 __ br(__ EQ, LOAD_LAST);
10668 __ bind(TAIL); // 1..15 characters left until last load (last 4 characters)
10669 __ add(cnt1, cnt1, cnt2, __ LSL, 1); // Address of 32 bytes before last 4 characters in UTF-16 string
10670 __ add(tmp2, tmp2, cnt2); // Address of 16 bytes before last 4 characters in Latin1 string
10671 __ ldr(tmp3, Address(cnt1, -8));
10672 compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2); // last 16 characters before last load
10673 __ b(LOAD_LAST);
10674 __ bind(DIFF2);
10675 __ mov(tmpU, tmp3);
10676 __ bind(DIFF1);
10677 __ pop(spilled_regs, sp);
10678 __ b(CALCULATE_DIFFERENCE);
10679 __ bind(LOAD_LAST);
10680 // Last 4 UTF-16 characters are already pre-loaded into tmp3 by compare_string_16_x_LU.
10681 // No need to load it again
10682 __ mov(tmpU, tmp3);
10683 __ pop(spilled_regs, sp);
10684
10685 // tmp2 points to the address of the last 4 Latin1 characters right now
10686 __ ldrs(vtmp, Address(tmp2));
10687 __ zip1(vtmp, __ T8B, vtmp, vtmpZ);
10688 __ fmovd(tmpL, vtmp);
10689
10690 __ eor(rscratch2, tmpU, tmpL);
10691 __ cbz(rscratch2, DONE);
10692
10693 // Find the first different characters in the longwords and
10694 // compute their difference.
10695 __ bind(CALCULATE_DIFFERENCE);
10696 __ rev(rscratch2, rscratch2);
10697 __ clz(rscratch2, rscratch2);
10698 __ andr(rscratch2, rscratch2, -16);
10699 __ lsrv(tmp1, tmp1, rscratch2);
10700 __ uxthw(tmp1, tmp1);
10701 __ lsrv(rscratch1, rscratch1, rscratch2);
10702 __ uxthw(rscratch1, rscratch1);
10703 __ subw(result, tmp1, rscratch1);
10704 __ bind(DONE);
10705 __ ret(lr);
10706
10707 // record the stub entry and end
10708 store_archive_data(stub_id, entry, __ pc());
10709
10710 return entry;
10711 }
10712
10713 // r0 = input (float16)
10714 // v0 = result (float)
10715 // v1 = temporary float register
10716 address generate_float16ToFloat() {
10717 StubId stub_id = StubId::stubgen_hf2f_id;
10718 int entry_count = StubInfo::entry_count(stub_id);
10719 assert(entry_count == 1, "sanity check");
10720 address start = load_archive_data(stub_id);
10721 if (start != nullptr) {
10722 return start;
10723 }
10724 __ align(CodeEntryAlignment);
10725 StubCodeMark mark(this, stub_id);
10726 address entry = __ pc();
10727 BLOCK_COMMENT("Entry:");
10728 __ flt16_to_flt(v0, r0, v1);
10729 __ ret(lr);
10730
10731 // record the stub entry and end
10732 store_archive_data(stub_id, entry, __ pc());
10733
10734 return entry;
10735 }
10736
10737 // v0 = input (float)
10738 // r0 = result (float16)
10739 // v1 = temporary float register
10740 address generate_floatToFloat16() {
10741 StubId stub_id = StubId::stubgen_f2hf_id;
10742 int entry_count = StubInfo::entry_count(stub_id);
10743 assert(entry_count == 1, "sanity check");
10744 address start = load_archive_data(stub_id);
10745 if (start != nullptr) {
10746 return start;
10747 }
10748 __ align(CodeEntryAlignment);
10749 StubCodeMark mark(this, stub_id);
10750 address entry = __ pc();
10751 BLOCK_COMMENT("Entry:");
10752 __ flt_to_flt16(r0, v0, v1);
10753 __ ret(lr);
10754
10755 // record the stub entry and end
10756 store_archive_data(stub_id, entry, __ pc());
10757
10758 return entry;
10759 }
10760
10761 address generate_method_entry_barrier() {
10762 StubId stub_id = StubId::stubgen_method_entry_barrier_id;
10763 int entry_count = StubInfo::entry_count(stub_id);
10764 assert(entry_count == 1, "sanity check");
10765 address start = load_archive_data(stub_id);
10766 if (start != nullptr) {
10767 return start;
10768 }
10769 __ align(CodeEntryAlignment);
10770 StubCodeMark mark(this, stub_id);
10771
10772 Label deoptimize_label;
10773
10774 start = __ pc();
10775
10776 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
10777
10778 if (bs_asm->nmethod_patching_type() == NMethodPatchingType::conc_instruction_and_data_patch) {
10779 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
10780 // We can get here despite the nmethod being good, if we have not
10781 // yet applied our cross modification fence (or data fence).
10782 Address thread_epoch_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()) + 4);
10783 __ lea(rscratch2, ExternalAddress(bs_asm->patching_epoch_addr()));
10784 __ ldrw(rscratch2, rscratch2);
10785 __ strw(rscratch2, thread_epoch_addr);
10786 __ isb();
10787 __ membar(__ LoadLoad);
10788 }
10789
10790 __ set_last_Java_frame(sp, rfp, lr, rscratch1);
10791
10792 __ enter();
10793 __ add(rscratch2, sp, wordSize); // rscratch2 points to the saved lr
10794
10795 __ sub(sp, sp, 4 * wordSize); // four words for the returned {sp, fp, lr, pc}
10796
10797 __ push_call_clobbered_registers();
10798
10799 __ mov(c_rarg0, rscratch2);
10800 __ call_VM_leaf
10801 (CAST_FROM_FN_PTR
10802 (address, BarrierSetNMethod::nmethod_stub_entry_barrier), 1);
10803
10804 __ reset_last_Java_frame(true);
10805
10806 __ mov(rscratch1, r0);
10807
10808 __ pop_call_clobbered_registers();
10809
10810 __ cbnz(rscratch1, deoptimize_label);
10811
10812 __ leave();
10813 __ ret(lr);
10814
10815 __ BIND(deoptimize_label);
10816
10817 __ ldp(/* new sp */ rscratch1, rfp, Address(sp, 0 * wordSize));
10818 __ ldp(lr, /* new pc*/ rscratch2, Address(sp, 2 * wordSize));
10819
10820 __ mov(sp, rscratch1);
10821 __ br(rscratch2);
10822
10823 // record the stub entry and end
10824 store_archive_data(stub_id, start, __ pc());
10825
10826 return start;
10827 }
10828
10829 // r0 = result
10830 // r1 = str1
10831 // r2 = cnt1
10832 // r3 = str2
10833 // r4 = cnt2
10834 // r10 = tmp1
10835 // r11 = tmp2
10836 address generate_compare_long_string_same_encoding(bool isLL) {
10837 StubId stub_id = (isLL ? StubId::stubgen_compare_long_string_LL_id : StubId::stubgen_compare_long_string_UU_id);
10838 int entry_count = StubInfo::entry_count(stub_id);
10839 assert(entry_count == 1, "sanity check");
10840 address start = load_archive_data(stub_id);
10841 if (start != nullptr) {
10842 return start;
10843 }
10844 __ align(CodeEntryAlignment);
10845 StubCodeMark mark(this, stub_id);
10846 address entry = __ pc();
10847 Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
10848 tmp1 = r10, tmp2 = r11, tmp1h = rscratch1, tmp2h = rscratch2;
10849
10850 Label LARGE_LOOP_PREFETCH, LOOP_COMPARE16, DIFF, LESS16, LESS8, CAL_DIFFERENCE, LENGTH_DIFF;
10851
10852 // exit from large loop when less than 64 bytes left to read or we're about
10853 // to prefetch memory behind array border
10854 int largeLoopExitCondition = MAX2(64, SoftwarePrefetchHintDistance)/(isLL ? 1 : 2);
10855
10856 // before jumping to stub, pre-load 8 bytes already, so do comparison directly
10857 __ eor(rscratch2, tmp1, tmp2);
10858 __ cbnz(rscratch2, CAL_DIFFERENCE);
10859
10860 __ sub(cnt2, cnt2, wordSize/(isLL ? 1 : 2));
10861 // update pointers, because of previous read
10862 __ add(str1, str1, wordSize);
10863 __ add(str2, str2, wordSize);
10864 if (SoftwarePrefetchHintDistance >= 0) {
10865 __ align(OptoLoopAlignment);
10866 __ bind(LARGE_LOOP_PREFETCH);
10867 __ prfm(Address(str1, SoftwarePrefetchHintDistance));
10868 __ prfm(Address(str2, SoftwarePrefetchHintDistance));
10869
10870 for (int i = 0; i < 4; i++) {
10871 __ ldp(tmp1, tmp1h, Address(str1, i * 16));
10872 __ ldp(tmp2, tmp2h, Address(str2, i * 16));
10873 __ cmp(tmp1, tmp2);
10874 __ ccmp(tmp1h, tmp2h, 0, Assembler::EQ);
10875 __ br(Assembler::NE, DIFF);
10876 }
10877 __ sub(cnt2, cnt2, isLL ? 64 : 32);
10878 __ add(str1, str1, 64);
10879 __ add(str2, str2, 64);
10880 __ subs(rscratch2, cnt2, largeLoopExitCondition);
10881 __ br(Assembler::GE, LARGE_LOOP_PREFETCH);
10882 __ cbz(cnt2, LENGTH_DIFF); // no more chars left?
10883 }
10884
10885 __ subs(rscratch1, cnt2, isLL ? 16 : 8);
10886 __ br(Assembler::LE, LESS16);
10887 __ align(OptoLoopAlignment);
10888 __ bind(LOOP_COMPARE16);
10889 __ ldp(tmp1, tmp1h, Address(__ post(str1, 16)));
10890 __ ldp(tmp2, tmp2h, Address(__ post(str2, 16)));
10891 __ cmp(tmp1, tmp2);
10892 __ ccmp(tmp1h, tmp2h, 0, Assembler::EQ);
10893 __ br(Assembler::NE, DIFF);
10894 __ sub(cnt2, cnt2, isLL ? 16 : 8);
10895 __ subs(rscratch2, cnt2, isLL ? 16 : 8);
10896 __ br(Assembler::LT, LESS16);
10897
10898 __ ldp(tmp1, tmp1h, Address(__ post(str1, 16)));
10899 __ ldp(tmp2, tmp2h, Address(__ post(str2, 16)));
10900 __ cmp(tmp1, tmp2);
10901 __ ccmp(tmp1h, tmp2h, 0, Assembler::EQ);
10902 __ br(Assembler::NE, DIFF);
10903 __ sub(cnt2, cnt2, isLL ? 16 : 8);
10904 __ subs(rscratch2, cnt2, isLL ? 16 : 8);
10905 __ br(Assembler::GE, LOOP_COMPARE16);
10906 __ cbz(cnt2, LENGTH_DIFF);
10907
10908 __ bind(LESS16);
10909 // each 8 compare
10910 __ subs(cnt2, cnt2, isLL ? 8 : 4);
10911 __ br(Assembler::LE, LESS8);
10912 __ ldr(tmp1, Address(__ post(str1, 8)));
10913 __ ldr(tmp2, Address(__ post(str2, 8)));
10914 __ eor(rscratch2, tmp1, tmp2);
10915 __ cbnz(rscratch2, CAL_DIFFERENCE);
10916 __ sub(cnt2, cnt2, isLL ? 8 : 4);
10917
10918 __ bind(LESS8); // directly load last 8 bytes
10919 if (!isLL) {
10920 __ add(cnt2, cnt2, cnt2);
10921 }
10922 __ ldr(tmp1, Address(str1, cnt2));
10923 __ ldr(tmp2, Address(str2, cnt2));
10924 __ eor(rscratch2, tmp1, tmp2);
10925 __ cbz(rscratch2, LENGTH_DIFF);
10926 __ b(CAL_DIFFERENCE);
10927
10928 __ bind(DIFF);
10929 __ cmp(tmp1, tmp2);
10930 __ csel(tmp1, tmp1, tmp1h, Assembler::NE);
10931 __ csel(tmp2, tmp2, tmp2h, Assembler::NE);
10932 // reuse rscratch2 register for the result of eor instruction
10933 __ eor(rscratch2, tmp1, tmp2);
10934
10935 __ bind(CAL_DIFFERENCE);
10936 __ rev(rscratch2, rscratch2);
10937 __ clz(rscratch2, rscratch2);
10938 __ andr(rscratch2, rscratch2, isLL ? -8 : -16);
10939 __ lsrv(tmp1, tmp1, rscratch2);
10940 __ lsrv(tmp2, tmp2, rscratch2);
10941 if (isLL) {
10942 __ uxtbw(tmp1, tmp1);
10943 __ uxtbw(tmp2, tmp2);
10944 } else {
10945 __ uxthw(tmp1, tmp1);
10946 __ uxthw(tmp2, tmp2);
10947 }
10948 __ subw(result, tmp1, tmp2);
10949
10950 __ bind(LENGTH_DIFF);
10951 __ ret(lr);
10952
10953 // record the stub entry and end
10954 store_archive_data(stub_id, entry, __ pc());
10955
10956 return entry;
10957 }
10958
10959 enum string_compare_mode {
10960 LL,
10961 LU,
10962 UL,
10963 UU,
10964 };
10965
10966 // The following registers are declared in aarch64.ad
10967 // r0 = result
10968 // r1 = str1
10969 // r2 = cnt1
10970 // r3 = str2
10971 // r4 = cnt2
10972 // r10 = tmp1
10973 // r11 = tmp2
10974 // z0 = ztmp1
10975 // z1 = ztmp2
10976 // p0 = pgtmp1
10977 // p1 = pgtmp2
10978 address generate_compare_long_string_sve(string_compare_mode mode) {
10979 StubId stub_id;
10980 switch (mode) {
10981 case LL: stub_id = StubId::stubgen_compare_long_string_LL_id; break;
10982 case LU: stub_id = StubId::stubgen_compare_long_string_LU_id; break;
10983 case UL: stub_id = StubId::stubgen_compare_long_string_UL_id; break;
10984 case UU: stub_id = StubId::stubgen_compare_long_string_UU_id; break;
10985 default: ShouldNotReachHere();
10986 }
10987 int entry_count = StubInfo::entry_count(stub_id);
10988 assert(entry_count == 1, "sanity check");
10989 address start = load_archive_data(stub_id);
10990 if (start != nullptr) {
10991 return start;
10992 }
10993 __ align(CodeEntryAlignment);
10994 StubCodeMark mark(this, stub_id);
10995 address entry = __ pc();
10996 Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
10997 tmp1 = r10, tmp2 = r11;
10998
10999 Label LOOP, DONE, MISMATCH;
11000 Register vec_len = tmp1;
11001 Register idx = tmp2;
11002 // The minimum of the string lengths has been stored in cnt2.
11003 Register cnt = cnt2;
11004 FloatRegister ztmp1 = z0, ztmp2 = z1;
11005 PRegister pgtmp1 = p0, pgtmp2 = p1;
11006
11007 #define LOAD_PAIR(ztmp1, ztmp2, pgtmp1, src1, src2, idx) \
11008 switch (mode) { \
11009 case LL: \
11010 __ sve_ld1b(ztmp1, __ B, pgtmp1, Address(str1, idx)); \
11011 __ sve_ld1b(ztmp2, __ B, pgtmp1, Address(str2, idx)); \
11012 break; \
11013 case LU: \
11014 __ sve_ld1b(ztmp1, __ H, pgtmp1, Address(str1, idx)); \
11015 __ sve_ld1h(ztmp2, __ H, pgtmp1, Address(str2, idx, Address::lsl(1))); \
11016 break; \
11017 case UL: \
11018 __ sve_ld1h(ztmp1, __ H, pgtmp1, Address(str1, idx, Address::lsl(1))); \
11019 __ sve_ld1b(ztmp2, __ H, pgtmp1, Address(str2, idx)); \
11020 break; \
11021 case UU: \
11022 __ sve_ld1h(ztmp1, __ H, pgtmp1, Address(str1, idx, Address::lsl(1))); \
11023 __ sve_ld1h(ztmp2, __ H, pgtmp1, Address(str2, idx, Address::lsl(1))); \
11024 break; \
11025 default: \
11026 ShouldNotReachHere(); \
11027 }
11028
11029 __ mov(idx, 0);
11030 __ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt);
11031
11032 if (mode == LL) {
11033 __ sve_cntb(vec_len);
11034 } else {
11035 __ sve_cnth(vec_len);
11036 }
11037
11038 __ sub(rscratch1, cnt, vec_len);
11039
11040 __ bind(LOOP);
11041
11042 // main loop
11043 LOAD_PAIR(ztmp1, ztmp2, pgtmp1, src1, src2, idx);
11044 __ add(idx, idx, vec_len);
11045 // Compare strings.
11046 __ sve_cmp(Assembler::NE, pgtmp2, mode == LL ? __ B : __ H, pgtmp1, ztmp1, ztmp2);
11047 __ br(__ NE, MISMATCH);
11048 __ cmp(idx, rscratch1);
11049 __ br(__ LT, LOOP);
11050
11051 // post loop, last iteration
11052 __ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt);
11053
11054 LOAD_PAIR(ztmp1, ztmp2, pgtmp1, src1, src2, idx);
11055 __ sve_cmp(Assembler::NE, pgtmp2, mode == LL ? __ B : __ H, pgtmp1, ztmp1, ztmp2);
11056 __ br(__ EQ, DONE);
11057
11058 __ bind(MISMATCH);
11059
11060 // Crop the vector to find its location.
11061 __ sve_brkb(pgtmp2, pgtmp1, pgtmp2, false /* isMerge */);
11062 // Extract the first different characters of each string.
11063 __ sve_lasta(rscratch1, mode == LL ? __ B : __ H, pgtmp2, ztmp1);
11064 __ sve_lasta(rscratch2, mode == LL ? __ B : __ H, pgtmp2, ztmp2);
11065
11066 // Compute the difference of the first different characters.
11067 __ sub(result, rscratch1, rscratch2);
11068
11069 __ bind(DONE);
11070 __ ret(lr);
11071 #undef LOAD_PAIR
11072
11073 // record the stub entry and end
11074 store_archive_data(stub_id, entry, __ pc());
11075
11076 return entry;
11077 }
11078
11079 void generate_compare_long_strings() {
11080 if (UseSVE == 0) {
11081 StubRoutines::aarch64::_compare_long_string_LL
11082 = generate_compare_long_string_same_encoding(true);
11083 StubRoutines::aarch64::_compare_long_string_UU
11084 = generate_compare_long_string_same_encoding(false);
11085 StubRoutines::aarch64::_compare_long_string_LU
11086 = generate_compare_long_string_different_encoding(true);
11087 StubRoutines::aarch64::_compare_long_string_UL
11088 = generate_compare_long_string_different_encoding(false);
11089 } else {
11090 StubRoutines::aarch64::_compare_long_string_LL
11091 = generate_compare_long_string_sve(LL);
11092 StubRoutines::aarch64::_compare_long_string_UU
11093 = generate_compare_long_string_sve(UU);
11094 StubRoutines::aarch64::_compare_long_string_LU
11095 = generate_compare_long_string_sve(LU);
11096 StubRoutines::aarch64::_compare_long_string_UL
11097 = generate_compare_long_string_sve(UL);
11098 }
11099 }
11100
11101 // R0 = result
11102 // R1 = str2
11103 // R2 = cnt1
11104 // R3 = str1
11105 // R4 = cnt2
11106 // Clobbers: rscratch1, rscratch2, v0, v1, rflags
11107 //
11108 // This generic linear code use few additional ideas, which makes it faster:
11109 // 1) we can safely keep at least 1st register of pattern(since length >= 8)
11110 // in order to skip initial loading(help in systems with 1 ld pipeline)
11111 // 2) we can use "fast" algorithm of finding single character to search for
11112 // first symbol with less branches(1 branch per each loaded register instead
11113 // of branch for each symbol), so, this is where constants like
11114 // 0x0101...01, 0x00010001...0001, 0x7f7f...7f, 0x7fff7fff...7fff comes from
11115 // 3) after loading and analyzing 1st register of source string, it can be
11116 // used to search for every 1st character entry, saving few loads in
11117 // comparison with "simplier-but-slower" implementation
11118 // 4) in order to avoid lots of push/pop operations, code below is heavily
11119 // re-using/re-initializing/compressing register values, which makes code
11120 // larger and a bit less readable, however, most of extra operations are
11121 // issued during loads or branches, so, penalty is minimal
11122 address generate_string_indexof_linear(bool str1_isL, bool str2_isL) {
11123 StubId stub_id;
11124 if (str1_isL) {
11125 if (str2_isL) {
11126 stub_id = StubId::stubgen_string_indexof_linear_ll_id;
11127 } else {
11128 stub_id = StubId::stubgen_string_indexof_linear_ul_id;
11129 }
11130 } else {
11131 if (str2_isL) {
11132 ShouldNotReachHere();
11133 } else {
11134 stub_id = StubId::stubgen_string_indexof_linear_uu_id;
11135 }
11136 }
11137 int entry_count = StubInfo::entry_count(stub_id);
11138 assert(entry_count == 1, "sanity check");
11139 address start = load_archive_data(stub_id);
11140 if (start != nullptr) {
11141 return start;
11142 }
11143 __ align(CodeEntryAlignment);
11144 StubCodeMark mark(this, stub_id);
11145 address entry = __ pc();
11146
11147 int str1_chr_size = str1_isL ? 1 : 2;
11148 int str2_chr_size = str2_isL ? 1 : 2;
11149 int str1_chr_shift = str1_isL ? 0 : 1;
11150 int str2_chr_shift = str2_isL ? 0 : 1;
11151 bool isL = str1_isL && str2_isL;
11152 // parameters
11153 Register result = r0, str2 = r1, cnt1 = r2, str1 = r3, cnt2 = r4;
11154 // temporary registers
11155 Register tmp1 = r20, tmp2 = r21, tmp3 = r22, tmp4 = r23;
11156 RegSet spilled_regs = RegSet::range(tmp1, tmp4);
11157 // redefinitions
11158 Register ch1 = rscratch1, ch2 = rscratch2, first = tmp3;
11159
11160 __ push(spilled_regs, sp);
11161 Label L_LOOP, L_LOOP_PROCEED, L_SMALL, L_HAS_ZERO,
11162 L_HAS_ZERO_LOOP, L_CMP_LOOP, L_CMP_LOOP_NOMATCH, L_SMALL_PROCEED,
11163 L_SMALL_HAS_ZERO_LOOP, L_SMALL_CMP_LOOP_NOMATCH, L_SMALL_CMP_LOOP,
11164 L_POST_LOOP, L_CMP_LOOP_LAST_CMP, L_HAS_ZERO_LOOP_NOMATCH,
11165 L_SMALL_CMP_LOOP_LAST_CMP, L_SMALL_CMP_LOOP_LAST_CMP2,
11166 L_CMP_LOOP_LAST_CMP2, DONE, NOMATCH;
11167 // Read whole register from str1. It is safe, because length >=8 here
11168 __ ldr(ch1, Address(str1));
11169 // Read whole register from str2. It is safe, because length >=8 here
11170 __ ldr(ch2, Address(str2));
11171 __ sub(cnt2, cnt2, cnt1);
11172 __ andr(first, ch1, str1_isL ? 0xFF : 0xFFFF);
11173 if (str1_isL != str2_isL) {
11174 __ eor(v0, __ T16B, v0, v0);
11175 }
11176 __ mov(tmp1, str2_isL ? 0x0101010101010101 : 0x0001000100010001);
11177 __ mul(first, first, tmp1);
11178 // check if we have less than 1 register to check
11179 __ subs(cnt2, cnt2, wordSize/str2_chr_size - 1);
11180 if (str1_isL != str2_isL) {
11181 __ fmovd(v1, ch1);
11182 }
11183 __ br(__ LE, L_SMALL);
11184 __ eor(ch2, first, ch2);
11185 if (str1_isL != str2_isL) {
11186 __ zip1(v1, __ T16B, v1, v0);
11187 }
11188 __ sub(tmp2, ch2, tmp1);
11189 __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
11190 __ bics(tmp2, tmp2, ch2);
11191 if (str1_isL != str2_isL) {
11192 __ fmovd(ch1, v1);
11193 }
11194 __ br(__ NE, L_HAS_ZERO);
11195 __ subs(cnt2, cnt2, wordSize/str2_chr_size);
11196 __ add(result, result, wordSize/str2_chr_size);
11197 __ add(str2, str2, wordSize);
11198 __ br(__ LT, L_POST_LOOP);
11199 __ BIND(L_LOOP);
11200 __ ldr(ch2, Address(str2));
11201 __ eor(ch2, first, ch2);
11202 __ sub(tmp2, ch2, tmp1);
11203 __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
11204 __ bics(tmp2, tmp2, ch2);
11205 __ br(__ NE, L_HAS_ZERO);
11206 __ BIND(L_LOOP_PROCEED);
11207 __ subs(cnt2, cnt2, wordSize/str2_chr_size);
11208 __ add(str2, str2, wordSize);
11209 __ add(result, result, wordSize/str2_chr_size);
11210 __ br(__ GE, L_LOOP);
11211 __ BIND(L_POST_LOOP);
11212 __ subs(zr, cnt2, -wordSize/str2_chr_size); // no extra characters to check
11213 __ br(__ LE, NOMATCH);
11214 __ ldr(ch2, Address(str2));
11215 __ sub(cnt2, zr, cnt2, __ LSL, LogBitsPerByte + str2_chr_shift);
11216 __ eor(ch2, first, ch2);
11217 __ sub(tmp2, ch2, tmp1);
11218 __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
11219 __ mov(tmp4, -1); // all bits set
11220 __ b(L_SMALL_PROCEED);
11221 __ align(OptoLoopAlignment);
11222 __ BIND(L_SMALL);
11223 __ sub(cnt2, zr, cnt2, __ LSL, LogBitsPerByte + str2_chr_shift);
11224 __ eor(ch2, first, ch2);
11225 if (str1_isL != str2_isL) {
11226 __ zip1(v1, __ T16B, v1, v0);
11227 }
11228 __ sub(tmp2, ch2, tmp1);
11229 __ mov(tmp4, -1); // all bits set
11230 __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
11231 if (str1_isL != str2_isL) {
11232 __ fmovd(ch1, v1); // move converted 4 symbols
11233 }
11234 __ BIND(L_SMALL_PROCEED);
11235 __ lsrv(tmp4, tmp4, cnt2); // mask. zeroes on useless bits.
11236 __ bic(tmp2, tmp2, ch2);
11237 __ ands(tmp2, tmp2, tmp4); // clear useless bits and check
11238 __ rbit(tmp2, tmp2);
11239 __ br(__ EQ, NOMATCH);
11240 __ BIND(L_SMALL_HAS_ZERO_LOOP);
11241 __ clz(tmp4, tmp2); // potentially long. Up to 4 cycles on some cpu's
11242 __ cmp(cnt1, u1(wordSize/str2_chr_size));
11243 __ br(__ LE, L_SMALL_CMP_LOOP_LAST_CMP2);
11244 if (str2_isL) { // LL
11245 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte); // address of "index"
11246 __ ldr(ch2, Address(str2)); // read whole register of str2. Safe.
11247 __ lslv(tmp2, tmp2, tmp4); // shift off leading zeroes from match info
11248 __ add(result, result, tmp4, __ LSR, LogBitsPerByte);
11249 __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
11250 } else {
11251 __ mov(ch2, 0xE); // all bits in byte set except last one
11252 __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
11253 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11254 __ lslv(tmp2, tmp2, tmp4);
11255 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11256 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11257 __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
11258 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11259 }
11260 __ cmp(ch1, ch2);
11261 __ mov(tmp4, wordSize/str2_chr_size);
11262 __ br(__ NE, L_SMALL_CMP_LOOP_NOMATCH);
11263 __ BIND(L_SMALL_CMP_LOOP);
11264 str1_isL ? __ ldrb(first, Address(str1, tmp4, Address::lsl(str1_chr_shift)))
11265 : __ ldrh(first, Address(str1, tmp4, Address::lsl(str1_chr_shift)));
11266 str2_isL ? __ ldrb(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)))
11267 : __ ldrh(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)));
11268 __ add(tmp4, tmp4, 1);
11269 __ cmp(tmp4, cnt1);
11270 __ br(__ GE, L_SMALL_CMP_LOOP_LAST_CMP);
11271 __ cmp(first, ch2);
11272 __ br(__ EQ, L_SMALL_CMP_LOOP);
11273 __ BIND(L_SMALL_CMP_LOOP_NOMATCH);
11274 __ cbz(tmp2, NOMATCH); // no more matches. exit
11275 __ clz(tmp4, tmp2);
11276 __ add(result, result, 1); // advance index
11277 __ add(str2, str2, str2_chr_size); // advance pointer
11278 __ b(L_SMALL_HAS_ZERO_LOOP);
11279 __ align(OptoLoopAlignment);
11280 __ BIND(L_SMALL_CMP_LOOP_LAST_CMP);
11281 __ cmp(first, ch2);
11282 __ br(__ NE, L_SMALL_CMP_LOOP_NOMATCH);
11283 __ b(DONE);
11284 __ align(OptoLoopAlignment);
11285 __ BIND(L_SMALL_CMP_LOOP_LAST_CMP2);
11286 if (str2_isL) { // LL
11287 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte); // address of "index"
11288 __ ldr(ch2, Address(str2)); // read whole register of str2. Safe.
11289 __ lslv(tmp2, tmp2, tmp4); // shift off leading zeroes from match info
11290 __ add(result, result, tmp4, __ LSR, LogBitsPerByte);
11291 __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
11292 } else {
11293 __ mov(ch2, 0xE); // all bits in byte set except last one
11294 __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
11295 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11296 __ lslv(tmp2, tmp2, tmp4);
11297 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11298 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11299 __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
11300 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11301 }
11302 __ cmp(ch1, ch2);
11303 __ br(__ NE, L_SMALL_CMP_LOOP_NOMATCH);
11304 __ b(DONE);
11305 __ align(OptoLoopAlignment);
11306 __ BIND(L_HAS_ZERO);
11307 __ rbit(tmp2, tmp2);
11308 __ clz(tmp4, tmp2); // potentially long. Up to 4 cycles on some CPU's
11309 // Now, perform compression of counters(cnt2 and cnt1) into one register.
11310 // It's fine because both counters are 32bit and are not changed in this
11311 // loop. Just restore it on exit. So, cnt1 can be re-used in this loop.
11312 __ orr(cnt2, cnt2, cnt1, __ LSL, BitsPerByte * wordSize / 2);
11313 __ sub(result, result, 1);
11314 __ BIND(L_HAS_ZERO_LOOP);
11315 __ mov(cnt1, wordSize/str2_chr_size);
11316 __ cmp(cnt1, cnt2, __ LSR, BitsPerByte * wordSize / 2);
11317 __ br(__ GE, L_CMP_LOOP_LAST_CMP2); // case of 8 bytes only to compare
11318 if (str2_isL) {
11319 __ lsr(ch2, tmp4, LogBitsPerByte + str2_chr_shift); // char index
11320 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11321 __ lslv(tmp2, tmp2, tmp4);
11322 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11323 __ add(tmp4, tmp4, 1);
11324 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11325 __ lsl(tmp2, tmp2, 1);
11326 __ mov(tmp4, wordSize/str2_chr_size);
11327 } else {
11328 __ mov(ch2, 0xE);
11329 __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
11330 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11331 __ lslv(tmp2, tmp2, tmp4);
11332 __ add(tmp4, tmp4, 1);
11333 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11334 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte);
11335 __ lsl(tmp2, tmp2, 1);
11336 __ mov(tmp4, wordSize/str2_chr_size);
11337 __ sub(str2, str2, str2_chr_size);
11338 }
11339 __ cmp(ch1, ch2);
11340 __ mov(tmp4, wordSize/str2_chr_size);
11341 __ br(__ NE, L_CMP_LOOP_NOMATCH);
11342 __ BIND(L_CMP_LOOP);
11343 str1_isL ? __ ldrb(cnt1, Address(str1, tmp4, Address::lsl(str1_chr_shift)))
11344 : __ ldrh(cnt1, Address(str1, tmp4, Address::lsl(str1_chr_shift)));
11345 str2_isL ? __ ldrb(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)))
11346 : __ ldrh(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)));
11347 __ add(tmp4, tmp4, 1);
11348 __ cmp(tmp4, cnt2, __ LSR, BitsPerByte * wordSize / 2);
11349 __ br(__ GE, L_CMP_LOOP_LAST_CMP);
11350 __ cmp(cnt1, ch2);
11351 __ br(__ EQ, L_CMP_LOOP);
11352 __ BIND(L_CMP_LOOP_NOMATCH);
11353 // here we're not matched
11354 __ cbz(tmp2, L_HAS_ZERO_LOOP_NOMATCH); // no more matches. Proceed to main loop
11355 __ clz(tmp4, tmp2);
11356 __ add(str2, str2, str2_chr_size); // advance pointer
11357 __ b(L_HAS_ZERO_LOOP);
11358 __ align(OptoLoopAlignment);
11359 __ BIND(L_CMP_LOOP_LAST_CMP);
11360 __ cmp(cnt1, ch2);
11361 __ br(__ NE, L_CMP_LOOP_NOMATCH);
11362 __ b(DONE);
11363 __ align(OptoLoopAlignment);
11364 __ BIND(L_CMP_LOOP_LAST_CMP2);
11365 if (str2_isL) {
11366 __ lsr(ch2, tmp4, LogBitsPerByte + str2_chr_shift); // char index
11367 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11368 __ lslv(tmp2, tmp2, tmp4);
11369 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11370 __ add(tmp4, tmp4, 1);
11371 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11372 __ lsl(tmp2, tmp2, 1);
11373 } else {
11374 __ mov(ch2, 0xE);
11375 __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
11376 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11377 __ lslv(tmp2, tmp2, tmp4);
11378 __ add(tmp4, tmp4, 1);
11379 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11380 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte);
11381 __ lsl(tmp2, tmp2, 1);
11382 __ sub(str2, str2, str2_chr_size);
11383 }
11384 __ cmp(ch1, ch2);
11385 __ br(__ NE, L_CMP_LOOP_NOMATCH);
11386 __ b(DONE);
11387 __ align(OptoLoopAlignment);
11388 __ BIND(L_HAS_ZERO_LOOP_NOMATCH);
11389 // 1) Restore "result" index. Index was wordSize/str2_chr_size * N until
11390 // L_HAS_ZERO block. Byte octet was analyzed in L_HAS_ZERO_LOOP,
11391 // so, result was increased at max by wordSize/str2_chr_size - 1, so,
11392 // respective high bit wasn't changed. L_LOOP_PROCEED will increase
11393 // result by analyzed characters value, so, we can just reset lower bits
11394 // in result here. Clear 2 lower bits for UU/UL and 3 bits for LL
11395 // 2) restore cnt1 and cnt2 values from "compressed" cnt2
11396 // 3) advance str2 value to represent next str2 octet. result & 7/3 is
11397 // index of last analyzed substring inside current octet. So, str2 in at
11398 // respective start address. We need to advance it to next octet
11399 __ andr(tmp2, result, wordSize/str2_chr_size - 1); // symbols analyzed
11400 __ lsr(cnt1, cnt2, BitsPerByte * wordSize / 2);
11401 __ bfm(result, zr, 0, 2 - str2_chr_shift);
11402 __ sub(str2, str2, tmp2, __ LSL, str2_chr_shift); // restore str2
11403 __ movw(cnt2, cnt2);
11404 __ b(L_LOOP_PROCEED);
11405 __ align(OptoLoopAlignment);
11406 __ BIND(NOMATCH);
11407 __ mov(result, -1);
11408 __ BIND(DONE);
11409 __ pop(spilled_regs, sp);
11410 __ ret(lr);
11411
11412 // record the stub entry and end
11413 store_archive_data(stub_id, entry, __ pc());
11414
11415 return entry;
11416 }
11417
11418 void generate_string_indexof_stubs() {
11419 StubRoutines::aarch64::_string_indexof_linear_ll = generate_string_indexof_linear(true, true);
11420 StubRoutines::aarch64::_string_indexof_linear_uu = generate_string_indexof_linear(false, false);
11421 StubRoutines::aarch64::_string_indexof_linear_ul = generate_string_indexof_linear(true, false);
11422 }
11423
11424 void inflate_and_store_2_fp_registers(bool generatePrfm,
11425 FloatRegister src1, FloatRegister src2) {
11426 Register dst = r1;
11427 __ zip1(v1, __ T16B, src1, v0);
11428 __ zip2(v2, __ T16B, src1, v0);
11429 if (generatePrfm) {
11430 __ prfm(Address(dst, SoftwarePrefetchHintDistance), PSTL1STRM);
11431 }
11432 __ zip1(v3, __ T16B, src2, v0);
11433 __ zip2(v4, __ T16B, src2, v0);
11434 __ st1(v1, v2, v3, v4, __ T16B, Address(__ post(dst, 64)));
11435 }
11436
11437 // R0 = src
11438 // R1 = dst
11439 // R2 = len
11440 // R3 = len >> 3
11441 // V0 = 0
11442 // v1 = loaded 8 bytes
11443 // Clobbers: r0, r1, r3, rscratch1, rflags, v0-v6
11444 address generate_large_byte_array_inflate() {
11445 StubId stub_id = StubId::stubgen_large_byte_array_inflate_id;
11446 int entry_count = StubInfo::entry_count(stub_id);
11447 assert(entry_count == 1, "sanity check");
11448 address start = load_archive_data(stub_id);
11449 if (start != nullptr) {
11450 return start;
11451 }
11452 __ align(CodeEntryAlignment);
11453 StubCodeMark mark(this, stub_id);
11454 address entry = __ pc();
11455 Label LOOP, LOOP_START, LOOP_PRFM, LOOP_PRFM_START, DONE;
11456 Register src = r0, dst = r1, len = r2, octetCounter = r3;
11457 const int large_loop_threshold = MAX2(64, SoftwarePrefetchHintDistance)/8 + 4;
11458
11459 // do one more 8-byte read to have address 16-byte aligned in most cases
11460 // also use single store instruction
11461 __ ldrd(v2, __ post(src, 8));
11462 __ sub(octetCounter, octetCounter, 2);
11463 __ zip1(v1, __ T16B, v1, v0);
11464 __ zip1(v2, __ T16B, v2, v0);
11465 __ st1(v1, v2, __ T16B, __ post(dst, 32));
11466 __ ld1(v3, v4, v5, v6, __ T16B, Address(__ post(src, 64)));
11467 __ subs(rscratch1, octetCounter, large_loop_threshold);
11468 __ br(__ LE, LOOP_START);
11469 __ b(LOOP_PRFM_START);
11470 __ bind(LOOP_PRFM);
11471 __ ld1(v3, v4, v5, v6, __ T16B, Address(__ post(src, 64)));
11472 __ bind(LOOP_PRFM_START);
11473 __ prfm(Address(src, SoftwarePrefetchHintDistance));
11474 __ sub(octetCounter, octetCounter, 8);
11475 __ subs(rscratch1, octetCounter, large_loop_threshold);
11476 inflate_and_store_2_fp_registers(true, v3, v4);
11477 inflate_and_store_2_fp_registers(true, v5, v6);
11478 __ br(__ GT, LOOP_PRFM);
11479 __ cmp(octetCounter, (u1)8);
11480 __ br(__ LT, DONE);
11481 __ bind(LOOP);
11482 __ ld1(v3, v4, v5, v6, __ T16B, Address(__ post(src, 64)));
11483 __ bind(LOOP_START);
11484 __ sub(octetCounter, octetCounter, 8);
11485 __ cmp(octetCounter, (u1)8);
11486 inflate_and_store_2_fp_registers(false, v3, v4);
11487 inflate_and_store_2_fp_registers(false, v5, v6);
11488 __ br(__ GE, LOOP);
11489 __ bind(DONE);
11490 __ ret(lr);
11491
11492 // record the stub entry and end
11493 store_archive_data(stub_id, entry, __ pc());
11494
11495 return entry;
11496 }
11497
11498 /**
11499 * Arguments:
11500 *
11501 * Input:
11502 * c_rarg0 - current state address
11503 * c_rarg1 - H key address
11504 * c_rarg2 - data address
11505 * c_rarg3 - number of blocks
11506 *
11507 * Output:
11508 * Updated state at c_rarg0
11509 */
11510 address generate_ghash_processBlocks_small() {
11511 // Bafflingly, GCM uses little-endian for the byte order, but
11512 // big-endian for the bit order. For example, the polynomial 1 is
11513 // represented as the 16-byte string 80 00 00 00 | 12 bytes of 00.
11514 //
11515 // So, we must either reverse the bytes in each word and do
11516 // everything big-endian or reverse the bits in each byte and do
11517 // it little-endian. On AArch64 it's more idiomatic to reverse
11518 // the bits in each byte (we have an instruction, RBIT, to do
11519 // that) and keep the data in little-endian bit order through the
11520 // calculation, bit-reversing the inputs and outputs.
11521
11522 StubId stub_id = StubId::stubgen_ghash_processBlocks_small_id;
11523 int entry_count = StubInfo::entry_count(stub_id);
11524 assert(entry_count == 1, "sanity check");
11525 address start = load_archive_data(stub_id);
11526 if (start != nullptr) {
11527 return start;
11528 }
11529 __ align(CodeEntryAlignment);
11530 StubCodeMark mark(this, stub_id);
11531 Label polynomial; // local data generated at end of stub
11532 start = __ pc();
11533
11534 Register state = c_rarg0;
11535 Register subkeyH = c_rarg1;
11536 Register data = c_rarg2;
11537 Register blocks = c_rarg3;
11538
11539 FloatRegister vzr = v30;
11540 __ eor(vzr, __ T16B, vzr, vzr); // zero register
11541
11542 __ adr(rscratch1, polynomial);
11543 __ ldrq(v24, rscratch1); // The field polynomial
11544
11545 __ ldrq(v0, Address(state));
11546 __ ldrq(v1, Address(subkeyH));
11547
11548 __ rev64(v0, __ T16B, v0); // Bit-reverse words in state and subkeyH
11549 __ rbit(v0, __ T16B, v0);
11550 __ rev64(v1, __ T16B, v1);
11551 __ rbit(v1, __ T16B, v1);
11552
11553 __ ext(v4, __ T16B, v1, v1, 0x08); // long-swap subkeyH into v1
11554 __ eor(v4, __ T16B, v4, v1); // xor subkeyH into subkeyL (Karatsuba: (A1+A0))
11555
11556 {
11557 Label L_ghash_loop;
11558 __ bind(L_ghash_loop);
11559
11560 __ ldrq(v2, Address(__ post(data, 0x10))); // Load the data, bit
11561 // reversing each byte
11562 __ rbit(v2, __ T16B, v2);
11563 __ eor(v2, __ T16B, v0, v2); // bit-swapped data ^ bit-swapped state
11564
11565 // Multiply state in v2 by subkey in v1
11566 __ ghash_multiply(/*result_lo*/v5, /*result_hi*/v7,
11567 /*a*/v1, /*b*/v2, /*a1_xor_a0*/v4,
11568 /*temps*/v6, v3, /*reuse/clobber b*/v2);
11569 // Reduce v7:v5 by the field polynomial
11570 __ ghash_reduce(/*result*/v0, /*lo*/v5, /*hi*/v7, /*p*/v24, vzr, /*temp*/v3);
11571
11572 __ sub(blocks, blocks, 1);
11573 __ cbnz(blocks, L_ghash_loop);
11574 }
11575
11576 // The bit-reversed result is at this point in v0
11577 __ rev64(v0, __ T16B, v0);
11578 __ rbit(v0, __ T16B, v0);
11579
11580 __ st1(v0, __ T16B, state);
11581 __ ret(lr);
11582
11583 // bind label and generate local polynomial data
11584 __ align(wordSize * 2);
11585 __ bind(polynomial);
11586 __ emit_int64(0x87); // The low-order bits of the field
11587 // polynomial (i.e. p = z^7+z^2+z+1)
11588 // repeated in the low and high parts of a
11589 // 128-bit vector
11590 __ emit_int64(0x87);
11591
11592 // record the stub entry and end
11593 store_archive_data(stub_id, start, __ pc());
11594
11595 return start;
11596 }
11597
11598 address generate_ghash_processBlocks(address small) {
11599 StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
11600 int entry_count = StubInfo::entry_count(stub_id);
11601 assert(entry_count == 1, "sanity check");
11602 address start = load_archive_data(stub_id);
11603 if (start != nullptr) {
11604 return start;
11605 }
11606 Label polynomial; // local data generated after stub
11607 __ align(CodeEntryAlignment);
11608 StubCodeMark mark(this, stub_id);
11609 start = __ pc();
11610
11611 Register state = c_rarg0;
11612 Register subkeyH = c_rarg1;
11613 Register data = c_rarg2;
11614 Register blocks = c_rarg3;
11615
11616 const int unroll = 4;
11617
11618 __ cmp(blocks, (unsigned char)(unroll * 2));
11619 __ br(__ LT, small);
11620
11621 if (unroll > 1) {
11622 // Save state before entering routine
11623 __ sub(sp, sp, 4 * 16);
11624 __ st1(v12, v13, v14, v15, __ T16B, Address(sp));
11625 __ sub(sp, sp, 4 * 16);
11626 __ st1(v8, v9, v10, v11, __ T16B, Address(sp));
11627 }
11628
11629 __ ghash_processBlocks_wide(polynomial, state, subkeyH, data, blocks, unroll);
11630
11631 if (unroll > 1) {
11632 // And restore state
11633 __ ld1(v8, v9, v10, v11, __ T16B, __ post(sp, 4 * 16));
11634 __ ld1(v12, v13, v14, v15, __ T16B, __ post(sp, 4 * 16));
11635 }
11636
11637 __ cmp(blocks, (unsigned char)0);
11638 __ br(__ GT, small);
11639
11640 __ ret(lr);
11641
11642 // bind label and generate polynomial data
11643 __ align(wordSize * 2);
11644 __ bind(polynomial);
11645 __ emit_int64(0x87); // The low-order bits of the field
11646 // polynomial (i.e. p = z^7+z^2+z+1)
11647 // repeated in the low and high parts of a
11648 // 128-bit vector
11649 __ emit_int64(0x87);
11650
11651 // record the stub entry and end
11652 store_archive_data(stub_id, start, __ pc());
11653
11654 return start;
11655 }
11656
11657 void generate_base64_encode_simdround(Register src, Register dst,
11658 FloatRegister codec, u8 size) {
11659
11660 FloatRegister in0 = v4, in1 = v5, in2 = v6;
11661 FloatRegister out0 = v16, out1 = v17, out2 = v18, out3 = v19;
11662 FloatRegister ind0 = v20, ind1 = v21, ind2 = v22, ind3 = v23;
11663
11664 Assembler::SIMD_Arrangement arrangement = size == 16 ? __ T16B : __ T8B;
11665
11666 __ ld3(in0, in1, in2, arrangement, __ post(src, 3 * size));
11667
11668 __ ushr(ind0, arrangement, in0, 2);
11669
11670 __ ushr(ind1, arrangement, in1, 2);
11671 __ shl(in0, arrangement, in0, 6);
11672 __ orr(ind1, arrangement, ind1, in0);
11673 __ ushr(ind1, arrangement, ind1, 2);
11674
11675 __ ushr(ind2, arrangement, in2, 4);
11676 __ shl(in1, arrangement, in1, 4);
11677 __ orr(ind2, arrangement, in1, ind2);
11678 __ ushr(ind2, arrangement, ind2, 2);
11679
11680 __ shl(ind3, arrangement, in2, 2);
11681 __ ushr(ind3, arrangement, ind3, 2);
11682
11683 __ tbl(out0, arrangement, codec, 4, ind0);
11684 __ tbl(out1, arrangement, codec, 4, ind1);
11685 __ tbl(out2, arrangement, codec, 4, ind2);
11686 __ tbl(out3, arrangement, codec, 4, ind3);
11687
11688 __ st4(out0, out1, out2, out3, arrangement, __ post(dst, 4 * size));
11689 }
11690
11691 /**
11692 * Arguments:
11693 *
11694 * Input:
11695 * c_rarg0 - src_start
11696 * c_rarg1 - src_offset
11697 * c_rarg2 - src_length
11698 * c_rarg3 - dest_start
11699 * c_rarg4 - dest_offset
11700 * c_rarg5 - isURL
11701 *
11702 */
11703 address generate_base64_encodeBlock() {
11704
11705 StubId stub_id = StubId::stubgen_base64_encodeBlock_id;
11706 int entry_count = StubInfo::entry_count(stub_id);
11707 assert(entry_count == 1, "sanity check");
11708 address start = load_archive_data(stub_id);
11709 if (start != nullptr) {
11710 return start;
11711 }
11712 __ align(CodeEntryAlignment);
11713 StubCodeMark mark(this, stub_id);
11714 start = __ pc();
11715
11716 Register src = c_rarg0; // source array
11717 Register soff = c_rarg1; // source start offset
11718 Register send = c_rarg2; // source end offset
11719 Register dst = c_rarg3; // dest array
11720 Register doff = c_rarg4; // position for writing to dest array
11721 Register isURL = c_rarg5; // Base64 or URL character set
11722
11723 // c_rarg6 and c_rarg7 are free to use as temps
11724 Register codec = c_rarg6;
11725 Register length = c_rarg7;
11726
11727 Label ProcessData, Process48B, Process24B, Process3B, SIMDExit, Exit;
11728
11729 __ add(src, src, soff);
11730 __ add(dst, dst, doff);
11731 __ sub(length, send, soff);
11732
11733 // load the codec base address
11734 __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64));
11735 __ cbz(isURL, ProcessData);
11736 __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64URL));
11737
11738 __ BIND(ProcessData);
11739
11740 // too short to formup a SIMD loop, roll back
11741 __ cmp(length, (u1)24);
11742 __ br(Assembler::LT, Process3B);
11743
11744 __ ld1(v0, v1, v2, v3, __ T16B, Address(codec));
11745
11746 __ BIND(Process48B);
11747 __ cmp(length, (u1)48);
11748 __ br(Assembler::LT, Process24B);
11749 generate_base64_encode_simdround(src, dst, v0, 16);
11750 __ sub(length, length, 48);
11751 __ b(Process48B);
11752
11753 __ BIND(Process24B);
11754 __ cmp(length, (u1)24);
11755 __ br(Assembler::LT, SIMDExit);
11756 generate_base64_encode_simdround(src, dst, v0, 8);
11757 __ sub(length, length, 24);
11758
11759 __ BIND(SIMDExit);
11760 __ cbz(length, Exit);
11761
11762 __ BIND(Process3B);
11763 // 3 src bytes, 24 bits
11764 __ ldrb(r10, __ post(src, 1));
11765 __ ldrb(r11, __ post(src, 1));
11766 __ ldrb(r12, __ post(src, 1));
11767 __ orrw(r11, r11, r10, Assembler::LSL, 8);
11768 __ orrw(r12, r12, r11, Assembler::LSL, 8);
11769 // codec index
11770 __ ubfmw(r15, r12, 18, 23);
11771 __ ubfmw(r14, r12, 12, 17);
11772 __ ubfmw(r13, r12, 6, 11);
11773 __ andw(r12, r12, 63);
11774 // get the code based on the codec
11775 __ ldrb(r15, Address(codec, r15, Address::uxtw(0)));
11776 __ ldrb(r14, Address(codec, r14, Address::uxtw(0)));
11777 __ ldrb(r13, Address(codec, r13, Address::uxtw(0)));
11778 __ ldrb(r12, Address(codec, r12, Address::uxtw(0)));
11779 __ strb(r15, __ post(dst, 1));
11780 __ strb(r14, __ post(dst, 1));
11781 __ strb(r13, __ post(dst, 1));
11782 __ strb(r12, __ post(dst, 1));
11783 __ sub(length, length, 3);
11784 __ cbnz(length, Process3B);
11785
11786 __ BIND(Exit);
11787 __ ret(lr);
11788
11789 // record the stub entry and end
11790 store_archive_data(stub_id, start, __ pc());
11791
11792 return start;
11793 }
11794
11795 void generate_base64_decode_simdround(Register src, Register dst,
11796 FloatRegister codecL, FloatRegister codecH, int size, Label& Exit) {
11797
11798 FloatRegister in0 = v16, in1 = v17, in2 = v18, in3 = v19;
11799 FloatRegister out0 = v20, out1 = v21, out2 = v22;
11800
11801 FloatRegister decL0 = v23, decL1 = v24, decL2 = v25, decL3 = v26;
11802 FloatRegister decH0 = v28, decH1 = v29, decH2 = v30, decH3 = v31;
11803
11804 Label NoIllegalData, ErrorInLowerHalf, StoreLegalData;
11805
11806 Assembler::SIMD_Arrangement arrangement = size == 16 ? __ T16B : __ T8B;
11807
11808 __ ld4(in0, in1, in2, in3, arrangement, __ post(src, 4 * size));
11809
11810 // we need unsigned saturating subtract, to make sure all input values
11811 // in range [0, 63] will have 0U value in the higher half lookup
11812 __ uqsubv(decH0, __ T16B, in0, v27);
11813 __ uqsubv(decH1, __ T16B, in1, v27);
11814 __ uqsubv(decH2, __ T16B, in2, v27);
11815 __ uqsubv(decH3, __ T16B, in3, v27);
11816
11817 // lower half lookup
11818 __ tbl(decL0, arrangement, codecL, 4, in0);
11819 __ tbl(decL1, arrangement, codecL, 4, in1);
11820 __ tbl(decL2, arrangement, codecL, 4, in2);
11821 __ tbl(decL3, arrangement, codecL, 4, in3);
11822
11823 // higher half lookup
11824 __ tbx(decH0, arrangement, codecH, 4, decH0);
11825 __ tbx(decH1, arrangement, codecH, 4, decH1);
11826 __ tbx(decH2, arrangement, codecH, 4, decH2);
11827 __ tbx(decH3, arrangement, codecH, 4, decH3);
11828
11829 // combine lower and higher
11830 __ orr(decL0, arrangement, decL0, decH0);
11831 __ orr(decL1, arrangement, decL1, decH1);
11832 __ orr(decL2, arrangement, decL2, decH2);
11833 __ orr(decL3, arrangement, decL3, decH3);
11834
11835 // check illegal inputs, value larger than 63 (maximum of 6 bits)
11836 __ cm(Assembler::HI, decH0, arrangement, decL0, v27);
11837 __ cm(Assembler::HI, decH1, arrangement, decL1, v27);
11838 __ cm(Assembler::HI, decH2, arrangement, decL2, v27);
11839 __ cm(Assembler::HI, decH3, arrangement, decL3, v27);
11840 __ orr(in0, arrangement, decH0, decH1);
11841 __ orr(in1, arrangement, decH2, decH3);
11842 __ orr(in2, arrangement, in0, in1);
11843 __ umaxv(in3, arrangement, in2);
11844 __ umov(rscratch2, in3, __ B, 0);
11845
11846 // get the data to output
11847 __ shl(out0, arrangement, decL0, 2);
11848 __ ushr(out1, arrangement, decL1, 4);
11849 __ orr(out0, arrangement, out0, out1);
11850 __ shl(out1, arrangement, decL1, 4);
11851 __ ushr(out2, arrangement, decL2, 2);
11852 __ orr(out1, arrangement, out1, out2);
11853 __ shl(out2, arrangement, decL2, 6);
11854 __ orr(out2, arrangement, out2, decL3);
11855
11856 __ cbz(rscratch2, NoIllegalData);
11857
11858 // handle illegal input
11859 __ umov(r10, in2, __ D, 0);
11860 if (size == 16) {
11861 __ cbnz(r10, ErrorInLowerHalf);
11862
11863 // illegal input is in higher half, store the lower half now.
11864 __ st3(out0, out1, out2, __ T8B, __ post(dst, 24));
11865
11866 __ umov(r10, in2, __ D, 1);
11867 __ umov(r11, out0, __ D, 1);
11868 __ umov(r12, out1, __ D, 1);
11869 __ umov(r13, out2, __ D, 1);
11870 __ b(StoreLegalData);
11871
11872 __ BIND(ErrorInLowerHalf);
11873 }
11874 __ umov(r11, out0, __ D, 0);
11875 __ umov(r12, out1, __ D, 0);
11876 __ umov(r13, out2, __ D, 0);
11877
11878 __ BIND(StoreLegalData);
11879 __ tbnz(r10, 5, Exit); // 0xff indicates illegal input
11880 __ strb(r11, __ post(dst, 1));
11881 __ strb(r12, __ post(dst, 1));
11882 __ strb(r13, __ post(dst, 1));
11883 __ lsr(r10, r10, 8);
11884 __ lsr(r11, r11, 8);
11885 __ lsr(r12, r12, 8);
11886 __ lsr(r13, r13, 8);
11887 __ b(StoreLegalData);
11888
11889 __ BIND(NoIllegalData);
11890 __ st3(out0, out1, out2, arrangement, __ post(dst, 3 * size));
11891 }
11892
11893
11894 /**
11895 * Arguments:
11896 *
11897 * Input:
11898 * c_rarg0 - src_start
11899 * c_rarg1 - src_offset
11900 * c_rarg2 - src_length
11901 * c_rarg3 - dest_start
11902 * c_rarg4 - dest_offset
11903 * c_rarg5 - isURL
11904 * c_rarg6 - isMIME
11905 *
11906 */
11907 address generate_base64_decodeBlock() {
11908
11909 // The SIMD part of this Base64 decode intrinsic is based on the algorithm outlined
11910 // on http://0x80.pl/articles/base64-simd-neon.html#encoding-quadwords, in section
11911 // titled "Base64 decoding".
11912
11913 StubId stub_id = StubId::stubgen_base64_decodeBlock_id;
11914 int entry_count = StubInfo::entry_count(stub_id);
11915 assert(entry_count == 1, "sanity check");
11916 address start = load_archive_data(stub_id);
11917 if (start != nullptr) {
11918 return start;
11919 }
11920 __ align(CodeEntryAlignment);
11921 StubCodeMark mark(this, stub_id);
11922 start = __ pc();
11923
11924 Register src = c_rarg0; // source array
11925 Register soff = c_rarg1; // source start offset
11926 Register send = c_rarg2; // source end offset
11927 Register dst = c_rarg3; // dest array
11928 Register doff = c_rarg4; // position for writing to dest array
11929 Register isURL = c_rarg5; // Base64 or URL character set
11930 Register isMIME = c_rarg6; // Decoding MIME block - unused in this implementation
11931
11932 Register length = send; // reuse send as length of source data to process
11933
11934 Register simd_codec = c_rarg6;
11935 Register nosimd_codec = c_rarg7;
11936
11937 Label ProcessData, Process64B, Process32B, Process4B, SIMDEnter, SIMDExit, Exit;
11938
11939 __ enter();
11940
11941 __ add(src, src, soff);
11942 __ add(dst, dst, doff);
11943
11944 __ mov(doff, dst);
11945
11946 __ sub(length, send, soff);
11947 __ bfm(length, zr, 0, 1);
11948
11949 __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForNoSIMD));
11950 __ cbz(isURL, ProcessData);
11951 __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForNoSIMD));
11952
11953 __ BIND(ProcessData);
11954 __ mov(rscratch1, length);
11955 __ cmp(length, (u1)144); // 144 = 80 + 64
11956 __ br(Assembler::LT, Process4B);
11957
11958 // In the MIME case, the line length cannot be more than 76
11959 // bytes (see RFC 2045). This is too short a block for SIMD
11960 // to be worthwhile, so we use non-SIMD here.
11961 __ movw(rscratch1, 79);
11962
11963 __ BIND(Process4B);
11964 __ ldrw(r14, __ post(src, 4));
11965 __ ubfxw(r10, r14, 0, 8);
11966 __ ubfxw(r11, r14, 8, 8);
11967 __ ubfxw(r12, r14, 16, 8);
11968 __ ubfxw(r13, r14, 24, 8);
11969 // get the de-code
11970 __ ldrb(r10, Address(nosimd_codec, r10, Address::uxtw(0)));
11971 __ ldrb(r11, Address(nosimd_codec, r11, Address::uxtw(0)));
11972 __ ldrb(r12, Address(nosimd_codec, r12, Address::uxtw(0)));
11973 __ ldrb(r13, Address(nosimd_codec, r13, Address::uxtw(0)));
11974 // error detection, 255u indicates an illegal input
11975 __ orrw(r14, r10, r11);
11976 __ orrw(r15, r12, r13);
11977 __ orrw(r14, r14, r15);
11978 __ tbnz(r14, 7, Exit);
11979 // recover the data
11980 __ lslw(r14, r10, 10);
11981 __ bfiw(r14, r11, 4, 6);
11982 __ bfmw(r14, r12, 2, 5);
11983 __ rev16w(r14, r14);
11984 __ bfiw(r13, r12, 6, 2);
11985 __ strh(r14, __ post(dst, 2));
11986 __ strb(r13, __ post(dst, 1));
11987 // non-simd loop
11988 __ subsw(rscratch1, rscratch1, 4);
11989 __ br(Assembler::GT, Process4B);
11990
11991 // if exiting from PreProcess80B, rscratch1 == -1;
11992 // otherwise, rscratch1 == 0.
11993 __ cbzw(rscratch1, Exit);
11994 __ sub(length, length, 80);
11995
11996 __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForSIMD));
11997 __ cbz(isURL, SIMDEnter);
11998 __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForSIMD));
11999
12000 __ BIND(SIMDEnter);
12001 __ ld1(v0, v1, v2, v3, __ T16B, __ post(simd_codec, 64));
12002 __ ld1(v4, v5, v6, v7, __ T16B, Address(simd_codec));
12003 __ mov(rscratch1, 63);
12004 __ dup(v27, __ T16B, rscratch1);
12005
12006 __ BIND(Process64B);
12007 __ cmp(length, (u1)64);
12008 __ br(Assembler::LT, Process32B);
12009 generate_base64_decode_simdround(src, dst, v0, v4, 16, Exit);
12010 __ sub(length, length, 64);
12011 __ b(Process64B);
12012
12013 __ BIND(Process32B);
12014 __ cmp(length, (u1)32);
12015 __ br(Assembler::LT, SIMDExit);
12016 generate_base64_decode_simdround(src, dst, v0, v4, 8, Exit);
12017 __ sub(length, length, 32);
12018 __ b(Process32B);
12019
12020 __ BIND(SIMDExit);
12021 __ cbz(length, Exit);
12022 __ movw(rscratch1, length);
12023 __ b(Process4B);
12024
12025 __ BIND(Exit);
12026 __ sub(c_rarg0, dst, doff);
12027
12028 __ leave();
12029 __ ret(lr);
12030
12031 // record the stub entry and end
12032 store_archive_data(stub_id, start, __ pc());
12033
12034 return start;
12035 }
12036
12037 // Support for spin waits.
12038 address generate_spin_wait() {
12039 StubId stub_id = StubId::stubgen_spin_wait_id;
12040 int entry_count = StubInfo::entry_count(stub_id);
12041 assert(entry_count == 1, "sanity check");
12042 address start = load_archive_data(stub_id);
12043 if (start != nullptr) {
12044 return start;
12045 }
12046 __ align(CodeEntryAlignment);
12047 StubCodeMark mark(this, stub_id);
12048 start = __ pc();
12049
12050 __ spin_wait();
12051 __ ret(lr);
12052
12053 // record the stub entry and end
12054 store_archive_data(stub_id, start, __ pc());
12055
12056 return start;
12057 }
12058
12059 void generate_lookup_secondary_supers_table_stub() {
12060 StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id;
12061 GrowableArray<address> entries;
12062 int entry_count = StubInfo::entry_count(stub_id);
12063 assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check");
12064 address start = load_archive_data(stub_id, &entries);
12065 if (start != nullptr) {
12066 assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1,
12067 "unexpected extra entry count %d", entries.length());
12068 StubRoutines::_lookup_secondary_supers_table_stubs[0] = start;
12069 for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
12070 StubRoutines::_lookup_secondary_supers_table_stubs[slot] = entries.at(slot - 1);
12071 }
12072 return;
12073 }
12074
12075 StubCodeMark mark(this, stub_id);
12076
12077 const Register
12078 r_super_klass = r0,
12079 r_array_base = r1,
12080 r_array_length = r2,
12081 r_array_index = r3,
12082 r_sub_klass = r4,
12083 r_bitmap = rscratch2,
12084 result = r5;
12085 const FloatRegister
12086 vtemp = v0;
12087
12088 for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
12089 address next_entry = __ pc();
12090 StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry;
12091 if (slot == 0) {
12092 start = next_entry;
12093 } else {
12094 entries.append(next_entry);
12095 }
12096 Label L_success;
12097 __ enter();
12098 __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass,
12099 r_array_base, r_array_length, r_array_index,
12100 vtemp, result, slot,
12101 /*stub_is_near*/true);
12102 __ leave();
12103 __ ret(lr);
12104 }
12105 // record the stub entry and end plus all the auxiliary entries
12106 store_archive_data(stub_id, start, __ pc(), &entries);
12107 }
12108
12109 // Slow path implementation for UseSecondarySupersTable.
12110 address generate_lookup_secondary_supers_table_slow_path_stub() {
12111 StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id;
12112 int entry_count = StubInfo::entry_count(stub_id);
12113 assert(entry_count == 1, "sanity check");
12114 address start = load_archive_data(stub_id);
12115 if (start != nullptr) {
12116 return start;
12117 }
12118 StubCodeMark mark(this, stub_id);
12119 start = __ pc();
12120 const Register
12121 r_super_klass = r0, // argument
12122 r_array_base = r1, // argument
12123 temp1 = r2, // temp
12124 r_array_index = r3, // argument
12125 r_bitmap = rscratch2, // argument
12126 result = r5; // argument
12127
12128 __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, result);
12129 __ ret(lr);
12130
12131 // record the stub entry and end
12132 store_archive_data(stub_id, start, __ pc());
12133
12134 return start;
12135 }
12136
12137 #if defined (LINUX) && !defined (__ARM_FEATURE_ATOMICS)
12138
12139 // ARMv8.1 LSE versions of the atomic stubs used by AtomicAccess::PlatformXX.
12140 //
12141 // If LSE is in use, generate LSE versions of all the stubs. The
12142 // non-LSE versions are in atomic_aarch64.S.
12143
12144 // class AtomicStubMark records the entry point of a stub and the
12145 // stub pointer which will point to it. The stub pointer is set to
12146 // the entry point when ~AtomicStubMark() is called, which must be
12147 // after ICache::invalidate_range. This ensures safe publication of
12148 // the generated code.
12149 class AtomicStubMark {
12150 address _entry_point;
12151 aarch64_atomic_stub_t *_stub;
12152 MacroAssembler *_masm;
12153 public:
12154 AtomicStubMark(MacroAssembler *masm, aarch64_atomic_stub_t *stub) {
12155 _masm = masm;
12156 __ align(32);
12157 _entry_point = __ pc();
12158 _stub = stub;
12159 }
12160 ~AtomicStubMark() {
12161 *_stub = (aarch64_atomic_stub_t)_entry_point;
12162 }
12163 };
12164
12165 // NB: For memory_order_conservative we need a trailing membar after
12166 // LSE atomic operations but not a leading membar.
12167 //
12168 // We don't need a leading membar because a clause in the Arm ARM
12169 // says:
12170 //
12171 // Barrier-ordered-before
12172 //
12173 // Barrier instructions order prior Memory effects before subsequent
12174 // Memory effects generated by the same Observer. A read or a write
12175 // RW1 is Barrier-ordered-before a read or a write RW 2 from the same
12176 // Observer if and only if RW1 appears in program order before RW 2
12177 // and [ ... ] at least one of RW 1 and RW 2 is generated by an atomic
12178 // instruction with both Acquire and Release semantics.
12179 //
12180 // All the atomic instructions {ldaddal, swapal, casal} have Acquire
12181 // and Release semantics, therefore we don't need a leading
12182 // barrier. However, there is no corresponding Barrier-ordered-after
12183 // relationship, therefore we need a trailing membar to prevent a
12184 // later store or load from being reordered with the store in an
12185 // atomic instruction.
12186 //
12187 // This was checked by using the herd7 consistency model simulator
12188 // (http://diy.inria.fr/) with this test case:
12189 //
12190 // AArch64 LseCas
12191 // { 0:X1=x; 0:X2=y; 1:X1=x; 1:X2=y; }
12192 // P0 | P1;
12193 // LDR W4, [X2] | MOV W3, #0;
12194 // DMB LD | MOV W4, #1;
12195 // LDR W3, [X1] | CASAL W3, W4, [X1];
12196 // | DMB ISH;
12197 // | STR W4, [X2];
12198 // exists
12199 // (0:X3=0 /\ 0:X4=1)
12200 //
12201 // If X3 == 0 && X4 == 1, the store to y in P1 has been reordered
12202 // with the store to x in P1. Without the DMB in P1 this may happen.
12203 //
12204 // At the time of writing we don't know of any AArch64 hardware that
12205 // reorders stores in this way, but the Reference Manual permits it.
12206
12207 void gen_cas_entry(Assembler::operand_size size,
12208 atomic_memory_order order) {
12209 Register prev = r3, ptr = c_rarg0, compare_val = c_rarg1,
12210 exchange_val = c_rarg2;
12211 bool acquire, release;
12212 switch (order) {
12213 case memory_order_relaxed:
12214 acquire = false;
12215 release = false;
12216 break;
12217 case memory_order_release:
12218 acquire = false;
12219 release = true;
12220 break;
12221 default:
12222 acquire = true;
12223 release = true;
12224 break;
12225 }
12226 __ mov(prev, compare_val);
12227 __ lse_cas(prev, exchange_val, ptr, size, acquire, release, /*not_pair*/true);
12228 if (order == memory_order_conservative) {
12229 __ membar(Assembler::StoreStore|Assembler::StoreLoad);
12230 }
12231 if (size == Assembler::xword) {
12232 __ mov(r0, prev);
12233 } else {
12234 __ movw(r0, prev);
12235 }
12236 __ ret(lr);
12237 }
12238
12239 void gen_ldadd_entry(Assembler::operand_size size, atomic_memory_order order) {
12240 Register prev = r2, addr = c_rarg0, incr = c_rarg1;
12241 // If not relaxed, then default to conservative. Relaxed is the only
12242 // case we use enough to be worth specializing.
12243 if (order == memory_order_relaxed) {
12244 __ ldadd(size, incr, prev, addr);
12245 } else {
12246 __ ldaddal(size, incr, prev, addr);
12247 __ membar(Assembler::StoreStore|Assembler::StoreLoad);
12248 }
12249 if (size == Assembler::xword) {
12250 __ mov(r0, prev);
12251 } else {
12252 __ movw(r0, prev);
12253 }
12254 __ ret(lr);
12255 }
12256
12257 void gen_swpal_entry(Assembler::operand_size size) {
12258 Register prev = r2, addr = c_rarg0, incr = c_rarg1;
12259 __ swpal(size, incr, prev, addr);
12260 __ membar(Assembler::StoreStore|Assembler::StoreLoad);
12261 if (size == Assembler::xword) {
12262 __ mov(r0, prev);
12263 } else {
12264 __ movw(r0, prev);
12265 }
12266 __ ret(lr);
12267 }
12268
12269 void generate_atomic_entry_points() {
12270 if (! UseLSE) {
12271 return;
12272 }
12273 StubId stub_id = StubId::stubgen_atomic_entry_points_id;
12274 GrowableArray<address> entries;
12275 int entry_count = StubInfo::entry_count(stub_id);
12276 address start = load_archive_data(stub_id, &entries);
12277 if (start != nullptr) {
12278 assert(entries.length() == entry_count - 1,
12279 "unexpected extra entry count %d", entries.length());
12280 aarch64_atomic_fetch_add_4_impl = (aarch64_atomic_stub_t)start;
12281 int idx = 0;
12282 aarch64_atomic_fetch_add_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12283 aarch64_atomic_fetch_add_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12284 aarch64_atomic_fetch_add_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12285 aarch64_atomic_xchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12286 aarch64_atomic_xchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12287 aarch64_atomic_cmpxchg_1_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12288 aarch64_atomic_cmpxchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12289 aarch64_atomic_cmpxchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12290 aarch64_atomic_cmpxchg_1_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12291 aarch64_atomic_cmpxchg_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12292 aarch64_atomic_cmpxchg_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12293 aarch64_atomic_cmpxchg_4_release_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12294 aarch64_atomic_cmpxchg_8_release_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12295 aarch64_atomic_cmpxchg_4_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12296 aarch64_atomic_cmpxchg_8_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12297 assert(idx == entries.length(), "sanity!");
12298 return;
12299 }
12300
12301 __ align(CodeEntryAlignment);
12302 StubCodeMark mark(this, stub_id);
12303 start = __ pc();
12304 address end;
12305 {
12306 // ADD, memory_order_conservative
12307 AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl);
12308 gen_ldadd_entry(Assembler::word, memory_order_conservative);
12309
12310 AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl);
12311 gen_ldadd_entry(Assembler::xword, memory_order_conservative);
12312
12313 // ADD, memory_order_relaxed
12314 AtomicStubMark mark_fetch_add_4_relaxed
12315 (_masm, &aarch64_atomic_fetch_add_4_relaxed_impl);
12316 gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed);
12317
12318 AtomicStubMark mark_fetch_add_8_relaxed
12319 (_masm, &aarch64_atomic_fetch_add_8_relaxed_impl);
12320 gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed);
12321
12322 // XCHG, memory_order_conservative
12323 AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl);
12324 gen_swpal_entry(Assembler::word);
12325
12326 AtomicStubMark mark_xchg_8(_masm, &aarch64_atomic_xchg_8_impl);
12327 gen_swpal_entry(Assembler::xword);
12328
12329 // CAS, memory_order_conservative
12330 AtomicStubMark mark_cmpxchg_1(_masm, &aarch64_atomic_cmpxchg_1_impl);
12331 gen_cas_entry(MacroAssembler::byte, memory_order_conservative);
12332
12333 AtomicStubMark mark_cmpxchg_4(_masm, &aarch64_atomic_cmpxchg_4_impl);
12334 gen_cas_entry(MacroAssembler::word, memory_order_conservative);
12335
12336 AtomicStubMark mark_cmpxchg_8(_masm, &aarch64_atomic_cmpxchg_8_impl);
12337 gen_cas_entry(MacroAssembler::xword, memory_order_conservative);
12338
12339 // CAS, memory_order_relaxed
12340 AtomicStubMark mark_cmpxchg_1_relaxed
12341 (_masm, &aarch64_atomic_cmpxchg_1_relaxed_impl);
12342 gen_cas_entry(MacroAssembler::byte, memory_order_relaxed);
12343
12344 AtomicStubMark mark_cmpxchg_4_relaxed
12345 (_masm, &aarch64_atomic_cmpxchg_4_relaxed_impl);
12346 gen_cas_entry(MacroAssembler::word, memory_order_relaxed);
12347
12348 AtomicStubMark mark_cmpxchg_8_relaxed
12349 (_masm, &aarch64_atomic_cmpxchg_8_relaxed_impl);
12350 gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
12351
12352 AtomicStubMark mark_cmpxchg_4_release
12353 (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
12354 gen_cas_entry(MacroAssembler::word, memory_order_release);
12355
12356 AtomicStubMark mark_cmpxchg_8_release
12357 (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
12358 gen_cas_entry(MacroAssembler::xword, memory_order_release);
12359
12360 AtomicStubMark mark_cmpxchg_4_seq_cst
12361 (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
12362 gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
12363
12364 AtomicStubMark mark_cmpxchg_8_seq_cst
12365 (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
12366 gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
12367
12368 end = __ pc();
12369
12370 ICache::invalidate_range(start, end - start);
12371 // exit block to force update of AtomicStubMark targets
12372 }
12373
12374 assert(start == (address)aarch64_atomic_fetch_add_4_impl,
12375 "atomic stub should be at start of buffer");
12376 // record the stub start and end plus all the entries saved by the
12377 // AtomicStubMark destructor
12378 entries.append((address)aarch64_atomic_fetch_add_8_impl);
12379 entries.append((address)aarch64_atomic_fetch_add_4_relaxed_impl);
12380 entries.append((address)aarch64_atomic_fetch_add_8_relaxed_impl);
12381 entries.append((address)aarch64_atomic_xchg_4_impl);
12382 entries.append((address)aarch64_atomic_xchg_8_impl);
12383 entries.append((address)aarch64_atomic_cmpxchg_1_impl);
12384 entries.append((address)aarch64_atomic_cmpxchg_4_impl);
12385 entries.append((address)aarch64_atomic_cmpxchg_8_impl);
12386 entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
12387 entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
12388 entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
12389 entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
12390 entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
12391 entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
12392 entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
12393
12394 assert(entries.length() == entry_count - 1,
12395 "unexpected extra entry count %d", entries.length());
12396
12397 store_archive_data(stub_id, start, end, &entries);
12398 }
12399 #endif // LINUX
12400
12401 address generate_cont_thaw(Continuation::thaw_kind kind) {
12402 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
12403 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
12404
12405 address start = __ pc();
12406
12407 if (return_barrier) {
12408 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
12409 __ mov(sp, rscratch1);
12410 }
12411 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12412
12413 if (return_barrier) {
12414 // preserve possible return value from a method returning to the return barrier
12415 __ fmovd(rscratch1, v0);
12416 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
12417 }
12418
12419 __ movw(c_rarg1, (return_barrier ? 1 : 0));
12420 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
12421 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
12422
12423 if (return_barrier) {
12424 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12425 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
12426 __ fmovd(v0, rscratch1);
12427 }
12428 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12429
12430
12431 Label thaw_success;
12432 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
12433 __ cbnz(rscratch2, thaw_success);
12434 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
12435 __ br(rscratch1);
12436 __ bind(thaw_success);
12437
12438 // make room for the thawed frames
12439 __ sub(rscratch1, sp, rscratch2);
12440 __ andr(rscratch1, rscratch1, -16); // align
12441 __ mov(sp, rscratch1);
12442
12443 if (return_barrier) {
12444 // save original return value -- again
12445 __ fmovd(rscratch1, v0);
12446 __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
12447 }
12448
12449 // If we want, we can templatize thaw by kind, and have three different entries
12450 __ movw(c_rarg1, (uint32_t)kind);
12451
12452 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
12453 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
12454
12455 if (return_barrier) {
12456 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12457 __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
12458 __ fmovd(v0, rscratch1);
12459 } else {
12460 __ mov(r0, zr); // return 0 (success) from doYield
12461 }
12462
12463 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
12464 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
12465 __ mov(rfp, sp);
12466
12467 if (return_barrier_exception) {
12468 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
12469 __ authenticate_return_address(c_rarg1);
12470 __ verify_oop(r0);
12471 // save return value containing the exception oop in callee-saved R19
12472 __ mov(r19, r0);
12473
12474 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
12475
12476 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
12477 // __ reinitialize_ptrue();
12478
12479 // see OptoRuntime::generate_exception_blob: r0 -- exception oop, r3 -- exception pc
12480
12481 __ mov(r1, r0); // the exception handler
12482 __ mov(r0, r19); // restore return value containing the exception oop
12483 __ verify_oop(r0);
12484
12485 __ leave();
12486 __ mov(r3, lr);
12487 __ br(r1); // the exception handler
12488 } else {
12489 // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame
12490 __ leave();
12491 __ ret(lr);
12492 }
12493
12494 return start;
12495 }
12496
12497 address generate_cont_thaw() {
12498 if (!Continuations::enabled()) return nullptr;
12499
12500 StubId stub_id = StubId::stubgen_cont_thaw_id;
12501 int entry_count = StubInfo::entry_count(stub_id);
12502 assert(entry_count == 1, "sanity check");
12503 address start = load_archive_data(stub_id);
12504 if (start != nullptr) {
12505 return start;
12506 }
12507 StubCodeMark mark(this, stub_id);
12508 start = __ pc();
12509 generate_cont_thaw(Continuation::thaw_top);
12510
12511 // record the stub start and end
12512 store_archive_data(stub_id, start, __ pc());
12513
12514 return start;
12515 }
12516
12517 address generate_cont_returnBarrier() {
12518 if (!Continuations::enabled()) return nullptr;
12519
12520 // TODO: will probably need multiple return barriers depending on return type
12521 StubId stub_id = StubId::stubgen_cont_returnBarrier_id;
12522 int entry_count = StubInfo::entry_count(stub_id);
12523 assert(entry_count == 1, "sanity check");
12524 address start = load_archive_data(stub_id);
12525 if (start != nullptr) {
12526 return start;
12527 }
12528 StubCodeMark mark(this, stub_id);
12529 start = __ pc();
12530
12531 generate_cont_thaw(Continuation::thaw_return_barrier);
12532
12533 // record the stub start and end
12534 store_archive_data(stub_id, start, __ pc());
12535
12536 return start;
12537 }
12538
12539 address generate_cont_returnBarrier_exception() {
12540 if (!Continuations::enabled()) return nullptr;
12541
12542 StubId stub_id = StubId::stubgen_cont_returnBarrierExc_id;
12543 int entry_count = StubInfo::entry_count(stub_id);
12544 assert(entry_count == 1, "sanity check");
12545 address start = load_archive_data(stub_id);
12546 if (start != nullptr) {
12547 return start;
12548 }
12549 StubCodeMark mark(this, stub_id);
12550 start = __ pc();
12551
12552 generate_cont_thaw(Continuation::thaw_return_barrier_exception);
12553
12554 // record the stub start and end
12555 store_archive_data(stub_id, start, __ pc());
12556
12557 return start;
12558 }
12559
12560 address generate_cont_preempt_stub() {
12561 if (!Continuations::enabled()) return nullptr;
12562 StubId stub_id = StubId::stubgen_cont_preempt_id;
12563 int entry_count = StubInfo::entry_count(stub_id);
12564 assert(entry_count == 1, "sanity check");
12565 address start = load_archive_data(stub_id);
12566 if (start != nullptr) {
12567 return start;
12568 }
12569 StubCodeMark mark(this, stub_id);
12570 start = __ pc();
12571
12572 __ reset_last_Java_frame(true);
12573
12574 // Set sp to enterSpecial frame, i.e. remove all frames copied into the heap.
12575 __ ldr(rscratch2, Address(rthread, JavaThread::cont_entry_offset()));
12576 __ mov(sp, rscratch2);
12577
12578 Label preemption_cancelled;
12579 __ ldrb(rscratch1, Address(rthread, JavaThread::preemption_cancelled_offset()));
12580 __ cbnz(rscratch1, preemption_cancelled);
12581
12582 // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount.
12583 SharedRuntime::continuation_enter_cleanup(_masm);
12584 __ leave();
12585 __ ret(lr);
12586
12587 // We acquired the monitor after freezing the frames so call thaw to continue execution.
12588 __ bind(preemption_cancelled);
12589 __ strb(zr, Address(rthread, JavaThread::preemption_cancelled_offset()));
12590 __ lea(rfp, Address(sp, checked_cast<int32_t>(ContinuationEntry::size())));
12591 __ lea(rscratch1, ExternalAddress(ContinuationEntry::thaw_call_pc_address()));
12592 __ ldr(rscratch1, Address(rscratch1));
12593 __ br(rscratch1);
12594
12595 // record the stub start and end
12596 store_archive_data(stub_id, start, __ pc());
12597
12598 return start;
12599 }
12600
12601 // In sun.security.util.math.intpoly.IntegerPolynomial1305, integers
12602 // are represented as long[5], with BITS_PER_LIMB = 26.
12603 // Pack five 26-bit limbs into three 64-bit registers.
12604 void pack_26(Register dest0, Register dest1, Register dest2, Register src) {
12605 __ ldp(dest0, rscratch1, Address(src, 0)); // 26 bits
12606 __ add(dest0, dest0, rscratch1, Assembler::LSL, 26); // 26 bits
12607 __ ldp(rscratch1, rscratch2, Address(src, 2 * sizeof (jlong)));
12608 __ add(dest0, dest0, rscratch1, Assembler::LSL, 52); // 12 bits
12609
12610 __ add(dest1, zr, rscratch1, Assembler::LSR, 12); // 14 bits
12611 __ add(dest1, dest1, rscratch2, Assembler::LSL, 14); // 26 bits
12612 __ ldr(rscratch1, Address(src, 4 * sizeof (jlong)));
12613 __ add(dest1, dest1, rscratch1, Assembler::LSL, 40); // 24 bits
12614
12615 if (dest2->is_valid()) {
12616 __ add(dest2, zr, rscratch1, Assembler::LSR, 24); // 2 bits
12617 } else {
12618 #ifdef ASSERT
12619 Label OK;
12620 __ cmp(zr, rscratch1, Assembler::LSR, 24); // 2 bits
12621 __ br(__ EQ, OK);
12622 __ stop("high bits of Poly1305 integer should be zero");
12623 __ should_not_reach_here();
12624 __ bind(OK);
12625 #endif
12626 }
12627 }
12628
12629 // As above, but return only a 128-bit integer, packed into two
12630 // 64-bit registers.
12631 void pack_26(Register dest0, Register dest1, Register src) {
12632 pack_26(dest0, dest1, noreg, src);
12633 }
12634
12635 // Multiply and multiply-accumulate unsigned 64-bit registers.
12636 void wide_mul(Register prod_lo, Register prod_hi, Register n, Register m) {
12637 __ mul(prod_lo, n, m);
12638 __ umulh(prod_hi, n, m);
12639 }
12640 void wide_madd(Register sum_lo, Register sum_hi, Register n, Register m) {
12641 wide_mul(rscratch1, rscratch2, n, m);
12642 __ adds(sum_lo, sum_lo, rscratch1);
12643 __ adc(sum_hi, sum_hi, rscratch2);
12644 }
12645
12646 // Poly1305, RFC 7539
12647
12648 // See https://loup-vaillant.fr/tutorials/poly1305-design for a
12649 // description of the tricks used to simplify and accelerate this
12650 // computation.
12651
12652 address generate_poly1305_processBlocks() {
12653 StubId stub_id = StubId::stubgen_poly1305_processBlocks_id;
12654 int entry_count = StubInfo::entry_count(stub_id);
12655 assert(entry_count == 1, "sanity check");
12656 address start = load_archive_data(stub_id);
12657 if (start != nullptr) {
12658 return start;
12659 }
12660 __ align(CodeEntryAlignment);
12661 StubCodeMark mark(this, stub_id);
12662 start = __ pc();
12663 Label here;
12664 __ enter();
12665 RegSet callee_saved = RegSet::range(r19, r28);
12666 __ push(callee_saved, sp);
12667
12668 RegSetIterator<Register> regs = (RegSet::range(c_rarg0, r28) - r18_tls - rscratch1 - rscratch2).begin();
12669
12670 // Arguments
12671 const Register input_start = *regs, length = *++regs, acc_start = *++regs, r_start = *++regs;
12672
12673 // R_n is the 128-bit randomly-generated key, packed into two
12674 // registers. The caller passes this key to us as long[5], with
12675 // BITS_PER_LIMB = 26.
12676 const Register R_0 = *++regs, R_1 = *++regs;
12677 pack_26(R_0, R_1, r_start);
12678
12679 // RR_n is (R_n >> 2) * 5
12680 const Register RR_0 = *++regs, RR_1 = *++regs;
12681 __ lsr(RR_0, R_0, 2);
12682 __ add(RR_0, RR_0, RR_0, Assembler::LSL, 2);
12683 __ lsr(RR_1, R_1, 2);
12684 __ add(RR_1, RR_1, RR_1, Assembler::LSL, 2);
12685
12686 // U_n is the current checksum
12687 const Register U_0 = *++regs, U_1 = *++regs, U_2 = *++regs;
12688 pack_26(U_0, U_1, U_2, acc_start);
12689
12690 static constexpr int BLOCK_LENGTH = 16;
12691 Label DONE, LOOP;
12692
12693 __ cmp(length, checked_cast<u1>(BLOCK_LENGTH));
12694 __ br(Assembler::LT, DONE); {
12695 __ bind(LOOP);
12696
12697 // S_n is to be the sum of U_n and the next block of data
12698 const Register S_0 = *++regs, S_1 = *++regs, S_2 = *++regs;
12699 __ ldp(S_0, S_1, __ post(input_start, 2 * wordSize));
12700 __ adds(S_0, U_0, S_0);
12701 __ adcs(S_1, U_1, S_1);
12702 __ adc(S_2, U_2, zr);
12703 __ add(S_2, S_2, 1);
12704
12705 const Register U_0HI = *++regs, U_1HI = *++regs;
12706
12707 // NB: this logic depends on some of the special properties of
12708 // Poly1305 keys. In particular, because we know that the top
12709 // four bits of R_0 and R_1 are zero, we can add together
12710 // partial products without any risk of needing to propagate a
12711 // carry out.
12712 wide_mul(U_0, U_0HI, S_0, R_0); wide_madd(U_0, U_0HI, S_1, RR_1); wide_madd(U_0, U_0HI, S_2, RR_0);
12713 wide_mul(U_1, U_1HI, S_0, R_1); wide_madd(U_1, U_1HI, S_1, R_0); wide_madd(U_1, U_1HI, S_2, RR_1);
12714 __ andr(U_2, R_0, 3);
12715 __ mul(U_2, S_2, U_2);
12716
12717 // Recycle registers S_0, S_1, S_2
12718 regs = (regs.remaining() + S_0 + S_1 + S_2).begin();
12719
12720 // Partial reduction mod 2**130 - 5
12721 __ adds(U_1, U_0HI, U_1);
12722 __ adc(U_2, U_1HI, U_2);
12723 // Sum now in U_2:U_1:U_0.
12724 // Dead: U_0HI, U_1HI.
12725 regs = (regs.remaining() + U_0HI + U_1HI).begin();
12726
12727 // U_2:U_1:U_0 += (U_2 >> 2) * 5 in two steps
12728
12729 // First, U_2:U_1:U_0 += (U_2 >> 2)
12730 __ lsr(rscratch1, U_2, 2);
12731 __ andr(U_2, U_2, (u8)3);
12732 __ adds(U_0, U_0, rscratch1);
12733 __ adcs(U_1, U_1, zr);
12734 __ adc(U_2, U_2, zr);
12735 // Second, U_2:U_1:U_0 += (U_2 >> 2) << 2
12736 __ adds(U_0, U_0, rscratch1, Assembler::LSL, 2);
12737 __ adcs(U_1, U_1, zr);
12738 __ adc(U_2, U_2, zr);
12739
12740 __ sub(length, length, checked_cast<u1>(BLOCK_LENGTH));
12741 __ cmp(length, checked_cast<u1>(BLOCK_LENGTH));
12742 __ br(~ Assembler::LT, LOOP);
12743 }
12744
12745 // Further reduce modulo 2^130 - 5
12746 __ lsr(rscratch1, U_2, 2);
12747 __ add(rscratch1, rscratch1, rscratch1, Assembler::LSL, 2); // rscratch1 = U_2 * 5
12748 __ adds(U_0, U_0, rscratch1); // U_0 += U_2 * 5
12749 __ adcs(U_1, U_1, zr);
12750 __ andr(U_2, U_2, (u1)3);
12751 __ adc(U_2, U_2, zr);
12752
12753 // Unpack the sum into five 26-bit limbs and write to memory.
12754 __ ubfiz(rscratch1, U_0, 0, 26);
12755 __ ubfx(rscratch2, U_0, 26, 26);
12756 __ stp(rscratch1, rscratch2, Address(acc_start));
12757 __ ubfx(rscratch1, U_0, 52, 12);
12758 __ bfi(rscratch1, U_1, 12, 14);
12759 __ ubfx(rscratch2, U_1, 14, 26);
12760 __ stp(rscratch1, rscratch2, Address(acc_start, 2 * sizeof (jlong)));
12761 __ ubfx(rscratch1, U_1, 40, 24);
12762 __ bfi(rscratch1, U_2, 24, 3);
12763 __ str(rscratch1, Address(acc_start, 4 * sizeof (jlong)));
12764
12765 __ bind(DONE);
12766 __ pop(callee_saved, sp);
12767 __ leave();
12768 __ ret(lr);
12769
12770 // record the stub start and end
12771 store_archive_data(stub_id, start, __ pc());
12772
12773 return start;
12774 }
12775
12776 // exception handler for upcall stubs
12777 address generate_upcall_stub_exception_handler() {
12778 StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id;
12779 int entry_count = StubInfo::entry_count(stub_id);
12780 assert(entry_count == 1, "sanity check");
12781 address start = load_archive_data(stub_id);
12782 if (start != nullptr) {
12783 return start;
12784 }
12785 StubCodeMark mark(this, stub_id);
12786 start = __ pc();
12787
12788 // Native caller has no idea how to handle exceptions,
12789 // so we just crash here. Up to callee to catch exceptions.
12790 __ verify_oop(r0);
12791 __ movptr(rscratch1, CAST_FROM_FN_PTR(uint64_t, UpcallLinker::handle_uncaught_exception));
12792 __ blr(rscratch1);
12793 __ should_not_reach_here();
12794
12795 // record the stub start and end
12796 store_archive_data(stub_id, start, __ pc());
12797
12798 return start;
12799 }
12800
12801 // load Method* target of MethodHandle
12802 // j_rarg0 = jobject receiver
12803 // rmethod = result
12804 address generate_upcall_stub_load_target() {
12805 StubId stub_id = StubId::stubgen_upcall_stub_load_target_id;
12806 int entry_count = StubInfo::entry_count(stub_id);
12807 assert(entry_count == 1, "sanity check");
12808 address start = load_archive_data(stub_id);
12809 if (start != nullptr) {
12810 return start;
12811 }
12812 StubCodeMark mark(this, stub_id);
12813 start = __ pc();
12814
12815 __ resolve_global_jobject(j_rarg0, rscratch1, rscratch2);
12816 // Load target method from receiver
12817 __ load_heap_oop(rmethod, Address(j_rarg0, java_lang_invoke_MethodHandle::form_offset()), rscratch1, rscratch2);
12818 __ load_heap_oop(rmethod, Address(rmethod, java_lang_invoke_LambdaForm::vmentry_offset()), rscratch1, rscratch2);
12819 __ load_heap_oop(rmethod, Address(rmethod, java_lang_invoke_MemberName::method_offset()), rscratch1, rscratch2);
12820 __ access_load_at(T_ADDRESS, IN_HEAP, rmethod,
12821 Address(rmethod, java_lang_invoke_ResolvedMethodName::vmtarget_offset()),
12822 noreg, noreg);
12823 __ str(rmethod, Address(rthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
12824
12825 __ ret(lr);
12826
12827 // record the stub start and end
12828 store_archive_data(stub_id, start, __ pc());
12829
12830 return start;
12831 }
12832
12833 #undef __
12834 #define __ masm->
12835
12836 class MontgomeryMultiplyGenerator : public MacroAssembler {
12837
12838 Register Pa_base, Pb_base, Pn_base, Pm_base, inv, Rlen, Ra, Rb, Rm, Rn,
12839 Pa, Pb, Pn, Pm, Rhi_ab, Rlo_ab, Rhi_mn, Rlo_mn, t0, t1, t2, Ri, Rj;
12840
12841 RegSet _toSave;
12842 bool _squaring;
12843
12844 public:
12845 MontgomeryMultiplyGenerator (Assembler *as, bool squaring)
12846 : MacroAssembler(as->code()), _squaring(squaring) {
12847
12848 // Register allocation
12849
12850 RegSetIterator<Register> regs = (RegSet::range(r0, r26) - r18_tls).begin();
12851 Pa_base = *regs; // Argument registers
12852 if (squaring)
12853 Pb_base = Pa_base;
12854 else
12855 Pb_base = *++regs;
12856 Pn_base = *++regs;
12857 Rlen= *++regs;
12858 inv = *++regs;
12859 Pm_base = *++regs;
12860
12861 // Working registers:
12862 Ra = *++regs; // The current digit of a, b, n, and m.
12863 Rb = *++regs;
12864 Rm = *++regs;
12865 Rn = *++regs;
12866
12867 Pa = *++regs; // Pointers to the current/next digit of a, b, n, and m.
12868 Pb = *++regs;
12869 Pm = *++regs;
12870 Pn = *++regs;
12871
12872 t0 = *++regs; // Three registers which form a
12873 t1 = *++regs; // triple-precision accumuator.
12874 t2 = *++regs;
12875
12876 Ri = *++regs; // Inner and outer loop indexes.
12877 Rj = *++regs;
12878
12879 Rhi_ab = *++regs; // Product registers: low and high parts
12880 Rlo_ab = *++regs; // of a*b and m*n.
12881 Rhi_mn = *++regs;
12882 Rlo_mn = *++regs;
12883
12884 // r19 and up are callee-saved.
12885 _toSave = RegSet::range(r19, *regs) + Pm_base;
12886 }
12887
12888 private:
12889 void save_regs() {
12890 push(_toSave, sp);
12891 }
12892
12893 void restore_regs() {
12894 pop(_toSave, sp);
12895 }
12896
12897 template <typename T>
12898 void unroll_2(Register count, T block) {
12899 Label loop, end, odd;
12900 tbnz(count, 0, odd);
12901 cbz(count, end);
12902 align(16);
12903 bind(loop);
12904 (this->*block)();
12905 bind(odd);
12906 (this->*block)();
12907 subs(count, count, 2);
12908 br(Assembler::GT, loop);
12909 bind(end);
12910 }
12911
12912 template <typename T>
12913 void unroll_2(Register count, T block, Register d, Register s, Register tmp) {
12914 Label loop, end, odd;
12915 tbnz(count, 0, odd);
12916 cbz(count, end);
12917 align(16);
12918 bind(loop);
12919 (this->*block)(d, s, tmp);
12920 bind(odd);
12921 (this->*block)(d, s, tmp);
12922 subs(count, count, 2);
12923 br(Assembler::GT, loop);
12924 bind(end);
12925 }
12926
12927 void pre1(RegisterOrConstant i) {
12928 block_comment("pre1");
12929 // Pa = Pa_base;
12930 // Pb = Pb_base + i;
12931 // Pm = Pm_base;
12932 // Pn = Pn_base + i;
12933 // Ra = *Pa;
12934 // Rb = *Pb;
12935 // Rm = *Pm;
12936 // Rn = *Pn;
12937 ldr(Ra, Address(Pa_base));
12938 ldr(Rb, Address(Pb_base, i, Address::uxtw(LogBytesPerWord)));
12939 ldr(Rm, Address(Pm_base));
12940 ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
12941 lea(Pa, Address(Pa_base));
12942 lea(Pb, Address(Pb_base, i, Address::uxtw(LogBytesPerWord)));
12943 lea(Pm, Address(Pm_base));
12944 lea(Pn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
12945
12946 // Zero the m*n result.
12947 mov(Rhi_mn, zr);
12948 mov(Rlo_mn, zr);
12949 }
12950
12951 // The core multiply-accumulate step of a Montgomery
12952 // multiplication. The idea is to schedule operations as a
12953 // pipeline so that instructions with long latencies (loads and
12954 // multiplies) have time to complete before their results are
12955 // used. This most benefits in-order implementations of the
12956 // architecture but out-of-order ones also benefit.
12957 void step() {
12958 block_comment("step");
12959 // MACC(Ra, Rb, t0, t1, t2);
12960 // Ra = *++Pa;
12961 // Rb = *--Pb;
12962 umulh(Rhi_ab, Ra, Rb);
12963 mul(Rlo_ab, Ra, Rb);
12964 ldr(Ra, pre(Pa, wordSize));
12965 ldr(Rb, pre(Pb, -wordSize));
12966 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n from the
12967 // previous iteration.
12968 // MACC(Rm, Rn, t0, t1, t2);
12969 // Rm = *++Pm;
12970 // Rn = *--Pn;
12971 umulh(Rhi_mn, Rm, Rn);
12972 mul(Rlo_mn, Rm, Rn);
12973 ldr(Rm, pre(Pm, wordSize));
12974 ldr(Rn, pre(Pn, -wordSize));
12975 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
12976 }
12977
12978 void post1() {
12979 block_comment("post1");
12980
12981 // MACC(Ra, Rb, t0, t1, t2);
12982 // Ra = *++Pa;
12983 // Rb = *--Pb;
12984 umulh(Rhi_ab, Ra, Rb);
12985 mul(Rlo_ab, Ra, Rb);
12986 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n
12987 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
12988
12989 // *Pm = Rm = t0 * inv;
12990 mul(Rm, t0, inv);
12991 str(Rm, Address(Pm));
12992
12993 // MACC(Rm, Rn, t0, t1, t2);
12994 // t0 = t1; t1 = t2; t2 = 0;
12995 umulh(Rhi_mn, Rm, Rn);
12996
12997 #ifndef PRODUCT
12998 // assert(m[i] * n[0] + t0 == 0, "broken Montgomery multiply");
12999 {
13000 mul(Rlo_mn, Rm, Rn);
13001 add(Rlo_mn, t0, Rlo_mn);
13002 Label ok;
13003 cbz(Rlo_mn, ok); {
13004 stop("broken Montgomery multiply");
13005 } bind(ok);
13006 }
13007 #endif
13008 // We have very carefully set things up so that
13009 // m[i]*n[0] + t0 == 0 (mod b), so we don't have to calculate
13010 // the lower half of Rm * Rn because we know the result already:
13011 // it must be -t0. t0 + (-t0) must generate a carry iff
13012 // t0 != 0. So, rather than do a mul and an adds we just set
13013 // the carry flag iff t0 is nonzero.
13014 //
13015 // mul(Rlo_mn, Rm, Rn);
13016 // adds(zr, t0, Rlo_mn);
13017 subs(zr, t0, 1); // Set carry iff t0 is nonzero
13018 adcs(t0, t1, Rhi_mn);
13019 adc(t1, t2, zr);
13020 mov(t2, zr);
13021 }
13022
13023 void pre2(RegisterOrConstant i, RegisterOrConstant len) {
13024 block_comment("pre2");
13025 // Pa = Pa_base + i-len;
13026 // Pb = Pb_base + len;
13027 // Pm = Pm_base + i-len;
13028 // Pn = Pn_base + len;
13029
13030 if (i.is_register()) {
13031 sub(Rj, i.as_register(), len);
13032 } else {
13033 mov(Rj, i.as_constant());
13034 sub(Rj, Rj, len);
13035 }
13036 // Rj == i-len
13037
13038 lea(Pa, Address(Pa_base, Rj, Address::uxtw(LogBytesPerWord)));
13039 lea(Pb, Address(Pb_base, len, Address::uxtw(LogBytesPerWord)));
13040 lea(Pm, Address(Pm_base, Rj, Address::uxtw(LogBytesPerWord)));
13041 lea(Pn, Address(Pn_base, len, Address::uxtw(LogBytesPerWord)));
13042
13043 // Ra = *++Pa;
13044 // Rb = *--Pb;
13045 // Rm = *++Pm;
13046 // Rn = *--Pn;
13047 ldr(Ra, pre(Pa, wordSize));
13048 ldr(Rb, pre(Pb, -wordSize));
13049 ldr(Rm, pre(Pm, wordSize));
13050 ldr(Rn, pre(Pn, -wordSize));
13051
13052 mov(Rhi_mn, zr);
13053 mov(Rlo_mn, zr);
13054 }
13055
13056 void post2(RegisterOrConstant i, RegisterOrConstant len) {
13057 block_comment("post2");
13058 if (i.is_constant()) {
13059 mov(Rj, i.as_constant()-len.as_constant());
13060 } else {
13061 sub(Rj, i.as_register(), len);
13062 }
13063
13064 adds(t0, t0, Rlo_mn); // The pending m*n, low part
13065
13066 // As soon as we know the least significant digit of our result,
13067 // store it.
13068 // Pm_base[i-len] = t0;
13069 str(t0, Address(Pm_base, Rj, Address::uxtw(LogBytesPerWord)));
13070
13071 // t0 = t1; t1 = t2; t2 = 0;
13072 adcs(t0, t1, Rhi_mn); // The pending m*n, high part
13073 adc(t1, t2, zr);
13074 mov(t2, zr);
13075 }
13076
13077 // A carry in t0 after Montgomery multiplication means that we
13078 // should subtract multiples of n from our result in m. We'll
13079 // keep doing that until there is no carry.
13080 void normalize(RegisterOrConstant len) {
13081 block_comment("normalize");
13082 // while (t0)
13083 // t0 = sub(Pm_base, Pn_base, t0, len);
13084 Label loop, post, again;
13085 Register cnt = t1, i = t2; // Re-use registers; we're done with them now
13086 cbz(t0, post); {
13087 bind(again); {
13088 mov(i, zr);
13089 mov(cnt, len);
13090 ldr(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
13091 ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
13092 subs(zr, zr, zr); // set carry flag, i.e. no borrow
13093 align(16);
13094 bind(loop); {
13095 sbcs(Rm, Rm, Rn);
13096 str(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
13097 add(i, i, 1);
13098 ldr(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
13099 ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
13100 sub(cnt, cnt, 1);
13101 } cbnz(cnt, loop);
13102 sbc(t0, t0, zr);
13103 } cbnz(t0, again);
13104 } bind(post);
13105 }
13106
13107 // Move memory at s to d, reversing words.
13108 // Increments d to end of copied memory
13109 // Destroys tmp1, tmp2
13110 // Preserves len
13111 // Leaves s pointing to the address which was in d at start
13112 void reverse(Register d, Register s, Register len, Register tmp1, Register tmp2) {
13113 assert(tmp1->encoding() < r19->encoding(), "register corruption");
13114 assert(tmp2->encoding() < r19->encoding(), "register corruption");
13115
13116 lea(s, Address(s, len, Address::uxtw(LogBytesPerWord)));
13117 mov(tmp1, len);
13118 unroll_2(tmp1, &MontgomeryMultiplyGenerator::reverse1, d, s, tmp2);
13119 sub(s, d, len, ext::uxtw, LogBytesPerWord);
13120 }
13121 // where
13122 void reverse1(Register d, Register s, Register tmp) {
13123 ldr(tmp, pre(s, -wordSize));
13124 ror(tmp, tmp, 32);
13125 str(tmp, post(d, wordSize));
13126 }
13127
13128 void step_squaring() {
13129 // An extra ACC
13130 step();
13131 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
13132 }
13133
13134 void last_squaring(RegisterOrConstant i) {
13135 Label dont;
13136 // if ((i & 1) == 0) {
13137 tbnz(i.as_register(), 0, dont); {
13138 // MACC(Ra, Rb, t0, t1, t2);
13139 // Ra = *++Pa;
13140 // Rb = *--Pb;
13141 umulh(Rhi_ab, Ra, Rb);
13142 mul(Rlo_ab, Ra, Rb);
13143 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
13144 } bind(dont);
13145 }
13146
13147 void extra_step_squaring() {
13148 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n
13149
13150 // MACC(Rm, Rn, t0, t1, t2);
13151 // Rm = *++Pm;
13152 // Rn = *--Pn;
13153 umulh(Rhi_mn, Rm, Rn);
13154 mul(Rlo_mn, Rm, Rn);
13155 ldr(Rm, pre(Pm, wordSize));
13156 ldr(Rn, pre(Pn, -wordSize));
13157 }
13158
13159 void post1_squaring() {
13160 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n
13161
13162 // *Pm = Rm = t0 * inv;
13163 mul(Rm, t0, inv);
13164 str(Rm, Address(Pm));
13165
13166 // MACC(Rm, Rn, t0, t1, t2);
13167 // t0 = t1; t1 = t2; t2 = 0;
13168 umulh(Rhi_mn, Rm, Rn);
13169
13170 #ifndef PRODUCT
13171 // assert(m[i] * n[0] + t0 == 0, "broken Montgomery multiply");
13172 {
13173 mul(Rlo_mn, Rm, Rn);
13174 add(Rlo_mn, t0, Rlo_mn);
13175 Label ok;
13176 cbz(Rlo_mn, ok); {
13177 stop("broken Montgomery multiply");
13178 } bind(ok);
13179 }
13180 #endif
13181 // We have very carefully set things up so that
13182 // m[i]*n[0] + t0 == 0 (mod b), so we don't have to calculate
13183 // the lower half of Rm * Rn because we know the result already:
13184 // it must be -t0. t0 + (-t0) must generate a carry iff
13185 // t0 != 0. So, rather than do a mul and an adds we just set
13186 // the carry flag iff t0 is nonzero.
13187 //
13188 // mul(Rlo_mn, Rm, Rn);
13189 // adds(zr, t0, Rlo_mn);
13190 subs(zr, t0, 1); // Set carry iff t0 is nonzero
13191 adcs(t0, t1, Rhi_mn);
13192 adc(t1, t2, zr);
13193 mov(t2, zr);
13194 }
13195
13196 void acc(Register Rhi, Register Rlo,
13197 Register t0, Register t1, Register t2) {
13198 adds(t0, t0, Rlo);
13199 adcs(t1, t1, Rhi);
13200 adc(t2, t2, zr);
13201 }
13202
13203 public:
13204 /**
13205 * Fast Montgomery multiplication. The derivation of the
13206 * algorithm is in A Cryptographic Library for the Motorola
13207 * DSP56000, Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237.
13208 *
13209 * Arguments:
13210 *
13211 * Inputs for multiplication:
13212 * c_rarg0 - int array elements a
13213 * c_rarg1 - int array elements b
13214 * c_rarg2 - int array elements n (the modulus)
13215 * c_rarg3 - int length
13216 * c_rarg4 - int inv
13217 * c_rarg5 - int array elements m (the result)
13218 *
13219 * Inputs for squaring:
13220 * c_rarg0 - int array elements a
13221 * c_rarg1 - int array elements n (the modulus)
13222 * c_rarg2 - int length
13223 * c_rarg3 - int inv
13224 * c_rarg4 - int array elements m (the result)
13225 *
13226 */
13227 address generate_multiply() {
13228 Label argh, nothing;
13229
13230 align(CodeEntryAlignment);
13231 address entry = pc();
13232
13233 cbzw(Rlen, nothing);
13234
13235 enter();
13236
13237 // Make room.
13238 cmpw(Rlen, 512);
13239 br(Assembler::HI, argh);
13240 sub(Ra, sp, Rlen, ext::uxtw, exact_log2(4 * sizeof (jint)));
13241 andr(sp, Ra, -2 * wordSize);
13242
13243 lsrw(Rlen, Rlen, 1); // length in longwords = len/2
13244
13245 {
13246 // Copy input args, reversing as we go. We use Ra as a
13247 // temporary variable.
13248 reverse(Ra, Pa_base, Rlen, t0, t1);
13249 if (!_squaring)
13250 reverse(Ra, Pb_base, Rlen, t0, t1);
13251 reverse(Ra, Pn_base, Rlen, t0, t1);
13252 }
13253
13254 // Push all call-saved registers and also Pm_base which we'll need
13255 // at the end.
13256 save_regs();
13257
13258 #ifndef PRODUCT
13259 // assert(inv * n[0] == -1UL, "broken inverse in Montgomery multiply");
13260 {
13261 ldr(Rn, Address(Pn_base, 0));
13262 mul(Rlo_mn, Rn, inv);
13263 subs(zr, Rlo_mn, -1);
13264 Label ok;
13265 br(EQ, ok); {
13266 stop("broken inverse in Montgomery multiply");
13267 } bind(ok);
13268 }
13269 #endif
13270
13271 mov(Pm_base, Ra);
13272
13273 mov(t0, zr);
13274 mov(t1, zr);
13275 mov(t2, zr);
13276
13277 block_comment("for (int i = 0; i < len; i++) {");
13278 mov(Ri, zr); {
13279 Label loop, end;
13280 cmpw(Ri, Rlen);
13281 br(Assembler::GE, end);
13282
13283 bind(loop);
13284 pre1(Ri);
13285
13286 block_comment(" for (j = i; j; j--) {"); {
13287 movw(Rj, Ri);
13288 unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
13289 } block_comment(" } // j");
13290
13291 post1();
13292 addw(Ri, Ri, 1);
13293 cmpw(Ri, Rlen);
13294 br(Assembler::LT, loop);
13295 bind(end);
13296 block_comment("} // i");
13297 }
13298
13299 block_comment("for (int i = len; i < 2*len; i++) {");
13300 mov(Ri, Rlen); {
13301 Label loop, end;
13302 cmpw(Ri, Rlen, Assembler::LSL, 1);
13303 br(Assembler::GE, end);
13304
13305 bind(loop);
13306 pre2(Ri, Rlen);
13307
13308 block_comment(" for (j = len*2-i-1; j; j--) {"); {
13309 lslw(Rj, Rlen, 1);
13310 subw(Rj, Rj, Ri);
13311 subw(Rj, Rj, 1);
13312 unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
13313 } block_comment(" } // j");
13314
13315 post2(Ri, Rlen);
13316 addw(Ri, Ri, 1);
13317 cmpw(Ri, Rlen, Assembler::LSL, 1);
13318 br(Assembler::LT, loop);
13319 bind(end);
13320 }
13321 block_comment("} // i");
13322
13323 normalize(Rlen);
13324
13325 mov(Ra, Pm_base); // Save Pm_base in Ra
13326 restore_regs(); // Restore caller's Pm_base
13327
13328 // Copy our result into caller's Pm_base
13329 reverse(Pm_base, Ra, Rlen, t0, t1);
13330
13331 leave();
13332 bind(nothing);
13333 ret(lr);
13334
13335 // handler for error case
13336 bind(argh);
13337 stop("MontgomeryMultiply total_allocation must be <= 8192");
13338
13339 return entry;
13340 }
13341 // In C, approximately:
13342
13343 // void
13344 // montgomery_multiply(julong Pa_base[], julong Pb_base[],
13345 // julong Pn_base[], julong Pm_base[],
13346 // julong inv, int len) {
13347 // julong t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
13348 // julong *Pa, *Pb, *Pn, *Pm;
13349 // julong Ra, Rb, Rn, Rm;
13350
13351 // int i;
13352
13353 // assert(inv * Pn_base[0] == -1UL, "broken inverse in Montgomery multiply");
13354
13355 // for (i = 0; i < len; i++) {
13356 // int j;
13357
13358 // Pa = Pa_base;
13359 // Pb = Pb_base + i;
13360 // Pm = Pm_base;
13361 // Pn = Pn_base + i;
13362
13363 // Ra = *Pa;
13364 // Rb = *Pb;
13365 // Rm = *Pm;
13366 // Rn = *Pn;
13367
13368 // int iters = i;
13369 // for (j = 0; iters--; j++) {
13370 // assert(Ra == Pa_base[j] && Rb == Pb_base[i-j], "must be");
13371 // MACC(Ra, Rb, t0, t1, t2);
13372 // Ra = *++Pa;
13373 // Rb = *--Pb;
13374 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13375 // MACC(Rm, Rn, t0, t1, t2);
13376 // Rm = *++Pm;
13377 // Rn = *--Pn;
13378 // }
13379
13380 // assert(Ra == Pa_base[i] && Rb == Pb_base[0], "must be");
13381 // MACC(Ra, Rb, t0, t1, t2);
13382 // *Pm = Rm = t0 * inv;
13383 // assert(Rm == Pm_base[i] && Rn == Pn_base[0], "must be");
13384 // MACC(Rm, Rn, t0, t1, t2);
13385
13386 // assert(t0 == 0, "broken Montgomery multiply");
13387
13388 // t0 = t1; t1 = t2; t2 = 0;
13389 // }
13390
13391 // for (i = len; i < 2*len; i++) {
13392 // int j;
13393
13394 // Pa = Pa_base + i-len;
13395 // Pb = Pb_base + len;
13396 // Pm = Pm_base + i-len;
13397 // Pn = Pn_base + len;
13398
13399 // Ra = *++Pa;
13400 // Rb = *--Pb;
13401 // Rm = *++Pm;
13402 // Rn = *--Pn;
13403
13404 // int iters = len*2-i-1;
13405 // for (j = i-len+1; iters--; j++) {
13406 // assert(Ra == Pa_base[j] && Rb == Pb_base[i-j], "must be");
13407 // MACC(Ra, Rb, t0, t1, t2);
13408 // Ra = *++Pa;
13409 // Rb = *--Pb;
13410 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13411 // MACC(Rm, Rn, t0, t1, t2);
13412 // Rm = *++Pm;
13413 // Rn = *--Pn;
13414 // }
13415
13416 // Pm_base[i-len] = t0;
13417 // t0 = t1; t1 = t2; t2 = 0;
13418 // }
13419
13420 // while (t0)
13421 // t0 = sub(Pm_base, Pn_base, t0, len);
13422 // }
13423
13424 /**
13425 * Fast Montgomery squaring. This uses asymptotically 25% fewer
13426 * multiplies than Montgomery multiplication so it should be up to
13427 * 25% faster. However, its loop control is more complex and it
13428 * may actually run slower on some machines.
13429 *
13430 * Arguments:
13431 *
13432 * Inputs:
13433 * c_rarg0 - int array elements a
13434 * c_rarg1 - int array elements n (the modulus)
13435 * c_rarg2 - int length
13436 * c_rarg3 - int inv
13437 * c_rarg4 - int array elements m (the result)
13438 *
13439 */
13440 address generate_square() {
13441 Label argh;
13442
13443 align(CodeEntryAlignment);
13444 address entry = pc();
13445
13446 enter();
13447
13448 // Make room.
13449 cmpw(Rlen, 512);
13450 br(Assembler::HI, argh);
13451 sub(Ra, sp, Rlen, ext::uxtw, exact_log2(4 * sizeof (jint)));
13452 andr(sp, Ra, -2 * wordSize);
13453
13454 lsrw(Rlen, Rlen, 1); // length in longwords = len/2
13455
13456 {
13457 // Copy input args, reversing as we go. We use Ra as a
13458 // temporary variable.
13459 reverse(Ra, Pa_base, Rlen, t0, t1);
13460 reverse(Ra, Pn_base, Rlen, t0, t1);
13461 }
13462
13463 // Push all call-saved registers and also Pm_base which we'll need
13464 // at the end.
13465 save_regs();
13466
13467 mov(Pm_base, Ra);
13468
13469 mov(t0, zr);
13470 mov(t1, zr);
13471 mov(t2, zr);
13472
13473 block_comment("for (int i = 0; i < len; i++) {");
13474 mov(Ri, zr); {
13475 Label loop, end;
13476 bind(loop);
13477 cmp(Ri, Rlen);
13478 br(Assembler::GE, end);
13479
13480 pre1(Ri);
13481
13482 block_comment("for (j = (i+1)/2; j; j--) {"); {
13483 add(Rj, Ri, 1);
13484 lsr(Rj, Rj, 1);
13485 unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
13486 } block_comment(" } // j");
13487
13488 last_squaring(Ri);
13489
13490 block_comment(" for (j = i/2; j; j--) {"); {
13491 lsr(Rj, Ri, 1);
13492 unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
13493 } block_comment(" } // j");
13494
13495 post1_squaring();
13496 add(Ri, Ri, 1);
13497 cmp(Ri, Rlen);
13498 br(Assembler::LT, loop);
13499
13500 bind(end);
13501 block_comment("} // i");
13502 }
13503
13504 block_comment("for (int i = len; i < 2*len; i++) {");
13505 mov(Ri, Rlen); {
13506 Label loop, end;
13507 bind(loop);
13508 cmp(Ri, Rlen, Assembler::LSL, 1);
13509 br(Assembler::GE, end);
13510
13511 pre2(Ri, Rlen);
13512
13513 block_comment(" for (j = (2*len-i-1)/2; j; j--) {"); {
13514 lsl(Rj, Rlen, 1);
13515 sub(Rj, Rj, Ri);
13516 sub(Rj, Rj, 1);
13517 lsr(Rj, Rj, 1);
13518 unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
13519 } block_comment(" } // j");
13520
13521 last_squaring(Ri);
13522
13523 block_comment(" for (j = (2*len-i)/2; j; j--) {"); {
13524 lsl(Rj, Rlen, 1);
13525 sub(Rj, Rj, Ri);
13526 lsr(Rj, Rj, 1);
13527 unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
13528 } block_comment(" } // j");
13529
13530 post2(Ri, Rlen);
13531 add(Ri, Ri, 1);
13532 cmp(Ri, Rlen, Assembler::LSL, 1);
13533
13534 br(Assembler::LT, loop);
13535 bind(end);
13536 block_comment("} // i");
13537 }
13538
13539 normalize(Rlen);
13540
13541 mov(Ra, Pm_base); // Save Pm_base in Ra
13542 restore_regs(); // Restore caller's Pm_base
13543
13544 // Copy our result into caller's Pm_base
13545 reverse(Pm_base, Ra, Rlen, t0, t1);
13546
13547 leave();
13548 ret(lr);
13549
13550 // handler for error case
13551 bind(argh);
13552 stop("MontgomeryMultiply total_allocation must be <= 8192");
13553
13554 return entry;
13555 }
13556 // In C, approximately:
13557
13558 // void
13559 // montgomery_square(julong Pa_base[], julong Pn_base[],
13560 // julong Pm_base[], julong inv, int len) {
13561 // julong t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
13562 // julong *Pa, *Pb, *Pn, *Pm;
13563 // julong Ra, Rb, Rn, Rm;
13564
13565 // int i;
13566
13567 // assert(inv * Pn_base[0] == -1UL, "broken inverse in Montgomery multiply");
13568
13569 // for (i = 0; i < len; i++) {
13570 // int j;
13571
13572 // Pa = Pa_base;
13573 // Pb = Pa_base + i;
13574 // Pm = Pm_base;
13575 // Pn = Pn_base + i;
13576
13577 // Ra = *Pa;
13578 // Rb = *Pb;
13579 // Rm = *Pm;
13580 // Rn = *Pn;
13581
13582 // int iters = (i+1)/2;
13583 // for (j = 0; iters--; j++) {
13584 // assert(Ra == Pa_base[j] && Rb == Pa_base[i-j], "must be");
13585 // MACC2(Ra, Rb, t0, t1, t2);
13586 // Ra = *++Pa;
13587 // Rb = *--Pb;
13588 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13589 // MACC(Rm, Rn, t0, t1, t2);
13590 // Rm = *++Pm;
13591 // Rn = *--Pn;
13592 // }
13593 // if ((i & 1) == 0) {
13594 // assert(Ra == Pa_base[j], "must be");
13595 // MACC(Ra, Ra, t0, t1, t2);
13596 // }
13597 // iters = i/2;
13598 // assert(iters == i-j, "must be");
13599 // for (; iters--; j++) {
13600 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13601 // MACC(Rm, Rn, t0, t1, t2);
13602 // Rm = *++Pm;
13603 // Rn = *--Pn;
13604 // }
13605
13606 // *Pm = Rm = t0 * inv;
13607 // assert(Rm == Pm_base[i] && Rn == Pn_base[0], "must be");
13608 // MACC(Rm, Rn, t0, t1, t2);
13609
13610 // assert(t0 == 0, "broken Montgomery multiply");
13611
13612 // t0 = t1; t1 = t2; t2 = 0;
13613 // }
13614
13615 // for (i = len; i < 2*len; i++) {
13616 // int start = i-len+1;
13617 // int end = start + (len - start)/2;
13618 // int j;
13619
13620 // Pa = Pa_base + i-len;
13621 // Pb = Pa_base + len;
13622 // Pm = Pm_base + i-len;
13623 // Pn = Pn_base + len;
13624
13625 // Ra = *++Pa;
13626 // Rb = *--Pb;
13627 // Rm = *++Pm;
13628 // Rn = *--Pn;
13629
13630 // int iters = (2*len-i-1)/2;
13631 // assert(iters == end-start, "must be");
13632 // for (j = start; iters--; j++) {
13633 // assert(Ra == Pa_base[j] && Rb == Pa_base[i-j], "must be");
13634 // MACC2(Ra, Rb, t0, t1, t2);
13635 // Ra = *++Pa;
13636 // Rb = *--Pb;
13637 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13638 // MACC(Rm, Rn, t0, t1, t2);
13639 // Rm = *++Pm;
13640 // Rn = *--Pn;
13641 // }
13642 // if ((i & 1) == 0) {
13643 // assert(Ra == Pa_base[j], "must be");
13644 // MACC(Ra, Ra, t0, t1, t2);
13645 // }
13646 // iters = (2*len-i)/2;
13647 // assert(iters == len-j, "must be");
13648 // for (; iters--; j++) {
13649 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13650 // MACC(Rm, Rn, t0, t1, t2);
13651 // Rm = *++Pm;
13652 // Rn = *--Pn;
13653 // }
13654 // Pm_base[i-len] = t0;
13655 // t0 = t1; t1 = t2; t2 = 0;
13656 // }
13657
13658 // while (t0)
13659 // t0 = sub(Pm_base, Pn_base, t0, len);
13660 // }
13661 };
13662
13663 // Initialization
13664 void generate_preuniverse_stubs() {
13665 // preuniverse stubs are not needed for aarch64
13666 }
13667
13668 void generate_initial_stubs() {
13669 // Generate initial stubs and initializes the entry points
13670
13671 // entry points that exist in all platforms Note: This is code
13672 // that could be shared among different platforms - however the
13673 // benefit seems to be smaller than the disadvantage of having a
13674 // much more complicated generator structure. See also comment in
13675 // stubRoutines.hpp.
13676
13677 StubRoutines::_forward_exception_entry = generate_forward_exception();
13678
13679 StubRoutines::_call_stub_entry =
13680 generate_call_stub(StubRoutines::_call_stub_return_address);
13681
13682 // is referenced by megamorphic call
13683 StubRoutines::_catch_exception_entry = generate_catch_exception();
13684
13685 // Initialize table for copy memory (arraycopy) check.
13686 if (UnsafeMemoryAccess::_table == nullptr) {
13687 UnsafeMemoryAccess::create_table(8 + 4); // 8 for copyMemory; 4 for setMemory
13688 }
13689
13690 if (UseCRC32Intrinsics) {
13691 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
13692 }
13693
13694 if (UseCRC32CIntrinsics) {
13695 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
13696 }
13697
13698 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
13699 StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
13700 }
13701
13702 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
13703 StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
13704 }
13705
13706 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
13707 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
13708 StubRoutines::_hf2f = generate_float16ToFloat();
13709 StubRoutines::_f2hf = generate_floatToFloat16();
13710 }
13711 }
13712
13713 void generate_continuation_stubs() {
13714 // Continuation stubs:
13715 StubRoutines::_cont_thaw = generate_cont_thaw();
13716 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
13717 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
13718 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
13719 }
13720
13721 void generate_final_stubs() {
13722 // support for verify_oop (must happen after universe_init)
13723 if (VerifyOops) {
13724 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
13725 }
13726
13727 // arraycopy stubs used by compilers
13728 generate_arraycopy_stubs();
13729
13730 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
13731
13732 StubRoutines::aarch64::_spin_wait = generate_spin_wait();
13733
13734 StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler();
13735 StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target();
13736
13737 #if defined (LINUX) && !defined (__ARM_FEATURE_ATOMICS)
13738
13739 generate_atomic_entry_points();
13740
13741 #endif // LINUX
13742
13743 #ifdef COMPILER2
13744 if (UseSecondarySupersTable) {
13745 StubRoutines::_lookup_secondary_supers_table_slow_path_stub = generate_lookup_secondary_supers_table_slow_path_stub();
13746 if (! InlineSecondarySupersTest) {
13747 generate_lookup_secondary_supers_table_stub();
13748 }
13749 }
13750 #endif
13751
13752 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_setMemory)) {
13753 StubRoutines::_unsafe_setmemory = generate_unsafe_setmemory();
13754 }
13755
13756 StubRoutines::aarch64::set_completed(); // Inidicate that arraycopy and zero_blocks stubs are generated
13757 }
13758
13759 void generate_compiler_stubs() {
13760 #ifdef COMPILER2
13761
13762 if (UseSVE == 0) {
13763 generate_iota_indices(StubId::stubgen_vector_iota_indices_id);
13764 }
13765
13766 // array equals stub for large arrays.
13767 if (!UseSimpleArrayEquals) {
13768 StubRoutines::aarch64::_large_array_equals = generate_large_array_equals();
13769 }
13770
13771 // arrays_hascode stub for large arrays.
13772 StubRoutines::aarch64::_large_arrays_hashcode_boolean = generate_large_arrays_hashcode(T_BOOLEAN);
13773 StubRoutines::aarch64::_large_arrays_hashcode_byte = generate_large_arrays_hashcode(T_BYTE);
13774 StubRoutines::aarch64::_large_arrays_hashcode_char = generate_large_arrays_hashcode(T_CHAR);
13775 StubRoutines::aarch64::_large_arrays_hashcode_int = generate_large_arrays_hashcode(T_INT);
13776 StubRoutines::aarch64::_large_arrays_hashcode_short = generate_large_arrays_hashcode(T_SHORT);
13777
13778 // byte_array_inflate stub for large arrays.
13779 StubRoutines::aarch64::_large_byte_array_inflate = generate_large_byte_array_inflate();
13780
13781 // countPositives stub for large arrays.
13782 StubRoutines::aarch64::_count_positives = generate_count_positives(StubRoutines::aarch64::_count_positives_long);
13783
13784 generate_compare_long_strings();
13785
13786 generate_string_indexof_stubs();
13787
13788 if (UseMultiplyToLenIntrinsic) {
13789 StubRoutines::_multiplyToLen = generate_multiplyToLen();
13790 }
13791
13792 if (UseSquareToLenIntrinsic) {
13793 StubRoutines::_squareToLen = generate_squareToLen();
13794 }
13795
13796 if (UseMulAddIntrinsic) {
13797 StubRoutines::_mulAdd = generate_mulAdd();
13798 }
13799
13800 if (UseSIMDForBigIntegerShiftIntrinsics) {
13801 StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift();
13802 StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift();
13803 }
13804
13805 if (UseMontgomeryMultiplyIntrinsic) {
13806 StubId stub_id = StubId::stubgen_montgomeryMultiply_id;
13807 address start = load_archive_data(stub_id);
13808 if (start == nullptr) {
13809 // we have to generate it
13810 StubCodeMark mark(this, stub_id);
13811 MontgomeryMultiplyGenerator g(_masm, /*squaring*/false);
13812 start = g.generate_multiply();
13813 // record the stub start and end
13814 store_archive_data(stub_id, start, _masm->pc());
13815 }
13816 StubRoutines::_montgomeryMultiply = start;
13817 }
13818
13819 if (UseMontgomerySquareIntrinsic) {
13820 StubId stub_id = StubId::stubgen_montgomerySquare_id;
13821 address start = load_archive_data(stub_id);
13822 if (start == nullptr) {
13823 // we have to generate it
13824 StubCodeMark mark(this, stub_id);
13825 MontgomeryMultiplyGenerator g(_masm, /*squaring*/true);
13826 // We use generate_multiply() rather than generate_square()
13827 // because it's faster for the sizes of modulus we care about.
13828 start = g.generate_multiply();
13829 // record the stub start and end
13830 store_archive_data(stub_id, start, _masm->pc());
13831 }
13832 StubRoutines::_montgomerySquare = start;
13833 }
13834
13835 if (UseChaCha20Intrinsics) {
13836 StubRoutines::_chacha20Block = generate_chacha20Block_blockpar();
13837 }
13838
13839 if (UseIntPolyIntrinsics) {
13840 StubRoutines::_intpoly_montgomeryMult_P256 = generate_intpoly_montgomeryMult_P256();
13841 StubRoutines::_intpoly_assign = generate_intpoly_assign();
13842 }
13843
13844 if (UseKyberIntrinsics) {
13845 StubRoutines::_kyberNtt = generate_kyberNtt();
13846 StubRoutines::_kyberInverseNtt = generate_kyberInverseNtt();
13847 StubRoutines::_kyberNttMult = generate_kyberNttMult();
13848 StubRoutines::_kyberAddPoly_2 = generate_kyberAddPoly_2();
13849 StubRoutines::_kyberAddPoly_3 = generate_kyberAddPoly_3();
13850 StubRoutines::_kyber12To16 = generate_kyber12To16();
13851 StubRoutines::_kyberBarrettReduce = generate_kyberBarrettReduce();
13852 }
13853
13854 if (UseDilithiumIntrinsics) {
13855 StubRoutines::_dilithiumAlmostNtt = generate_dilithiumAlmostNtt();
13856 StubRoutines::_dilithiumAlmostInverseNtt = generate_dilithiumAlmostInverseNtt();
13857 StubRoutines::_dilithiumNttMult = generate_dilithiumNttMult();
13858 StubRoutines::_dilithiumMontMulByConstant = generate_dilithiumMontMulByConstant();
13859 StubRoutines::_dilithiumDecomposePoly = generate_dilithiumDecomposePoly();
13860 }
13861
13862 if (UseBASE64Intrinsics) {
13863 StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock();
13864 StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock();
13865 }
13866
13867 // data cache line writeback
13868 StubRoutines::_data_cache_writeback = generate_data_cache_writeback();
13869 StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
13870
13871 if (UseAESIntrinsics) {
13872 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
13873 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
13874 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
13875 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt();
13876 StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt();
13877 }
13878 if (UseGHASHIntrinsics) {
13879 // StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
13880 StubRoutines::aarch64::_ghash_processBlocks_small = generate_ghash_processBlocks_small();
13881 StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(StubRoutines::aarch64::_ghash_processBlocks_small);
13882 }
13883 if (UseAESIntrinsics && UseGHASHIntrinsics) {
13884 StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt();
13885 }
13886
13887 if (UseMD5Intrinsics) {
13888 StubRoutines::_md5_implCompress = generate_md5_implCompress(StubId::stubgen_md5_implCompress_id);
13889 StubRoutines::_md5_implCompressMB = generate_md5_implCompress(StubId::stubgen_md5_implCompressMB_id);
13890 }
13891 if (UseSHA1Intrinsics) {
13892 StubRoutines::_sha1_implCompress = generate_sha1_implCompress(StubId::stubgen_sha1_implCompress_id);
13893 StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(StubId::stubgen_sha1_implCompressMB_id);
13894 }
13895 if (UseSHA256Intrinsics) {
13896 StubRoutines::_sha256_implCompress = generate_sha256_implCompress(StubId::stubgen_sha256_implCompress_id);
13897 StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(StubId::stubgen_sha256_implCompressMB_id);
13898 }
13899 if (UseSHA512Intrinsics) {
13900 StubRoutines::_sha512_implCompress = generate_sha512_implCompress(StubId::stubgen_sha512_implCompress_id);
13901 StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(StubId::stubgen_sha512_implCompressMB_id);
13902 }
13903 if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic) {
13904 StubRoutines::_double_keccak = generate_double_keccak();
13905 StubRoutines::_sha3_implCompress = generate_sha3_implCompress(StubId::stubgen_sha3_implCompress_id);
13906 StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress(StubId::stubgen_sha3_implCompressMB_id);
13907 } else if (UseSHA3Intrinsics) {
13908 StubRoutines::_sha3_implCompress = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompress_id);
13909 StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompressMB_id);
13910 }
13911
13912 if (UsePoly1305Intrinsics) {
13913 StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks();
13914 }
13915
13916 // The difference between AArch64 vs. x86_64 intrinsics implementation
13917 // include the lack of square() intrinsics; usage caused a 3.3% performance
13918 // degradation due to the efficiencies of the symmetric squaring shape in
13919 // Java vs. the inefficiencies of the leaf calls and the additional cycles
13920 // required for 64 bit multiplication in AArch64.
13921 if (UseIntPoly25519Intrinsics) {
13922 StubRoutines::_intpoly_mult_25519 = generate_intpoly_mult_25519();
13923 }
13924
13925 // generate Adler32 intrinsics code
13926 if (UseAdler32Intrinsics) {
13927 StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
13928 }
13929
13930 #endif // COMPILER2
13931 }
13932
13933 public:
13934 StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
13935 switch(blob_id) {
13936 case BlobId::stubgen_preuniverse_id:
13937 generate_preuniverse_stubs();
13938 break;
13939 case BlobId::stubgen_initial_id:
13940 generate_initial_stubs();
13941 break;
13942 case BlobId::stubgen_continuation_id:
13943 generate_continuation_stubs();
13944 break;
13945 case BlobId::stubgen_compiler_id:
13946 generate_compiler_stubs();
13947 break;
13948 case BlobId::stubgen_final_id:
13949 generate_final_stubs();
13950 break;
13951 default:
13952 fatal("unexpected blob id: %s", StubInfo::name(blob_id));
13953 break;
13954 };
13955 }
13956
13957 #if INCLUDE_CDS
13958 static void init_AOTAddressTable(GrowableArray<address>& external_addresses) {
13959 // external data defined in this file
13960 #define ADD(addr) external_addresses.append((address)(addr));
13961 ADD(_sha256_round_consts);
13962 ADD(_sha512_round_consts);
13963 ADD(_sha3_round_consts);
13964 ADD(_double_keccak_round_consts);
13965 ADD(_modulus_P256);
13966 ADD(_encodeBlock_toBase64);
13967 ADD(_encodeBlock_toBase64URL);
13968 ADD(_decodeBlock_fromBase64ForNoSIMD);
13969 ADD(_decodeBlock_fromBase64URLForNoSIMD);
13970 ADD(_decodeBlock_fromBase64ForSIMD);
13971 ADD(_decodeBlock_fromBase64URLForSIMD);
13972 #undef ADD
13973 }
13974 #endif // INCLUDE_CDS
13975 }; // end class declaration
13976
13977 void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
13978 StubGenerator g(code, blob_id, stub_data);
13979 }
13980
13981 #if INCLUDE_CDS
13982 void StubGenerator_init_AOTAddressTable(GrowableArray<address>& addresses) {
13983 StubGenerator::init_AOTAddressTable(addresses);
13984 }
13985 #endif // INCLUDE_CDS
13986
13987 #if defined (LINUX)
13988
13989 // Define pointers to atomic stubs and initialize them to point to the
13990 // code in atomic_aarch64.S.
13991
13992 #define DEFAULT_ATOMIC_OP(OPNAME, SIZE, RELAXED) \
13993 extern "C" uint64_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl \
13994 (volatile void *ptr, uint64_t arg1, uint64_t arg2); \
13995 aarch64_atomic_stub_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _impl \
13996 = aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl;
13997
13998 DEFAULT_ATOMIC_OP(fetch_add, 4, )
13999 DEFAULT_ATOMIC_OP(fetch_add, 8, )
14000 DEFAULT_ATOMIC_OP(fetch_add, 4, _relaxed)
14001 DEFAULT_ATOMIC_OP(fetch_add, 8, _relaxed)
14002 DEFAULT_ATOMIC_OP(xchg, 4, )
14003 DEFAULT_ATOMIC_OP(xchg, 8, )
14004 DEFAULT_ATOMIC_OP(cmpxchg, 1, )
14005 DEFAULT_ATOMIC_OP(cmpxchg, 4, )
14006 DEFAULT_ATOMIC_OP(cmpxchg, 8, )
14007 DEFAULT_ATOMIC_OP(cmpxchg, 1, _relaxed)
14008 DEFAULT_ATOMIC_OP(cmpxchg, 4, _relaxed)
14009 DEFAULT_ATOMIC_OP(cmpxchg, 8, _relaxed)
14010 DEFAULT_ATOMIC_OP(cmpxchg, 4, _release)
14011 DEFAULT_ATOMIC_OP(cmpxchg, 8, _release)
14012 DEFAULT_ATOMIC_OP(cmpxchg, 4, _seq_cst)
14013 DEFAULT_ATOMIC_OP(cmpxchg, 8, _seq_cst)
14014
14015 #undef DEFAULT_ATOMIC_OP
14016
14017 #endif // LINUX