1 /*
2 * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2025, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/register.hpp"
29 #include "atomic_aarch64.hpp"
30 #include "compiler/oopMap.hpp"
31 #include "gc/shared/barrierSet.hpp"
32 #include "gc/shared/barrierSetAssembler.hpp"
33 #include "gc/shared/gc_globals.hpp"
34 #include "gc/shared/tlab_globals.hpp"
35 #include "interpreter/interpreter.hpp"
36 #include "memory/universe.hpp"
37 #include "nativeInst_aarch64.hpp"
38 #include "oops/instanceOop.hpp"
39 #include "oops/method.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "prims/methodHandles.hpp"
43 #include "prims/upcallLinker.hpp"
44 #include "runtime/arguments.hpp"
45 #include "runtime/atomicAccess.hpp"
46 #include "runtime/continuation.hpp"
47 #include "runtime/continuationEntry.inline.hpp"
48 #include "runtime/frame.inline.hpp"
49 #include "runtime/handles.inline.hpp"
50 #include "runtime/javaThread.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubCodeGenerator.hpp"
53 #include "runtime/stubRoutines.hpp"
54 #include "utilities/align.hpp"
55 #include "utilities/checkedCast.hpp"
56 #include "utilities/debug.hpp"
57 #include "utilities/globalDefinitions.hpp"
58 #include "utilities/intpow.hpp"
59 #include "utilities/powerOfTwo.hpp"
60 #ifdef COMPILER2
61 #include "opto/runtime.hpp"
62 #endif
63 #if INCLUDE_ZGC
64 #include "gc/z/zThreadLocalData.hpp"
65 #endif
66
67 // Declaration and definition of StubGenerator (no .hpp file).
68 // For a more detailed description of the stub routine structure
69 // see the comment in stubRoutines.hpp
70
71 #undef __
72 #define __ _masm->
73
74 #ifdef PRODUCT
75 #define BLOCK_COMMENT(str) /* nothing */
76 #else
77 #define BLOCK_COMMENT(str) __ block_comment(str)
78 #endif
79
80 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
81
82 // Constant data definitions
83
84 static const uint32_t _sha256_round_consts[64] = {
85 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
86 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
87 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
88 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
89 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
90 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
91 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
92 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
93 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
94 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
95 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
96 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
97 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
98 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
99 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
100 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
101 };
102
103 static const uint64_t _sha512_round_consts[80] = {
104 0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL,
105 0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L,
106 0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L,
107 0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L,
108 0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L,
109 0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L,
110 0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L,
111 0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L,
112 0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL,
113 0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L,
114 0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL,
115 0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL,
116 0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L,
117 0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L,
118 0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L,
119 0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L,
120 0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L,
121 0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL,
122 0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL,
123 0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL,
124 0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L,
125 0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L,
126 0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL,
127 0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL,
128 0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL,
129 0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL,
130 0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L
131 };
132
133 static const uint64_t _sha3_round_consts[24] = {
134 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
135 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
136 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
137 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
138 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
139 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
140 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
141 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
142 };
143
144 static const uint64_t _double_keccak_round_consts[24] = {
145 0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
146 0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
147 0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
148 0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
149 0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
150 0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
151 0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
152 0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
153 };
154
155 //Omit 3rd limb of modulus since it is 0
156 static const int64_t _modulus_P256[5] = {
157 0x000fffffffffffffL, 0x00000fffffffffffL,
158 0x0000001000000000L, 0x0000ffffffff0000L
159 };
160
161 static const char _encodeBlock_toBase64[64] = {
162 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
163 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
164 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
165 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
166 '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
167 };
168
169 static const char _encodeBlock_toBase64URL[64] = {
170 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
171 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
172 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
173 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
174 '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'
175 };
176
177 // Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64,
178 // except the trailing character '=' is also treated illegal value in this intrinsic. That
179 // is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here.
180 static const uint8_t _decodeBlock_fromBase64ForNoSIMD[256] = {
181 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
182 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
183 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
184 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
185 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
186 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u,
187 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
188 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
189 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
190 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
191 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
192 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
193 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
194 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
195 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
196 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
197 };
198
199 static const uint8_t _decodeBlock_fromBase64URLForNoSIMD[256] = {
200 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
201 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
202 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
203 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
204 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
205 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u,
206 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
207 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
208 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
209 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
210 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
211 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
212 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
213 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
214 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
215 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
216 };
217
218 // A legal value of base64 code is in range [0, 127]. We need two lookups
219 // with tbl/tbx and combine them to get the decode data. The 1st table vector
220 // lookup use tbl, out of range indices are set to 0 in destination. The 2nd
221 // table vector lookup use tbx, out of range indices are unchanged in
222 // destination. Input [64..126] is mapped to index [65, 127] in second lookup.
223 // The value of index 64 is set to 0, so that we know that we already get the
224 // decoded data with the 1st lookup.
225 static const uint8_t _decodeBlock_fromBase64ForSIMD[128] = {
226 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
227 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
228 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
229 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
230 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
231 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
232 255u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
233 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
234 };
235
236 static const uint8_t _decodeBlock_fromBase64URLForSIMD[128] = {
237 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
238 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
239 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
240 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
241 0u, 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u,
242 14u, 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u,
243 63u, 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u,
244 40u, 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u,
245 };
246
247
248 // Stub Code definitions
249
250 class StubGenerator: public StubCodeGenerator {
251 private:
252
253 #ifdef PRODUCT
254 #define inc_counter_np(counter) ((void)0)
255 #else
256 void inc_counter_np_(uint& counter) {
257 __ incrementw(ExternalAddress((address)&counter));
258 }
259 #define inc_counter_np(counter) \
260 BLOCK_COMMENT("inc_counter " #counter); \
261 inc_counter_np_(counter);
262 #endif
263
264 // Call stubs are used to call Java from C
265 //
266 // Arguments:
267 // c_rarg0: call wrapper address address
268 // c_rarg1: result address
269 // c_rarg2: result type BasicType
270 // c_rarg3: method Method*
271 // c_rarg4: (interpreter) entry point address
272 // c_rarg5: parameters intptr_t*
273 // c_rarg6: parameter size (in words) int
274 // c_rarg7: thread Thread*
275 //
276 // There is no return from the stub itself as any Java result
277 // is written to result
278 //
279 // we save r30 (lr) as the return PC at the base of the frame and
280 // link r29 (fp) below it as the frame pointer installing sp (r31)
281 // into fp.
282 //
283 // we save r0-r7, which accounts for all the c arguments.
284 //
285 // TODO: strictly do we need to save them all? they are treated as
286 // volatile by C so could we omit saving the ones we are going to
287 // place in global registers (thread? method?) or those we only use
288 // during setup of the Java call?
289 //
290 // we don't need to save r8 which C uses as an indirect result location
291 // return register.
292 //
293 // we don't need to save r9-r15 which both C and Java treat as
294 // volatile
295 //
296 // we don't need to save r16-18 because Java does not use them
297 //
298 // we save r19-r28 which Java uses as scratch registers and C
299 // expects to be callee-save
300 //
301 // we save the bottom 64 bits of each value stored in v8-v15; it is
302 // the responsibility of the caller to preserve larger values.
303 //
304 // so the stub frame looks like this when we enter Java code
305 //
306 // [ return_from_Java ] <--- sp
307 // [ argument word n ]
308 // ...
309 // -29 [ argument word 1 ]
310 // -28 [ saved Floating-point Control Register ]
311 // -26 [ saved v15 ] <--- sp_after_call
312 // -25 [ saved v14 ]
313 // -24 [ saved v13 ]
314 // -23 [ saved v12 ]
315 // -22 [ saved v11 ]
316 // -21 [ saved v10 ]
317 // -20 [ saved v9 ]
318 // -19 [ saved v8 ]
319 // -18 [ saved r28 ]
320 // -17 [ saved r27 ]
321 // -16 [ saved r26 ]
322 // -15 [ saved r25 ]
323 // -14 [ saved r24 ]
324 // -13 [ saved r23 ]
325 // -12 [ saved r22 ]
326 // -11 [ saved r21 ]
327 // -10 [ saved r20 ]
328 // -9 [ saved r19 ]
329 // -8 [ call wrapper (r0) ]
330 // -7 [ result (r1) ]
331 // -6 [ result type (r2) ]
332 // -5 [ method (r3) ]
333 // -4 [ entry point (r4) ]
334 // -3 [ parameters (r5) ]
335 // -2 [ parameter size (r6) ]
336 // -1 [ thread (r7) ]
337 // 0 [ saved fp (r29) ] <--- fp == saved sp (r31)
338 // 1 [ saved lr (r30) ]
339
340 // Call stub stack layout word offsets from fp
341 enum call_stub_layout {
342 sp_after_call_off = -28,
343
344 fpcr_off = sp_after_call_off,
345 d15_off = -26,
346 d13_off = -24,
347 d11_off = -22,
348 d9_off = -20,
349
350 r28_off = -18,
351 r26_off = -16,
352 r24_off = -14,
353 r22_off = -12,
354 r20_off = -10,
355 call_wrapper_off = -8,
356 result_off = -7,
357 result_type_off = -6,
358 method_off = -5,
359 entry_point_off = -4,
360 parameter_size_off = -2,
361 thread_off = -1,
362 fp_f = 0,
363 retaddr_off = 1,
364 };
365
366 address generate_call_stub(address& return_address) {
367 assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 &&
368 (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
369 "adjust this code");
370
371 StubId stub_id = StubId::stubgen_call_stub_id;
372 GrowableArray<address> entries;
373 int entry_count = StubInfo::entry_count(stub_id);
374 assert(entry_count == 2, "sanity check");
375 address start = load_archive_data(stub_id, &entries);
376 if (start != nullptr) {
377 assert(entries.length() == 1, "expected 1 extra entry");
378 return_address = entries.at(0);
379 return start;
380 }
381 StubCodeMark mark(this, stub_id);
382 start = __ pc();
383
384 const Address sp_after_call (rfp, sp_after_call_off * wordSize);
385
386 const Address fpcr_save (rfp, fpcr_off * wordSize);
387 const Address call_wrapper (rfp, call_wrapper_off * wordSize);
388 const Address result (rfp, result_off * wordSize);
389 const Address result_type (rfp, result_type_off * wordSize);
390 const Address method (rfp, method_off * wordSize);
391 const Address entry_point (rfp, entry_point_off * wordSize);
392 const Address parameter_size(rfp, parameter_size_off * wordSize);
393
394 const Address thread (rfp, thread_off * wordSize);
395
396 const Address d15_save (rfp, d15_off * wordSize);
397 const Address d13_save (rfp, d13_off * wordSize);
398 const Address d11_save (rfp, d11_off * wordSize);
399 const Address d9_save (rfp, d9_off * wordSize);
400
401 const Address r28_save (rfp, r28_off * wordSize);
402 const Address r26_save (rfp, r26_off * wordSize);
403 const Address r24_save (rfp, r24_off * wordSize);
404 const Address r22_save (rfp, r22_off * wordSize);
405 const Address r20_save (rfp, r20_off * wordSize);
406
407 // stub code
408
409 address aarch64_entry = __ pc();
410
411 // set up frame and move sp to end of save area
412 __ enter();
413 __ sub(sp, rfp, -sp_after_call_off * wordSize);
414
415 // save register parameters and Java scratch/global registers
416 // n.b. we save thread even though it gets installed in
417 // rthread because we want to sanity check rthread later
418 __ str(c_rarg7, thread);
419 __ strw(c_rarg6, parameter_size);
420 __ stp(c_rarg4, c_rarg5, entry_point);
421 __ stp(c_rarg2, c_rarg3, result_type);
422 __ stp(c_rarg0, c_rarg1, call_wrapper);
423
424 __ stp(r20, r19, r20_save);
425 __ stp(r22, r21, r22_save);
426 __ stp(r24, r23, r24_save);
427 __ stp(r26, r25, r26_save);
428 __ stp(r28, r27, r28_save);
429
430 __ stpd(v9, v8, d9_save);
431 __ stpd(v11, v10, d11_save);
432 __ stpd(v13, v12, d13_save);
433 __ stpd(v15, v14, d15_save);
434
435 __ get_fpcr(rscratch1);
436 __ str(rscratch1, fpcr_save);
437 // Set FPCR to the state we need. We do want Round to Nearest. We
438 // don't want non-IEEE rounding modes or floating-point traps.
439 __ bfi(rscratch1, zr, 22, 4); // Clear DN, FZ, and Rmode
440 __ bfi(rscratch1, zr, 8, 5); // Clear exception-control bits (8-12)
441 __ set_fpcr(rscratch1);
442
443 // install Java thread in global register now we have saved
444 // whatever value it held
445 __ mov(rthread, c_rarg7);
446 // And method
447 __ mov(rmethod, c_rarg3);
448
449 // set up the heapbase register
450 __ reinit_heapbase();
451
452 #ifdef ASSERT
453 // make sure we have no pending exceptions
454 {
455 Label L;
456 __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
457 __ cmp(rscratch1, (u1)NULL_WORD);
458 __ br(Assembler::EQ, L);
459 __ stop("StubRoutines::call_stub: entered with pending exception");
460 __ BIND(L);
461 }
462 #endif
463 // pass parameters if any
464 __ mov(esp, sp);
465 __ sub(rscratch1, sp, c_rarg6, ext::uxtw, LogBytesPerWord); // Move SP out of the way
466 __ andr(sp, rscratch1, -2 * wordSize);
467
468 BLOCK_COMMENT("pass parameters if any");
469 Label parameters_done;
470 // parameter count is still in c_rarg6
471 // and parameter pointer identifying param 1 is in c_rarg5
472 __ cbzw(c_rarg6, parameters_done);
473
474 address loop = __ pc();
475 __ ldr(rscratch1, Address(__ post(c_rarg5, wordSize)));
476 __ subsw(c_rarg6, c_rarg6, 1);
477 __ push(rscratch1);
478 __ br(Assembler::GT, loop);
479
480 __ BIND(parameters_done);
481
482 // call Java entry -- passing methdoOop, and current sp
483 // rmethod: Method*
484 // r19_sender_sp: sender sp
485 BLOCK_COMMENT("call Java function");
486 __ mov(r19_sender_sp, sp);
487 __ blr(c_rarg4);
488
489 // we do this here because the notify will already have been done
490 // if we get to the next instruction via an exception
491 //
492 // n.b. adding this instruction here affects the calculation of
493 // whether or not a routine returns to the call stub (used when
494 // doing stack walks) since the normal test is to check the return
495 // pc against the address saved below. so we may need to allow for
496 // this extra instruction in the check.
497
498 // save current address for use by exception handling code
499
500 return_address = __ pc();
501 entries.append(return_address);
502
503 // store result depending on type (everything that is not
504 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
505 // n.b. this assumes Java returns an integral result in r0
506 // and a floating result in j_farg0
507 // All of j_rargN may be used to return inline type fields so be careful
508 // not to clobber those.
509 // SharedRuntime::generate_buffered_inline_type_adapter() knows the register
510 // assignment of Rresult below.
511 Register Rresult = r14, Rresult_type = r15;
512 __ ldr(Rresult, result);
513 Label is_long, is_float, is_double, check_prim, exit;
514 __ ldr(Rresult_type, result_type);
515 __ cmp(Rresult_type, (u1)T_OBJECT);
516 __ br(Assembler::EQ, check_prim);
517 __ cmp(Rresult_type, (u1)T_LONG);
518 __ br(Assembler::EQ, is_long);
519 __ cmp(Rresult_type, (u1)T_FLOAT);
520 __ br(Assembler::EQ, is_float);
521 __ cmp(Rresult_type, (u1)T_DOUBLE);
522 __ br(Assembler::EQ, is_double);
523
524 // handle T_INT case
525 __ strw(r0, Address(Rresult));
526
527 __ BIND(exit);
528
529 // pop parameters
530 __ sub(esp, rfp, -sp_after_call_off * wordSize);
531
532 #ifdef ASSERT
533 // verify that threads correspond
534 {
535 Label L, S;
536 __ ldr(rscratch1, thread);
537 __ cmp(rthread, rscratch1);
538 __ br(Assembler::NE, S);
539 __ get_thread(rscratch1);
540 __ cmp(rthread, rscratch1);
541 __ br(Assembler::EQ, L);
542 __ BIND(S);
543 __ stop("StubRoutines::call_stub: threads must correspond");
544 __ BIND(L);
545 }
546 #endif
547
548 __ pop_cont_fastpath(rthread);
549
550 // restore callee-save registers
551 __ ldpd(v15, v14, d15_save);
552 __ ldpd(v13, v12, d13_save);
553 __ ldpd(v11, v10, d11_save);
554 __ ldpd(v9, v8, d9_save);
555
556 __ ldp(r28, r27, r28_save);
557 __ ldp(r26, r25, r26_save);
558 __ ldp(r24, r23, r24_save);
559 __ ldp(r22, r21, r22_save);
560 __ ldp(r20, r19, r20_save);
561
562 // restore fpcr
563 __ ldr(rscratch1, fpcr_save);
564 __ set_fpcr(rscratch1);
565
566 __ ldp(c_rarg0, c_rarg1, call_wrapper);
567 __ ldrw(c_rarg2, result_type);
568 __ ldr(c_rarg3, method);
569 __ ldp(c_rarg4, c_rarg5, entry_point);
570 __ ldp(c_rarg6, c_rarg7, parameter_size);
571
572 // leave frame and return to caller
573 __ leave();
574 __ ret(lr);
575
576 // handle return types different from T_INT
577 __ BIND(check_prim);
578 if (InlineTypeReturnedAsFields) {
579 // Check for scalarized return value
580 __ tbz(r0, 0, is_long);
581 // Load pack handler address
582 __ andr(rscratch1, r0, -2);
583 __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
584 __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_jobject_offset()));
585 __ blr(rscratch1);
586 __ b(exit);
587 }
588
589 __ BIND(is_long);
590 __ str(r0, Address(Rresult, 0));
591 __ br(Assembler::AL, exit);
592
593 __ BIND(is_float);
594 __ strs(j_farg0, Address(Rresult, 0));
595 __ br(Assembler::AL, exit);
596
597 __ BIND(is_double);
598 __ strd(j_farg0, Address(Rresult, 0));
599 __ br(Assembler::AL, exit);
600
601 // record the stub entry and end plus the auxiliary entry
602 store_archive_data(stub_id, start, __ pc(), &entries);
603
604 return start;
605 }
606
607 // Return point for a Java call if there's an exception thrown in
608 // Java code. The exception is caught and transformed into a
609 // pending exception stored in JavaThread that can be tested from
610 // within the VM.
611 //
612 // Note: Usually the parameters are removed by the callee. In case
613 // of an exception crossing an activation frame boundary, that is
614 // not the case if the callee is compiled code => need to setup the
615 // rsp.
616 //
617 // r0: exception oop
618
619 address generate_catch_exception() {
620 StubId stub_id = StubId::stubgen_catch_exception_id;
621 int entry_count = StubInfo::entry_count(stub_id);
622 assert(entry_count == 1, "sanity check");
623 address start = load_archive_data(stub_id);
624 if (start != nullptr) {
625 return start;
626 }
627 StubCodeMark mark(this, stub_id);
628 start = __ pc();
629
630 // same as in generate_call_stub():
631 const Address sp_after_call(rfp, sp_after_call_off * wordSize);
632 const Address thread (rfp, thread_off * wordSize);
633
634 #ifdef ASSERT
635 // verify that threads correspond
636 {
637 Label L, S;
638 __ ldr(rscratch1, thread);
639 __ cmp(rthread, rscratch1);
640 __ br(Assembler::NE, S);
641 __ get_thread(rscratch1);
642 __ cmp(rthread, rscratch1);
643 __ br(Assembler::EQ, L);
644 __ bind(S);
645 __ stop("StubRoutines::catch_exception: threads must correspond");
646 __ bind(L);
647 }
648 #endif
649
650 // set pending exception
651 __ verify_oop(r0);
652
653 __ str(r0, Address(rthread, Thread::pending_exception_offset()));
654 // special case -- add file name string to AOT address table
655 address file = (address)AOTCodeCache::add_C_string(__FILE__);
656 __ lea(rscratch1, ExternalAddress(file));
657 __ str(rscratch1, Address(rthread, Thread::exception_file_offset()));
658 __ movw(rscratch1, (int)__LINE__);
659 __ strw(rscratch1, Address(rthread, Thread::exception_line_offset()));
660
661 // complete return to VM
662 assert(StubRoutines::_call_stub_return_address != nullptr,
663 "_call_stub_return_address must have been generated before");
664 __ b(RuntimeAddress(StubRoutines::_call_stub_return_address));
665
666 // record the stub entry and end
667 store_archive_data(stub_id, start, __ pc());
668
669 return start;
670 }
671
672 // Continuation point for runtime calls returning with a pending
673 // exception. The pending exception check happened in the runtime
674 // or native call stub. The pending exception in Thread is
675 // converted into a Java-level exception.
676 //
677 // Contract with Java-level exception handlers:
678 // r0: exception
679 // r3: throwing pc
680 //
681 // NOTE: At entry of this stub, exception-pc must be in LR !!
682
683 // NOTE: this is always used as a jump target within generated code
684 // so it just needs to be generated code with no x86 prolog
685
686 address generate_forward_exception() {
687 StubId stub_id = StubId::stubgen_forward_exception_id;
688 int entry_count = StubInfo::entry_count(stub_id);
689 assert(entry_count == 1, "sanity check");
690 address start = load_archive_data(stub_id);
691 if (start != nullptr) {
692 return start;
693 }
694 StubCodeMark mark(this, stub_id);
695 start = __ pc();
696
697 // Upon entry, LR points to the return address returning into
698 // Java (interpreted or compiled) code; i.e., the return address
699 // becomes the throwing pc.
700 //
701 // Arguments pushed before the runtime call are still on the stack
702 // but the exception handler will reset the stack pointer ->
703 // ignore them. A potential result in registers can be ignored as
704 // well.
705
706 #ifdef ASSERT
707 // make sure this code is only executed if there is a pending exception
708 {
709 Label L;
710 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
711 __ cbnz(rscratch1, L);
712 __ stop("StubRoutines::forward exception: no pending exception (1)");
713 __ bind(L);
714 }
715 #endif
716
717 // compute exception handler into r19
718
719 // call the VM to find the handler address associated with the
720 // caller address. pass thread in r0 and caller pc (ret address)
721 // in r1. n.b. the caller pc is in lr, unlike x86 where it is on
722 // the stack.
723 __ mov(c_rarg1, lr);
724 // lr will be trashed by the VM call so we move it to R19
725 // (callee-saved) because we also need to pass it to the handler
726 // returned by this call.
727 __ mov(r19, lr);
728 BLOCK_COMMENT("call exception_handler_for_return_address");
729 __ call_VM_leaf(CAST_FROM_FN_PTR(address,
730 SharedRuntime::exception_handler_for_return_address),
731 rthread, c_rarg1);
732 // Reinitialize the ptrue predicate register, in case the external runtime
733 // call clobbers ptrue reg, as we may return to SVE compiled code.
734 __ reinitialize_ptrue();
735
736 // we should not really care that lr is no longer the callee
737 // address. we saved the value the handler needs in r19 so we can
738 // just copy it to r3. however, the C2 handler will push its own
739 // frame and then calls into the VM and the VM code asserts that
740 // the PC for the frame above the handler belongs to a compiled
741 // Java method. So, we restore lr here to satisfy that assert.
742 __ mov(lr, r19);
743 // setup r0 & r3 & clear pending exception
744 __ mov(r3, r19);
745 __ mov(r19, r0);
746 __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
747 __ str(zr, Address(rthread, Thread::pending_exception_offset()));
748
749 #ifdef ASSERT
750 // make sure exception is set
751 {
752 Label L;
753 __ cbnz(r0, L);
754 __ stop("StubRoutines::forward exception: no pending exception (2)");
755 __ bind(L);
756 }
757 #endif
758
759 // continue at exception handler
760 // r0: exception
761 // r3: throwing pc
762 // r19: exception handler
763 __ verify_oop(r0);
764 __ br(r19);
765
766 // record the stub entry and end
767 store_archive_data(stub_id, start, __ pc());
768
769 return start;
770 }
771
772 // Non-destructive plausibility checks for oops
773 //
774 // Arguments:
775 // r0: oop to verify
776 // rscratch1: error message
777 //
778 // Stack after saving c_rarg3:
779 // [tos + 0]: saved c_rarg3
780 // [tos + 1]: saved c_rarg2
781 // [tos + 2]: saved lr
782 // [tos + 3]: saved rscratch2
783 // [tos + 4]: saved r0
784 // [tos + 5]: saved rscratch1
785 address generate_verify_oop() {
786 StubId stub_id = StubId::stubgen_verify_oop_id;
787 int entry_count = StubInfo::entry_count(stub_id);
788 assert(entry_count == 1, "sanity check");
789 address start = load_archive_data(stub_id);
790 if (start != nullptr) {
791 return start;
792 }
793 StubCodeMark mark(this, stub_id);
794 start = __ pc();
795
796 Label exit, error;
797
798 // save c_rarg2 and c_rarg3
799 __ stp(c_rarg3, c_rarg2, Address(__ pre(sp, -16)));
800
801 // __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
802 __ lea(c_rarg2, ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
803 __ ldr(c_rarg3, Address(c_rarg2));
804 __ add(c_rarg3, c_rarg3, 1);
805 __ str(c_rarg3, Address(c_rarg2));
806
807 // object is in r0
808 // make sure object is 'reasonable'
809 __ cbz(r0, exit); // if obj is null it is OK
810
811 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
812 bs_asm->check_oop(_masm, r0, c_rarg2, c_rarg3, error);
813
814 // return if everything seems ok
815 __ bind(exit);
816
817 __ ldp(c_rarg3, c_rarg2, Address(__ post(sp, 16)));
818 __ ret(lr);
819
820 // handle errors
821 __ bind(error);
822 __ ldp(c_rarg3, c_rarg2, Address(__ post(sp, 16)));
823
824 __ push(RegSet::range(r0, r29), sp);
825 // debug(char* msg, int64_t pc, int64_t regs[])
826 __ mov(c_rarg0, rscratch1); // pass address of error message
827 __ mov(c_rarg1, lr); // pass return address
828 __ mov(c_rarg2, sp); // pass address of regs on stack
829 #ifndef PRODUCT
830 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
831 #endif
832 BLOCK_COMMENT("call MacroAssembler::debug");
833 __ mov(rscratch1, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
834 __ blr(rscratch1);
835 __ hlt(0);
836
837 // record the stub entry and end
838 store_archive_data(stub_id, start, __ pc());
839
840 return start;
841 }
842
843 // Generate indices for iota vector.
844 void generate_iota_indices(StubId stub_id) {
845 GrowableArray<address> entries;
846 int entry_count = StubInfo::entry_count(stub_id);
847 assert(entry_count == VECTOR_IOTA_COUNT, "sanity check");
848 address start = load_archive_data(stub_id, &entries);
849 if (start != nullptr) {
850 assert(entries.length() == entry_count - 1,
851 "unexpected entries count %d", entries.length());
852 StubRoutines::aarch64::_vector_iota_indices[0] = start;
853 for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
854 StubRoutines::aarch64::_vector_iota_indices[i] = entries.at(i - 1);
855 }
856 return;
857 }
858 __ align(CodeEntryAlignment);
859 StubCodeMark mark(this, stub_id);
860 start = __ pc();
861 // B
862 __ emit_data64(0x0706050403020100, relocInfo::none);
863 __ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none);
864 entries.append(__ pc());
865 // H
866 __ emit_data64(0x0003000200010000, relocInfo::none);
867 __ emit_data64(0x0007000600050004, relocInfo::none);
868 entries.append(__ pc());
869 // S
870 __ emit_data64(0x0000000100000000, relocInfo::none);
871 __ emit_data64(0x0000000300000002, relocInfo::none);
872 entries.append(__ pc());
873 // D
874 __ emit_data64(0x0000000000000000, relocInfo::none);
875 __ emit_data64(0x0000000000000001, relocInfo::none);
876 entries.append(__ pc());
877 // S - FP
878 __ emit_data64(0x3F80000000000000, relocInfo::none); // 0.0f, 1.0f
879 __ emit_data64(0x4040000040000000, relocInfo::none); // 2.0f, 3.0f
880 entries.append(__ pc());
881 // D - FP
882 __ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d
883 __ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d
884
885 // record the stub entry and end
886 store_archive_data(stub_id, start, __ pc(), &entries);
887
888 // install the entry addresses in the entry array
889 assert(entries.length() == entry_count - 1,
890 "unexpected entries count %d", entries.length());
891 StubRoutines::aarch64::_vector_iota_indices[0] = start;
892 for (int i = 1; i < VECTOR_IOTA_COUNT; i++) {
893 StubRoutines::aarch64::_vector_iota_indices[i] = entries.at(i - 1);
894 }
895 }
896
897 // The inner part of zero_words(). This is the bulk operation,
898 // zeroing words in blocks, possibly using DC ZVA to do it. The
899 // caller is responsible for zeroing the last few words.
900 //
901 // Inputs:
902 // r10: the HeapWord-aligned base address of an array to zero.
903 // r11: the count in HeapWords, r11 > 0.
904 //
905 // Returns r10 and r11, adjusted for the caller to clear.
906 // r10: the base address of the tail of words left to clear.
907 // r11: the number of words in the tail.
908 // r11 < MacroAssembler::zero_words_block_size.
909
910 address generate_zero_blocks() {
911 StubId stub_id = StubId::stubgen_zero_blocks_id;
912 int entry_count = StubInfo::entry_count(stub_id);
913 assert(entry_count == 1, "sanity check");
914 address start = load_archive_data(stub_id);
915 if (start != nullptr) {
916 return start;
917 }
918 __ align(CodeEntryAlignment);
919 StubCodeMark mark(this, stub_id);
920 Label done;
921 Label base_aligned;
922
923 Register base = r10, cnt = r11;
924
925 start = __ pc();
926
927 if (UseBlockZeroing) {
928 int zva_length = VM_Version::zva_length();
929
930 // Ensure ZVA length can be divided by 16. This is required by
931 // the subsequent operations.
932 assert (zva_length % 16 == 0, "Unexpected ZVA Length");
933
934 __ tbz(base, 3, base_aligned);
935 __ str(zr, Address(__ post(base, 8)));
936 __ sub(cnt, cnt, 1);
937 __ bind(base_aligned);
938
939 // Ensure count >= zva_length * 2 so that it still deserves a zva after
940 // alignment.
941 Label small;
942 int low_limit = MAX2(zva_length * 2, (int)BlockZeroingLowLimit);
943 __ subs(rscratch1, cnt, low_limit >> 3);
944 __ br(Assembler::LT, small);
945 __ zero_dcache_blocks(base, cnt);
946 __ bind(small);
947 }
948
949 {
950 // Number of stp instructions we'll unroll
951 const int unroll =
952 MacroAssembler::zero_words_block_size / 2;
953 // Clear the remaining blocks.
954 Label loop;
955 __ subs(cnt, cnt, unroll * 2);
956 __ br(Assembler::LT, done);
957 __ bind(loop);
958 for (int i = 0; i < unroll; i++)
959 __ stp(zr, zr, __ post(base, 16));
960 __ subs(cnt, cnt, unroll * 2);
961 __ br(Assembler::GE, loop);
962 __ bind(done);
963 __ add(cnt, cnt, unroll * 2);
964 }
965
966 __ ret(lr);
967
968 // record the stub entry and end
969 store_archive_data(stub_id, start, __ pc());
970
971 return start;
972 }
973
974
975 typedef enum {
976 copy_forwards = 1,
977 copy_backwards = -1
978 } copy_direction;
979
980 // Helper object to reduce noise when telling the GC barriers how to perform loads and stores
981 // for arraycopy stubs.
982 class ArrayCopyBarrierSetHelper : StackObj {
983 BarrierSetAssembler* _bs_asm;
984 MacroAssembler* _masm;
985 DecoratorSet _decorators;
986 BasicType _type;
987 Register _gct1;
988 Register _gct2;
989 Register _gct3;
990 FloatRegister _gcvt1;
991 FloatRegister _gcvt2;
992 FloatRegister _gcvt3;
993
994 public:
995 ArrayCopyBarrierSetHelper(MacroAssembler* masm,
996 DecoratorSet decorators,
997 BasicType type,
998 Register gct1,
999 Register gct2,
1000 Register gct3,
1001 FloatRegister gcvt1,
1002 FloatRegister gcvt2,
1003 FloatRegister gcvt3)
1004 : _bs_asm(BarrierSet::barrier_set()->barrier_set_assembler()),
1005 _masm(masm),
1006 _decorators(decorators),
1007 _type(type),
1008 _gct1(gct1),
1009 _gct2(gct2),
1010 _gct3(gct3),
1011 _gcvt1(gcvt1),
1012 _gcvt2(gcvt2),
1013 _gcvt3(gcvt3) {
1014 }
1015
1016 void copy_load_at_32(FloatRegister dst1, FloatRegister dst2, Address src) {
1017 _bs_asm->copy_load_at(_masm, _decorators, _type, 32,
1018 dst1, dst2, src,
1019 _gct1, _gct2, _gcvt1);
1020 }
1021
1022 void copy_store_at_32(Address dst, FloatRegister src1, FloatRegister src2) {
1023 _bs_asm->copy_store_at(_masm, _decorators, _type, 32,
1024 dst, src1, src2,
1025 _gct1, _gct2, _gct3, _gcvt1, _gcvt2, _gcvt3);
1026 }
1027
1028 void copy_load_at_16(Register dst1, Register dst2, Address src) {
1029 _bs_asm->copy_load_at(_masm, _decorators, _type, 16,
1030 dst1, dst2, src,
1031 _gct1);
1032 }
1033
1034 void copy_store_at_16(Address dst, Register src1, Register src2) {
1035 _bs_asm->copy_store_at(_masm, _decorators, _type, 16,
1036 dst, src1, src2,
1037 _gct1, _gct2, _gct3);
1038 }
1039
1040 void copy_load_at_8(Register dst, Address src) {
1041 _bs_asm->copy_load_at(_masm, _decorators, _type, 8,
1042 dst, noreg, src,
1043 _gct1);
1044 }
1045
1046 void copy_store_at_8(Address dst, Register src) {
1047 _bs_asm->copy_store_at(_masm, _decorators, _type, 8,
1048 dst, src, noreg,
1049 _gct1, _gct2, _gct3);
1050 }
1051 };
1052
1053 // Bulk copy of blocks of 8 words.
1054 //
1055 // count is a count of words.
1056 //
1057 // Precondition: count >= 8
1058 //
1059 // Postconditions:
1060 //
1061 // The least significant bit of count contains the remaining count
1062 // of words to copy. The rest of count is trash.
1063 //
1064 // s and d are adjusted to point to the remaining words to copy
1065 //
1066 address generate_copy_longs(StubId stub_id, DecoratorSet decorators, Register s, Register d, Register count) {
1067 int entry_count = StubInfo::entry_count(stub_id);
1068 assert(entry_count == 1, "sanity check");
1069 address start = load_archive_data(stub_id);
1070 if (start != nullptr) {
1071 return start;
1072 }
1073 BasicType type;
1074 copy_direction direction;
1075
1076 switch (stub_id) {
1077 case StubId::stubgen_copy_byte_f_id:
1078 direction = copy_forwards;
1079 type = T_BYTE;
1080 break;
1081 case StubId::stubgen_copy_byte_b_id:
1082 direction = copy_backwards;
1083 type = T_BYTE;
1084 break;
1085 case StubId::stubgen_copy_oop_f_id:
1086 direction = copy_forwards;
1087 type = T_OBJECT;
1088 break;
1089 case StubId::stubgen_copy_oop_b_id:
1090 direction = copy_backwards;
1091 type = T_OBJECT;
1092 break;
1093 case StubId::stubgen_copy_oop_uninit_f_id:
1094 direction = copy_forwards;
1095 type = T_OBJECT;
1096 break;
1097 case StubId::stubgen_copy_oop_uninit_b_id:
1098 direction = copy_backwards;
1099 type = T_OBJECT;
1100 break;
1101 default:
1102 ShouldNotReachHere();
1103 }
1104
1105 int unit = wordSize * direction;
1106 int bias = (UseSIMDForMemoryOps ? 4:2) * wordSize;
1107
1108 const Register t0 = r3, t1 = r4, t2 = r5, t3 = r6,
1109 t4 = r7, t5 = r11, t6 = r12, t7 = r13;
1110 const Register stride = r14;
1111 const Register gct1 = rscratch1, gct2 = rscratch2, gct3 = r10;
1112 const FloatRegister gcvt1 = v6, gcvt2 = v7, gcvt3 = v16; // Note that v8-v15 are callee saved
1113 ArrayCopyBarrierSetHelper bs(_masm, decorators, type, gct1, gct2, gct3, gcvt1, gcvt2, gcvt3);
1114
1115 assert_different_registers(rscratch1, rscratch2, t0, t1, t2, t3, t4, t5, t6, t7);
1116 assert_different_registers(s, d, count, rscratch1, rscratch2);
1117
1118 Label again, drain;
1119
1120 __ align(CodeEntryAlignment);
1121
1122 StubCodeMark mark(this, stub_id);
1123
1124 start = __ pc();
1125
1126 Label unaligned_copy_long;
1127 if (AvoidUnalignedAccesses) {
1128 __ tbnz(d, 3, unaligned_copy_long);
1129 }
1130
1131 if (direction == copy_forwards) {
1132 __ sub(s, s, bias);
1133 __ sub(d, d, bias);
1134 }
1135
1136 #ifdef ASSERT
1137 // Make sure we are never given < 8 words
1138 {
1139 Label L;
1140 __ cmp(count, (u1)8);
1141 __ br(Assembler::GE, L);
1142 __ stop("genrate_copy_longs called with < 8 words");
1143 __ bind(L);
1144 }
1145 #endif
1146
1147 // Fill 8 registers
1148 if (UseSIMDForMemoryOps) {
1149 bs.copy_load_at_32(v0, v1, Address(s, 4 * unit));
1150 bs.copy_load_at_32(v2, v3, Address(__ pre(s, 8 * unit)));
1151 } else {
1152 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1153 bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1154 bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1155 bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1156 }
1157
1158 __ subs(count, count, 16);
1159 __ br(Assembler::LO, drain);
1160
1161 int prefetch = PrefetchCopyIntervalInBytes;
1162 bool use_stride = false;
1163 if (direction == copy_backwards) {
1164 use_stride = prefetch > 256;
1165 prefetch = -prefetch;
1166 if (use_stride) __ mov(stride, prefetch);
1167 }
1168
1169 __ bind(again);
1170
1171 if (PrefetchCopyIntervalInBytes > 0)
1172 __ prfm(use_stride ? Address(s, stride) : Address(s, prefetch), PLDL1KEEP);
1173
1174 if (UseSIMDForMemoryOps) {
1175 bs.copy_store_at_32(Address(d, 4 * unit), v0, v1);
1176 bs.copy_load_at_32(v0, v1, Address(s, 4 * unit));
1177 bs.copy_store_at_32(Address(__ pre(d, 8 * unit)), v2, v3);
1178 bs.copy_load_at_32(v2, v3, Address(__ pre(s, 8 * unit)));
1179 } else {
1180 bs.copy_store_at_16(Address(d, 2 * unit), t0, t1);
1181 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1182 bs.copy_store_at_16(Address(d, 4 * unit), t2, t3);
1183 bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1184 bs.copy_store_at_16(Address(d, 6 * unit), t4, t5);
1185 bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1186 bs.copy_store_at_16(Address(__ pre(d, 8 * unit)), t6, t7);
1187 bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1188 }
1189
1190 __ subs(count, count, 8);
1191 __ br(Assembler::HS, again);
1192
1193 // Drain
1194 __ bind(drain);
1195 if (UseSIMDForMemoryOps) {
1196 bs.copy_store_at_32(Address(d, 4 * unit), v0, v1);
1197 bs.copy_store_at_32(Address(__ pre(d, 8 * unit)), v2, v3);
1198 } else {
1199 bs.copy_store_at_16(Address(d, 2 * unit), t0, t1);
1200 bs.copy_store_at_16(Address(d, 4 * unit), t2, t3);
1201 bs.copy_store_at_16(Address(d, 6 * unit), t4, t5);
1202 bs.copy_store_at_16(Address(__ pre(d, 8 * unit)), t6, t7);
1203 }
1204
1205 {
1206 Label L1, L2;
1207 __ tbz(count, exact_log2(4), L1);
1208 if (UseSIMDForMemoryOps) {
1209 bs.copy_load_at_32(v0, v1, Address(__ pre(s, 4 * unit)));
1210 bs.copy_store_at_32(Address(__ pre(d, 4 * unit)), v0, v1);
1211 } else {
1212 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1213 bs.copy_load_at_16(t2, t3, Address(__ pre(s, 4 * unit)));
1214 bs.copy_store_at_16(Address(d, 2 * unit), t0, t1);
1215 bs.copy_store_at_16(Address(__ pre(d, 4 * unit)), t2, t3);
1216 }
1217 __ bind(L1);
1218
1219 if (direction == copy_forwards) {
1220 __ add(s, s, bias);
1221 __ add(d, d, bias);
1222 }
1223
1224 __ tbz(count, 1, L2);
1225 bs.copy_load_at_16(t0, t1, Address(__ adjust(s, 2 * unit, direction == copy_backwards)));
1226 bs.copy_store_at_16(Address(__ adjust(d, 2 * unit, direction == copy_backwards)), t0, t1);
1227 __ bind(L2);
1228 }
1229
1230 __ ret(lr);
1231
1232 if (AvoidUnalignedAccesses) {
1233 Label drain, again;
1234 // Register order for storing. Order is different for backward copy.
1235
1236 __ bind(unaligned_copy_long);
1237
1238 // source address is even aligned, target odd aligned
1239 //
1240 // when forward copying word pairs we read long pairs at offsets
1241 // {0, 2, 4, 6} (in long words). when backwards copying we read
1242 // long pairs at offsets {-2, -4, -6, -8}. We adjust the source
1243 // address by -2 in the forwards case so we can compute the
1244 // source offsets for both as {2, 4, 6, 8} * unit where unit = 1
1245 // or -1.
1246 //
1247 // when forward copying we need to store 1 word, 3 pairs and
1248 // then 1 word at offsets {0, 1, 3, 5, 7}. Rather than use a
1249 // zero offset We adjust the destination by -1 which means we
1250 // have to use offsets { 1, 2, 4, 6, 8} * unit for the stores.
1251 //
1252 // When backwards copyng we need to store 1 word, 3 pairs and
1253 // then 1 word at offsets {-1, -3, -5, -7, -8} i.e. we use
1254 // offsets {1, 3, 5, 7, 8} * unit.
1255
1256 if (direction == copy_forwards) {
1257 __ sub(s, s, 16);
1258 __ sub(d, d, 8);
1259 }
1260
1261 // Fill 8 registers
1262 //
1263 // for forwards copy s was offset by -16 from the original input
1264 // value of s so the register contents are at these offsets
1265 // relative to the 64 bit block addressed by that original input
1266 // and so on for each successive 64 byte block when s is updated
1267 //
1268 // t0 at offset 0, t1 at offset 8
1269 // t2 at offset 16, t3 at offset 24
1270 // t4 at offset 32, t5 at offset 40
1271 // t6 at offset 48, t7 at offset 56
1272
1273 // for backwards copy s was not offset so the register contents
1274 // are at these offsets into the preceding 64 byte block
1275 // relative to that original input and so on for each successive
1276 // preceding 64 byte block when s is updated. this explains the
1277 // slightly counter-intuitive looking pattern of register usage
1278 // in the stp instructions for backwards copy.
1279 //
1280 // t0 at offset -16, t1 at offset -8
1281 // t2 at offset -32, t3 at offset -24
1282 // t4 at offset -48, t5 at offset -40
1283 // t6 at offset -64, t7 at offset -56
1284
1285 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1286 bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1287 bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1288 bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1289
1290 __ subs(count, count, 16);
1291 __ br(Assembler::LO, drain);
1292
1293 int prefetch = PrefetchCopyIntervalInBytes;
1294 bool use_stride = false;
1295 if (direction == copy_backwards) {
1296 use_stride = prefetch > 256;
1297 prefetch = -prefetch;
1298 if (use_stride) __ mov(stride, prefetch);
1299 }
1300
1301 __ bind(again);
1302
1303 if (PrefetchCopyIntervalInBytes > 0)
1304 __ prfm(use_stride ? Address(s, stride) : Address(s, prefetch), PLDL1KEEP);
1305
1306 if (direction == copy_forwards) {
1307 // allowing for the offset of -8 the store instructions place
1308 // registers into the target 64 bit block at the following
1309 // offsets
1310 //
1311 // t0 at offset 0
1312 // t1 at offset 8, t2 at offset 16
1313 // t3 at offset 24, t4 at offset 32
1314 // t5 at offset 40, t6 at offset 48
1315 // t7 at offset 56
1316
1317 bs.copy_store_at_8(Address(d, 1 * unit), t0);
1318 bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
1319 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1320 bs.copy_store_at_16(Address(d, 4 * unit), t3, t4);
1321 bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1322 bs.copy_store_at_16(Address(d, 6 * unit), t5, t6);
1323 bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1324 bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t7);
1325 bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1326 } else {
1327 // d was not offset when we started so the registers are
1328 // written into the 64 bit block preceding d with the following
1329 // offsets
1330 //
1331 // t1 at offset -8
1332 // t3 at offset -24, t0 at offset -16
1333 // t5 at offset -48, t2 at offset -32
1334 // t7 at offset -56, t4 at offset -48
1335 // t6 at offset -64
1336 //
1337 // note that this matches the offsets previously noted for the
1338 // loads
1339
1340 bs.copy_store_at_8(Address(d, 1 * unit), t1);
1341 bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
1342 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1343 bs.copy_store_at_16(Address(d, 5 * unit), t5, t2);
1344 bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1345 bs.copy_store_at_16(Address(d, 7 * unit), t7, t4);
1346 bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1347 bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t6);
1348 bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1349 }
1350
1351 __ subs(count, count, 8);
1352 __ br(Assembler::HS, again);
1353
1354 // Drain
1355 //
1356 // this uses the same pattern of offsets and register arguments
1357 // as above
1358 __ bind(drain);
1359 if (direction == copy_forwards) {
1360 bs.copy_store_at_8(Address(d, 1 * unit), t0);
1361 bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
1362 bs.copy_store_at_16(Address(d, 4 * unit), t3, t4);
1363 bs.copy_store_at_16(Address(d, 6 * unit), t5, t6);
1364 bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t7);
1365 } else {
1366 bs.copy_store_at_8(Address(d, 1 * unit), t1);
1367 bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
1368 bs.copy_store_at_16(Address(d, 5 * unit), t5, t2);
1369 bs.copy_store_at_16(Address(d, 7 * unit), t7, t4);
1370 bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t6);
1371 }
1372 // now we need to copy any remaining part block which may
1373 // include a 4 word block subblock and/or a 2 word subblock.
1374 // bits 2 and 1 in the count are the tell-tale for whether we
1375 // have each such subblock
1376 {
1377 Label L1, L2;
1378 __ tbz(count, exact_log2(4), L1);
1379 // this is the same as above but copying only 4 longs hence
1380 // with only one intervening stp between the str instructions
1381 // but note that the offsets and registers still follow the
1382 // same pattern
1383 bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1384 bs.copy_load_at_16(t2, t3, Address(__ pre(s, 4 * unit)));
1385 if (direction == copy_forwards) {
1386 bs.copy_store_at_8(Address(d, 1 * unit), t0);
1387 bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
1388 bs.copy_store_at_8(Address(__ pre(d, 4 * unit)), t3);
1389 } else {
1390 bs.copy_store_at_8(Address(d, 1 * unit), t1);
1391 bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
1392 bs.copy_store_at_8(Address(__ pre(d, 4 * unit)), t2);
1393 }
1394 __ bind(L1);
1395
1396 __ tbz(count, 1, L2);
1397 // this is the same as above but copying only 2 longs hence
1398 // there is no intervening stp between the str instructions
1399 // but note that the offset and register patterns are still
1400 // the same
1401 bs.copy_load_at_16(t0, t1, Address(__ pre(s, 2 * unit)));
1402 if (direction == copy_forwards) {
1403 bs.copy_store_at_8(Address(d, 1 * unit), t0);
1404 bs.copy_store_at_8(Address(__ pre(d, 2 * unit)), t1);
1405 } else {
1406 bs.copy_store_at_8(Address(d, 1 * unit), t1);
1407 bs.copy_store_at_8(Address(__ pre(d, 2 * unit)), t0);
1408 }
1409 __ bind(L2);
1410
1411 // for forwards copy we need to re-adjust the offsets we
1412 // applied so that s and d are follow the last words written
1413
1414 if (direction == copy_forwards) {
1415 __ add(s, s, 16);
1416 __ add(d, d, 8);
1417 }
1418
1419 }
1420
1421 __ ret(lr);
1422 }
1423
1424 // record the stub entry and end
1425 store_archive_data(stub_id, start, __ pc());
1426
1427 return start;
1428 }
1429
1430 // Small copy: less than 16 bytes.
1431 //
1432 // NB: Ignores all of the bits of count which represent more than 15
1433 // bytes, so a caller doesn't have to mask them.
1434
1435 void copy_memory_small(DecoratorSet decorators, BasicType type, Register s, Register d, Register count, int step) {
1436 bool is_backwards = step < 0;
1437 size_t granularity = g_uabs(step);
1438 int direction = is_backwards ? -1 : 1;
1439
1440 Label Lword, Lint, Lshort, Lbyte;
1441
1442 assert(granularity
1443 && granularity <= sizeof (jlong), "Impossible granularity in copy_memory_small");
1444
1445 const Register t0 = r3;
1446 const Register gct1 = rscratch1, gct2 = rscratch2, gct3 = r10;
1447 ArrayCopyBarrierSetHelper bs(_masm, decorators, type, gct1, gct2, gct3, fnoreg, fnoreg, fnoreg);
1448
1449 // ??? I don't know if this bit-test-and-branch is the right thing
1450 // to do. It does a lot of jumping, resulting in several
1451 // mispredicted branches. It might make more sense to do this
1452 // with something like Duff's device with a single computed branch.
1453
1454 __ tbz(count, 3 - exact_log2(granularity), Lword);
1455 bs.copy_load_at_8(t0, Address(__ adjust(s, direction * wordSize, is_backwards)));
1456 bs.copy_store_at_8(Address(__ adjust(d, direction * wordSize, is_backwards)), t0);
1457 __ bind(Lword);
1458
1459 if (granularity <= sizeof (jint)) {
1460 __ tbz(count, 2 - exact_log2(granularity), Lint);
1461 __ ldrw(t0, Address(__ adjust(s, sizeof (jint) * direction, is_backwards)));
1462 __ strw(t0, Address(__ adjust(d, sizeof (jint) * direction, is_backwards)));
1463 __ bind(Lint);
1464 }
1465
1466 if (granularity <= sizeof (jshort)) {
1467 __ tbz(count, 1 - exact_log2(granularity), Lshort);
1468 __ ldrh(t0, Address(__ adjust(s, sizeof (jshort) * direction, is_backwards)));
1469 __ strh(t0, Address(__ adjust(d, sizeof (jshort) * direction, is_backwards)));
1470 __ bind(Lshort);
1471 }
1472
1473 if (granularity <= sizeof (jbyte)) {
1474 __ tbz(count, 0, Lbyte);
1475 __ ldrb(t0, Address(__ adjust(s, sizeof (jbyte) * direction, is_backwards)));
1476 __ strb(t0, Address(__ adjust(d, sizeof (jbyte) * direction, is_backwards)));
1477 __ bind(Lbyte);
1478 }
1479 }
1480
1481 // All-singing all-dancing memory copy.
1482 //
1483 // Copy count units of memory from s to d. The size of a unit is
1484 // step, which can be positive or negative depending on the direction
1485 // of copy. If is_aligned is false, we align the source address.
1486 //
1487
1488 void copy_memory(DecoratorSet decorators, BasicType type, bool is_aligned,
1489 Register s, Register d, Register count, int step) {
1490 copy_direction direction = step < 0 ? copy_backwards : copy_forwards;
1491 bool is_backwards = step < 0;
1492 unsigned int granularity = g_uabs(step);
1493 const Register t0 = r3, t1 = r4;
1494
1495 // <= 80 (or 96 for SIMD) bytes do inline. Direction doesn't matter because we always
1496 // load all the data before writing anything
1497 Label copy4, copy8, copy16, copy32, copy80, copy_big, finish;
1498 const Register t2 = r5, t3 = r6, t4 = r7, t5 = r11;
1499 const Register t6 = r12, t7 = r13, t8 = r14, t9 = r15;
1500 const Register send = r17, dend = r16;
1501 const Register gct1 = rscratch1, gct2 = rscratch2, gct3 = r10;
1502 const FloatRegister gcvt1 = v6, gcvt2 = v7, gcvt3 = v16; // Note that v8-v15 are callee saved
1503 ArrayCopyBarrierSetHelper bs(_masm, decorators, type, gct1, gct2, gct3, gcvt1, gcvt2, gcvt3);
1504
1505 if (PrefetchCopyIntervalInBytes > 0)
1506 __ prfm(Address(s, 0), PLDL1KEEP);
1507 __ cmp(count, u1((UseSIMDForMemoryOps ? 96:80)/granularity));
1508 __ br(Assembler::HI, copy_big);
1509
1510 __ lea(send, Address(s, count, Address::lsl(exact_log2(granularity))));
1511 __ lea(dend, Address(d, count, Address::lsl(exact_log2(granularity))));
1512
1513 __ cmp(count, u1(16/granularity));
1514 __ br(Assembler::LS, copy16);
1515
1516 __ cmp(count, u1(64/granularity));
1517 __ br(Assembler::HI, copy80);
1518
1519 __ cmp(count, u1(32/granularity));
1520 __ br(Assembler::LS, copy32);
1521
1522 // 33..64 bytes
1523 if (UseSIMDForMemoryOps) {
1524 bs.copy_load_at_32(v0, v1, Address(s, 0));
1525 bs.copy_load_at_32(v2, v3, Address(send, -32));
1526 bs.copy_store_at_32(Address(d, 0), v0, v1);
1527 bs.copy_store_at_32(Address(dend, -32), v2, v3);
1528 } else {
1529 bs.copy_load_at_16(t0, t1, Address(s, 0));
1530 bs.copy_load_at_16(t2, t3, Address(s, 16));
1531 bs.copy_load_at_16(t4, t5, Address(send, -32));
1532 bs.copy_load_at_16(t6, t7, Address(send, -16));
1533
1534 bs.copy_store_at_16(Address(d, 0), t0, t1);
1535 bs.copy_store_at_16(Address(d, 16), t2, t3);
1536 bs.copy_store_at_16(Address(dend, -32), t4, t5);
1537 bs.copy_store_at_16(Address(dend, -16), t6, t7);
1538 }
1539 __ b(finish);
1540
1541 // 17..32 bytes
1542 __ bind(copy32);
1543 bs.copy_load_at_16(t0, t1, Address(s, 0));
1544 bs.copy_load_at_16(t6, t7, Address(send, -16));
1545
1546 bs.copy_store_at_16(Address(d, 0), t0, t1);
1547 bs.copy_store_at_16(Address(dend, -16), t6, t7);
1548 __ b(finish);
1549
1550 // 65..80/96 bytes
1551 // (96 bytes if SIMD because we do 32 byes per instruction)
1552 __ bind(copy80);
1553 if (UseSIMDForMemoryOps) {
1554 bs.copy_load_at_32(v0, v1, Address(s, 0));
1555 bs.copy_load_at_32(v2, v3, Address(s, 32));
1556 // Unaligned pointers can be an issue for copying.
1557 // The issue has more chances to happen when granularity of data is
1558 // less than 4(sizeof(jint)). Pointers for arrays of jint are at least
1559 // 4 byte aligned. Pointers for arrays of jlong are 8 byte aligned.
1560 // The most performance drop has been seen for the range 65-80 bytes.
1561 // For such cases using the pair of ldp/stp instead of the third pair of
1562 // ldpq/stpq fixes the performance issue.
1563 if (granularity < sizeof (jint)) {
1564 Label copy96;
1565 __ cmp(count, u1(80/granularity));
1566 __ br(Assembler::HI, copy96);
1567 bs.copy_load_at_16(t0, t1, Address(send, -16));
1568
1569 bs.copy_store_at_32(Address(d, 0), v0, v1);
1570 bs.copy_store_at_32(Address(d, 32), v2, v3);
1571
1572 bs.copy_store_at_16(Address(dend, -16), t0, t1);
1573 __ b(finish);
1574
1575 __ bind(copy96);
1576 }
1577 bs.copy_load_at_32(v4, v5, Address(send, -32));
1578
1579 bs.copy_store_at_32(Address(d, 0), v0, v1);
1580 bs.copy_store_at_32(Address(d, 32), v2, v3);
1581
1582 bs.copy_store_at_32(Address(dend, -32), v4, v5);
1583 } else {
1584 bs.copy_load_at_16(t0, t1, Address(s, 0));
1585 bs.copy_load_at_16(t2, t3, Address(s, 16));
1586 bs.copy_load_at_16(t4, t5, Address(s, 32));
1587 bs.copy_load_at_16(t6, t7, Address(s, 48));
1588 bs.copy_load_at_16(t8, t9, Address(send, -16));
1589
1590 bs.copy_store_at_16(Address(d, 0), t0, t1);
1591 bs.copy_store_at_16(Address(d, 16), t2, t3);
1592 bs.copy_store_at_16(Address(d, 32), t4, t5);
1593 bs.copy_store_at_16(Address(d, 48), t6, t7);
1594 bs.copy_store_at_16(Address(dend, -16), t8, t9);
1595 }
1596 __ b(finish);
1597
1598 // 0..16 bytes
1599 __ bind(copy16);
1600 __ cmp(count, u1(8/granularity));
1601 __ br(Assembler::LO, copy8);
1602
1603 // 8..16 bytes
1604 bs.copy_load_at_8(t0, Address(s, 0));
1605 bs.copy_load_at_8(t1, Address(send, -8));
1606 bs.copy_store_at_8(Address(d, 0), t0);
1607 bs.copy_store_at_8(Address(dend, -8), t1);
1608 __ b(finish);
1609
1610 if (granularity < 8) {
1611 // 4..7 bytes
1612 __ bind(copy8);
1613 __ tbz(count, 2 - exact_log2(granularity), copy4);
1614 __ ldrw(t0, Address(s, 0));
1615 __ ldrw(t1, Address(send, -4));
1616 __ strw(t0, Address(d, 0));
1617 __ strw(t1, Address(dend, -4));
1618 __ b(finish);
1619 if (granularity < 4) {
1620 // 0..3 bytes
1621 __ bind(copy4);
1622 __ cbz(count, finish); // get rid of 0 case
1623 if (granularity == 2) {
1624 __ ldrh(t0, Address(s, 0));
1625 __ strh(t0, Address(d, 0));
1626 } else { // granularity == 1
1627 // Now 1..3 bytes. Handle the 1 and 2 byte case by copying
1628 // the first and last byte.
1629 // Handle the 3 byte case by loading and storing base + count/2
1630 // (count == 1 (s+0)->(d+0), count == 2,3 (s+1) -> (d+1))
1631 // This does means in the 1 byte case we load/store the same
1632 // byte 3 times.
1633 __ lsr(count, count, 1);
1634 __ ldrb(t0, Address(s, 0));
1635 __ ldrb(t1, Address(send, -1));
1636 __ ldrb(t2, Address(s, count));
1637 __ strb(t0, Address(d, 0));
1638 __ strb(t1, Address(dend, -1));
1639 __ strb(t2, Address(d, count));
1640 }
1641 __ b(finish);
1642 }
1643 }
1644
1645 __ bind(copy_big);
1646 if (is_backwards) {
1647 __ lea(s, Address(s, count, Address::lsl(exact_log2(-step))));
1648 __ lea(d, Address(d, count, Address::lsl(exact_log2(-step))));
1649 }
1650
1651 // Now we've got the small case out of the way we can align the
1652 // source address on a 2-word boundary.
1653
1654 // Here we will materialize a count in r15, which is used by copy_memory_small
1655 // and the various generate_copy_longs stubs that we use for 2 word aligned bytes.
1656 // Up until here, we have used t9, which aliases r15, but from here on, that register
1657 // can not be used as a temp register, as it contains the count.
1658
1659 Label aligned;
1660
1661 if (is_aligned) {
1662 // We may have to adjust by 1 word to get s 2-word-aligned.
1663 __ tbz(s, exact_log2(wordSize), aligned);
1664 bs.copy_load_at_8(t0, Address(__ adjust(s, direction * wordSize, is_backwards)));
1665 bs.copy_store_at_8(Address(__ adjust(d, direction * wordSize, is_backwards)), t0);
1666 __ sub(count, count, wordSize/granularity);
1667 } else {
1668 if (is_backwards) {
1669 __ andr(r15, s, 2 * wordSize - 1);
1670 } else {
1671 __ neg(r15, s);
1672 __ andr(r15, r15, 2 * wordSize - 1);
1673 }
1674 // r15 is the byte adjustment needed to align s.
1675 __ cbz(r15, aligned);
1676 int shift = exact_log2(granularity);
1677 if (shift > 0) {
1678 __ lsr(r15, r15, shift);
1679 }
1680 __ sub(count, count, r15);
1681
1682 #if 0
1683 // ?? This code is only correct for a disjoint copy. It may or
1684 // may not make sense to use it in that case.
1685
1686 // Copy the first pair; s and d may not be aligned.
1687 __ ldp(t0, t1, Address(s, is_backwards ? -2 * wordSize : 0));
1688 __ stp(t0, t1, Address(d, is_backwards ? -2 * wordSize : 0));
1689
1690 // Align s and d, adjust count
1691 if (is_backwards) {
1692 __ sub(s, s, r15);
1693 __ sub(d, d, r15);
1694 } else {
1695 __ add(s, s, r15);
1696 __ add(d, d, r15);
1697 }
1698 #else
1699 copy_memory_small(decorators, type, s, d, r15, step);
1700 #endif
1701 }
1702
1703 __ bind(aligned);
1704
1705 // s is now 2-word-aligned.
1706
1707 // We have a count of units and some trailing bytes. Adjust the
1708 // count and do a bulk copy of words. If the shift is zero
1709 // perform a move instead to benefit from zero latency moves.
1710 int shift = exact_log2(wordSize/granularity);
1711 if (shift > 0) {
1712 __ lsr(r15, count, shift);
1713 } else {
1714 __ mov(r15, count);
1715 }
1716 if (direction == copy_forwards) {
1717 if (type != T_OBJECT) {
1718 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_byte_f()));
1719 __ blr(rscratch1);
1720 } else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
1721 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_uninit_f()));
1722 __ blr(rscratch1);
1723 } else {
1724 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_f()));
1725 __ blr(rscratch1);
1726 }
1727 } else {
1728 if (type != T_OBJECT) {
1729 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_byte_b()));
1730 __ blr(rscratch1);
1731 } else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
1732 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_uninit_b()));
1733 __ blr(rscratch1);
1734 } else {
1735 __ lea(rscratch1, RuntimeAddress(StubRoutines::aarch64::copy_oop_b()));
1736 __ blr(rscratch1);
1737 }
1738 }
1739
1740 // And the tail.
1741 copy_memory_small(decorators, type, s, d, count, step);
1742
1743 if (granularity >= 8) __ bind(copy8);
1744 if (granularity >= 4) __ bind(copy4);
1745 __ bind(finish);
1746 }
1747
1748
1749 void clobber_registers() {
1750 #ifdef ASSERT
1751 RegSet clobbered
1752 = MacroAssembler::call_clobbered_gp_registers() - rscratch1;
1753 __ mov(rscratch1, (uint64_t)0xdeadbeef);
1754 __ orr(rscratch1, rscratch1, rscratch1, Assembler::LSL, 32);
1755 for (RegSetIterator<Register> it = clobbered.begin(); *it != noreg; ++it) {
1756 __ mov(*it, rscratch1);
1757 }
1758 #endif
1759
1760 }
1761
1762 // Scan over array at a for count oops, verifying each one.
1763 // Preserves a and count, clobbers rscratch1 and rscratch2.
1764 void verify_oop_array (int size, Register a, Register count, Register temp) {
1765 Label loop, end;
1766 __ mov(rscratch1, a);
1767 __ mov(rscratch2, zr);
1768 __ bind(loop);
1769 __ cmp(rscratch2, count);
1770 __ br(Assembler::HS, end);
1771 if (size == wordSize) {
1772 __ ldr(temp, Address(a, rscratch2, Address::lsl(exact_log2(size))));
1773 __ verify_oop(temp);
1774 } else {
1775 __ ldrw(temp, Address(a, rscratch2, Address::lsl(exact_log2(size))));
1776 __ decode_heap_oop(temp); // calls verify_oop
1777 }
1778 __ add(rscratch2, rscratch2, 1);
1779 __ b(loop);
1780 __ bind(end);
1781 }
1782
1783 // Arguments:
1784 // stub_id - is used to name the stub and identify all details of
1785 // how to perform the copy.
1786 //
1787 // nopush_entry - is assigned to the stub's post push entry point
1788 // unless it is null
1789 //
1790 // Inputs:
1791 // c_rarg0 - source array address
1792 // c_rarg1 - destination array address
1793 // c_rarg2 - element count, treated as ssize_t, can be zero
1794 //
1795 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1796 // the hardware handle it. The two dwords within qwords that span
1797 // cache line boundaries will still be loaded and stored atomically.
1798 //
1799 // Side Effects: nopush_entry is set to the (post push) entry point
1800 // so it can be used by the corresponding conjoint
1801 // copy method
1802 //
1803 address generate_disjoint_copy(StubId stub_id, address *nopush_entry) {
1804 int size;
1805 bool aligned;
1806 bool is_oop;
1807 bool dest_uninitialized;
1808 switch (stub_id) {
1809 case StubId::stubgen_jbyte_disjoint_arraycopy_id:
1810 size = sizeof(jbyte);
1811 aligned = false;
1812 is_oop = false;
1813 dest_uninitialized = false;
1814 break;
1815 case StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id:
1816 size = sizeof(jbyte);
1817 aligned = true;
1818 is_oop = false;
1819 dest_uninitialized = false;
1820 break;
1821 case StubId::stubgen_jshort_disjoint_arraycopy_id:
1822 size = sizeof(jshort);
1823 aligned = false;
1824 is_oop = false;
1825 dest_uninitialized = false;
1826 break;
1827 case StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id:
1828 size = sizeof(jshort);
1829 aligned = true;
1830 is_oop = false;
1831 dest_uninitialized = false;
1832 break;
1833 case StubId::stubgen_jint_disjoint_arraycopy_id:
1834 size = sizeof(jint);
1835 aligned = false;
1836 is_oop = false;
1837 dest_uninitialized = false;
1838 break;
1839 case StubId::stubgen_arrayof_jint_disjoint_arraycopy_id:
1840 size = sizeof(jint);
1841 aligned = true;
1842 is_oop = false;
1843 dest_uninitialized = false;
1844 break;
1845 case StubId::stubgen_jlong_disjoint_arraycopy_id:
1846 // since this is always aligned we can (should!) use the same
1847 // stub as for case StubId::stubgen_arrayof_jlong_disjoint_arraycopy
1848 ShouldNotReachHere();
1849 break;
1850 case StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id:
1851 size = sizeof(jlong);
1852 aligned = true;
1853 is_oop = false;
1854 dest_uninitialized = false;
1855 break;
1856 case StubId::stubgen_oop_disjoint_arraycopy_id:
1857 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1858 aligned = !UseCompressedOops;
1859 is_oop = true;
1860 dest_uninitialized = false;
1861 break;
1862 case StubId::stubgen_arrayof_oop_disjoint_arraycopy_id:
1863 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1864 aligned = !UseCompressedOops;
1865 is_oop = true;
1866 dest_uninitialized = false;
1867 break;
1868 case StubId::stubgen_oop_disjoint_arraycopy_uninit_id:
1869 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1870 aligned = !UseCompressedOops;
1871 is_oop = true;
1872 dest_uninitialized = true;
1873 break;
1874 case StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id:
1875 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1876 aligned = !UseCompressedOops;
1877 is_oop = true;
1878 dest_uninitialized = true;
1879 break;
1880 default:
1881 ShouldNotReachHere();
1882 break;
1883 }
1884 // all stubs provide a 2nd entry which omits the frame push for
1885 // use when bailing out from a conjoint copy. However we may also
1886 // need some extra addressses for memory access protection.
1887 int entry_count = StubInfo::entry_count(stub_id);
1888 assert(entry_count == 2, "sanity check");
1889 assert(nopush_entry != nullptr, "all disjoint copy stubs export a nopush entry");
1890
1891 bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size);
1892 int extra_count = ((add_extras ? 1 : 0) * UnsafeMemoryAccess::COLUMN_COUNT);
1893 GrowableArray<address> entries;
1894 GrowableArray<address> extras;
1895 GrowableArray<address> *extras_ptr = (extra_count > 0 ? &extras : nullptr);
1896 address start = load_archive_data(stub_id, &entries, extras_ptr);
1897 if (start != nullptr) {
1898 assert(entries.length() == entry_count - 1,
1899 "unexpected entries count %d", entries.length());
1900 *nopush_entry = entries.at(0);
1901 assert(extras.length() == extra_count,
1902 "unexpected extra count %d", extras.length());
1903 if (add_extras) {
1904 // register one handler at offset 0
1905 register_unsafe_access_handlers(extras, 0, 1);
1906 }
1907 return start;
1908 }
1909
1910 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1911 RegSet saved_reg = RegSet::of(s, d, count);
1912
1913 __ align(CodeEntryAlignment);
1914 StubCodeMark mark(this, stub_id);
1915 start = __ pc();
1916 __ enter();
1917
1918 *nopush_entry = __ pc();
1919 entries.append(*nopush_entry);
1920
1921 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1922 BLOCK_COMMENT("Post-Push Entry:");
1923
1924 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
1925 if (dest_uninitialized) {
1926 decorators |= IS_DEST_UNINITIALIZED;
1927 }
1928 if (aligned) {
1929 decorators |= ARRAYCOPY_ALIGNED;
1930 }
1931
1932 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1933 bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_reg);
1934
1935 if (is_oop) {
1936 // save regs before copy_memory
1937 __ push(RegSet::of(d, count), sp);
1938 }
1939 {
1940 // UnsafeMemoryAccess page error: continue after unsafe access
1941 UnsafeMemoryAccessMark umam(this, add_extras, true);
1942 copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size);
1943 }
1944
1945 if (is_oop) {
1946 __ pop(RegSet::of(d, count), sp);
1947 if (VerifyOops)
1948 verify_oop_array(size, d, count, r16);
1949 }
1950
1951 bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1);
1952
1953 __ leave();
1954 __ mov(r0, zr); // return 0
1955 __ ret(lr);
1956
1957 address end = __ pc();
1958
1959 if (add_extras) {
1960 // retrieve the registered handler addresses
1961 retrieve_unsafe_access_handlers(start, end, extras);
1962 assert(extras.length() == extra_count
1963 , "incorrect handlers count %d", extras.length());
1964 }
1965
1966 // record the stub entry and end plus the no_push entry and any
1967 // extra handler addresses
1968 store_archive_data(stub_id, start, end, &entries, extras_ptr);
1969
1970 return start;
1971 }
1972
1973 // Arguments:
1974 // stub_id - is used to name the stub and identify all details of
1975 // how to perform the copy.
1976 //
1977 // nooverlap_target - identifes the (post push) entry for the
1978 // corresponding disjoint copy routine which can be
1979 // jumped to if the ranges do not actually overlap
1980 //
1981 // nopush_entry - is assigned to the stub's post push entry point
1982 // unless it is null
1983 //
1984 //
1985 // Inputs:
1986 // c_rarg0 - source array address
1987 // c_rarg1 - destination array address
1988 // c_rarg2 - element count, treated as ssize_t, can be zero
1989 //
1990 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1991 // the hardware handle it. The two dwords within qwords that span
1992 // cache line boundaries will still be loaded and stored atomically.
1993 //
1994 // Side Effects:
1995 // nopush_entry is set to the no-overlap entry point so it can be
1996 // used by some other conjoint copy method
1997 //
1998 address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) {
1999 int size;
2000 bool aligned;
2001 bool is_oop;
2002 bool dest_uninitialized;
2003 switch (stub_id) {
2004 case StubId::stubgen_jbyte_arraycopy_id:
2005 size = sizeof(jbyte);
2006 aligned = false;
2007 is_oop = false;
2008 dest_uninitialized = false;
2009 break;
2010 case StubId::stubgen_arrayof_jbyte_arraycopy_id:
2011 size = sizeof(jbyte);
2012 aligned = true;
2013 is_oop = false;
2014 dest_uninitialized = false;
2015 break;
2016 case StubId::stubgen_jshort_arraycopy_id:
2017 size = sizeof(jshort);
2018 aligned = false;
2019 is_oop = false;
2020 dest_uninitialized = false;
2021 break;
2022 case StubId::stubgen_arrayof_jshort_arraycopy_id:
2023 size = sizeof(jshort);
2024 aligned = true;
2025 is_oop = false;
2026 dest_uninitialized = false;
2027 break;
2028 case StubId::stubgen_jint_arraycopy_id:
2029 size = sizeof(jint);
2030 aligned = false;
2031 is_oop = false;
2032 dest_uninitialized = false;
2033 break;
2034 case StubId::stubgen_arrayof_jint_arraycopy_id:
2035 size = sizeof(jint);
2036 aligned = true;
2037 is_oop = false;
2038 dest_uninitialized = false;
2039 break;
2040 case StubId::stubgen_jlong_arraycopy_id:
2041 // since this is always aligned we can (should!) use the same
2042 // stub as for case StubId::stubgen_arrayof_jlong_disjoint_arraycopy
2043 ShouldNotReachHere();
2044 break;
2045 case StubId::stubgen_arrayof_jlong_arraycopy_id:
2046 size = sizeof(jlong);
2047 aligned = true;
2048 is_oop = false;
2049 dest_uninitialized = false;
2050 break;
2051 case StubId::stubgen_oop_arraycopy_id:
2052 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
2053 aligned = !UseCompressedOops;
2054 is_oop = true;
2055 dest_uninitialized = false;
2056 break;
2057 case StubId::stubgen_arrayof_oop_arraycopy_id:
2058 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
2059 aligned = !UseCompressedOops;
2060 is_oop = true;
2061 dest_uninitialized = false;
2062 break;
2063 case StubId::stubgen_oop_arraycopy_uninit_id:
2064 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
2065 aligned = !UseCompressedOops;
2066 is_oop = true;
2067 dest_uninitialized = true;
2068 break;
2069 case StubId::stubgen_arrayof_oop_arraycopy_uninit_id:
2070 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
2071 aligned = !UseCompressedOops;
2072 is_oop = true;
2073 dest_uninitialized = true;
2074 break;
2075 default:
2076 ShouldNotReachHere();
2077 }
2078 // only some conjoint stubs generate a 2nd entry
2079 int entry_count = StubInfo::entry_count(stub_id);
2080 int expected_entry_count = (nopush_entry == nullptr ? 1 : 2);
2081 assert(entry_count == expected_entry_count,
2082 "expected entry count %d does not match declared entry count %d for stub %s",
2083 expected_entry_count, entry_count, StubInfo::name(stub_id));
2084
2085 // We need to protect memory accesses in certain cases
2086 bool add_extras = !is_oop && (!aligned || sizeof(jlong) == size);
2087 int extra_count = ((add_extras ? 1 : 0) * UnsafeMemoryAccess::COLUMN_COUNT);
2088 GrowableArray<address> entries;
2089 GrowableArray<address> extras;
2090 GrowableArray<address> *entries_ptr = (nopush_entry != nullptr ? &entries : nullptr);
2091 GrowableArray<address> *extras_ptr = (extra_count > 0 ? &extras : nullptr);
2092 address start = load_archive_data(stub_id, entries_ptr, extras_ptr);
2093 if (start != nullptr) {
2094 assert(entries.length() == expected_entry_count - 1,
2095 "unexpected entries count %d", entries.length());
2096 assert(extras.length() == extra_count,
2097 "unexpected extra count %d", extras.length());
2098 if (nopush_entry != nullptr) {
2099 *nopush_entry = entries.at(0);
2100 }
2101 if (add_extras) {
2102 // register one handler at offset 0
2103 register_unsafe_access_handlers(extras, 0, 1);
2104 }
2105 return start;
2106 }
2107
2108 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
2109 RegSet saved_regs = RegSet::of(s, d, count);
2110 StubCodeMark mark(this, stub_id);
2111 start = __ pc();
2112 __ enter();
2113
2114 if (nopush_entry != nullptr) {
2115 *nopush_entry = __ pc();
2116 entries.append(*nopush_entry);
2117 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2118 BLOCK_COMMENT("Post-Push Entry:");
2119 }
2120
2121 // use fwd copy when (d-s) above_equal (count*size)
2122 Label L_overlapping;
2123 __ sub(rscratch1, d, s);
2124 __ cmp(rscratch1, count, Assembler::LSL, exact_log2(size));
2125 __ br(Assembler::LO, L_overlapping);
2126 __ b(RuntimeAddress(nooverlap_target));
2127 __ bind(L_overlapping);
2128
2129 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
2130 if (dest_uninitialized) {
2131 decorators |= IS_DEST_UNINITIALIZED;
2132 }
2133 if (aligned) {
2134 decorators |= ARRAYCOPY_ALIGNED;
2135 }
2136
2137 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2138 bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_regs);
2139
2140 if (is_oop) {
2141 // save regs before copy_memory
2142 __ push(RegSet::of(d, count), sp);
2143 }
2144 {
2145 // UnsafeMemoryAccess page error: continue after unsafe access
2146 UnsafeMemoryAccessMark umam(this, add_extras, true);
2147 copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size);
2148 }
2149 if (is_oop) {
2150 __ pop(RegSet::of(d, count), sp);
2151 if (VerifyOops)
2152 verify_oop_array(size, d, count, r16);
2153 }
2154 bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1);
2155 __ leave();
2156 __ mov(r0, zr); // return 0
2157 __ ret(lr);
2158
2159 assert(entries.length() == expected_entry_count - 1,
2160 "unexpected entries count %d", entries.length());
2161
2162 address end = __ pc();
2163
2164 if (add_extras) {
2165 // retrieve the registered handler addresses
2166 retrieve_unsafe_access_handlers(start, end, extras);
2167 assert(extras.length() == extra_count,
2168 "incorrect handlers count %d", extras.length());
2169 }
2170
2171 // record the stub entry and end plus any no_push entry and/or
2172 // extra handler addresses
2173 store_archive_data(stub_id, start, end, entries_ptr, extras_ptr);
2174
2175 return start;
2176 }
2177
2178 // Helper for generating a dynamic type check.
2179 // Smashes rscratch1, rscratch2.
2180 void generate_type_check(Register sub_klass,
2181 Register super_check_offset,
2182 Register super_klass,
2183 Register temp1,
2184 Register temp2,
2185 Register result,
2186 Label& L_success) {
2187 assert_different_registers(sub_klass, super_check_offset, super_klass);
2188
2189 BLOCK_COMMENT("type_check:");
2190
2191 Label L_miss;
2192
2193 __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, nullptr,
2194 super_check_offset);
2195 __ check_klass_subtype_slow_path(sub_klass, super_klass, temp1, temp2, &L_success, nullptr);
2196
2197 // Fall through on failure!
2198 __ BIND(L_miss);
2199 }
2200
2201 //
2202 // Generate checkcasting array copy stub
2203 //
2204 // Input:
2205 // c_rarg0 - source array address
2206 // c_rarg1 - destination array address
2207 // c_rarg2 - element count, treated as ssize_t, can be zero
2208 // c_rarg3 - size_t ckoff (super_check_offset)
2209 // c_rarg4 - oop ckval (super_klass)
2210 //
2211 // Output:
2212 // r0 == 0 - success
2213 // r0 == -1^K - failure, where K is partial transfer count
2214 //
2215 address generate_checkcast_copy(StubId stub_id, address *nopush_entry) {
2216 bool dest_uninitialized;
2217 switch (stub_id) {
2218 case StubId::stubgen_checkcast_arraycopy_id:
2219 dest_uninitialized = false;
2220 break;
2221 case StubId::stubgen_checkcast_arraycopy_uninit_id:
2222 dest_uninitialized = true;
2223 break;
2224 default:
2225 ShouldNotReachHere();
2226 }
2227
2228 // The normal stub provides a 2nd entry which omits the frame push
2229 // for use when bailing out from a disjoint copy.
2230 // Only some conjoint stubs generate a 2nd entry
2231 int entry_count = StubInfo::entry_count(stub_id);
2232 int expected_entry_count = (nopush_entry == nullptr ? 1 : 2);
2233 GrowableArray<address> entries;
2234 GrowableArray<address> *entries_ptr = (expected_entry_count == 1 ? nullptr : &entries);
2235 assert(entry_count == expected_entry_count,
2236 "expected entry count %d does not match declared entry count %d for stub %s",
2237 expected_entry_count, entry_count, StubInfo::name(stub_id));
2238 address start = load_archive_data(stub_id, entries_ptr);
2239 if (start != nullptr) {
2240 assert(entries.length() + 1 == expected_entry_count,
2241 "expected entry count %d does not match return entry count %d for stub %s",
2242 expected_entry_count, entries.length() + 1, StubInfo::name(stub_id));
2243 if (nopush_entry != nullptr) {
2244 *nopush_entry = entries.at(0);
2245 }
2246 return start;
2247 }
2248
2249 Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop;
2250
2251 // Input registers (after setup_arg_regs)
2252 const Register from = c_rarg0; // source array address
2253 const Register to = c_rarg1; // destination array address
2254 const Register count = c_rarg2; // elementscount
2255 const Register ckoff = c_rarg3; // super_check_offset
2256 const Register ckval = c_rarg4; // super_klass
2257
2258 RegSet wb_pre_saved_regs = RegSet::range(c_rarg0, c_rarg4);
2259
2260 // Registers used as temps (r19, r20, r21, r22 are save-on-entry)
2261 const Register copied_oop = r22; // actual oop copied
2262 const Register count_save = r21; // orig elementscount
2263 const Register start_to = r20; // destination array start address
2264 const Register r19_klass = r19; // oop._klass
2265
2266 // Registers used as gc temps (r5, r6, r7 are save-on-call)
2267 const Register gct1 = r5, gct2 = r6, gct3 = r7;
2268
2269 //---------------------------------------------------------------
2270 // Assembler stub will be used for this call to arraycopy
2271 // if the two arrays are subtypes of Object[] but the
2272 // destination array type is not equal to or a supertype
2273 // of the source type. Each element must be separately
2274 // checked.
2275
2276 assert_different_registers(from, to, count, ckoff, ckval, start_to,
2277 copied_oop, r19_klass, count_save, rscratch1);
2278
2279 __ align(CodeEntryAlignment);
2280 StubCodeMark mark(this, stub_id);
2281 start = __ pc();
2282
2283 __ enter(); // required for proper stackwalking of RuntimeStub frame
2284
2285 #ifdef ASSERT
2286 // caller guarantees that the arrays really are different
2287 // otherwise, we would have to make conjoint checks
2288 { Label L;
2289 __ b(L); // conjoint check not yet implemented
2290 __ stop("checkcast_copy within a single array");
2291 __ bind(L);
2292 }
2293 #endif //ASSERT
2294
2295 // Caller of this entry point must set up the argument registers.
2296 if (nopush_entry != nullptr) {
2297 *nopush_entry = __ pc();
2298 entries.append(*nopush_entry);
2299 BLOCK_COMMENT("Entry:");
2300 }
2301
2302 // Empty array: Nothing to do.
2303 __ cbz(count, L_done);
2304 __ push(RegSet::of(r19, r20, r21, r22), sp);
2305
2306 #ifdef ASSERT
2307 BLOCK_COMMENT("assert consistent ckoff/ckval");
2308 // The ckoff and ckval must be mutually consistent,
2309 // even though caller generates both.
2310 { Label L;
2311 int sco_offset = in_bytes(Klass::super_check_offset_offset());
2312 __ ldrw(start_to, Address(ckval, sco_offset));
2313 __ cmpw(ckoff, start_to);
2314 __ br(Assembler::EQ, L);
2315 __ stop("super_check_offset inconsistent");
2316 __ bind(L);
2317 }
2318 #endif //ASSERT
2319
2320 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT;
2321 bool is_oop = true;
2322 int element_size = UseCompressedOops ? 4 : 8;
2323 if (dest_uninitialized) {
2324 decorators |= IS_DEST_UNINITIALIZED;
2325 }
2326
2327 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2328 bs->arraycopy_prologue(_masm, decorators, is_oop, from, to, count, wb_pre_saved_regs);
2329
2330 // save the original count
2331 __ mov(count_save, count);
2332
2333 // Copy from low to high addresses
2334 __ mov(start_to, to); // Save destination array start address
2335 __ b(L_load_element);
2336
2337 // ======== begin loop ========
2338 // (Loop is rotated; its entry is L_load_element.)
2339 // Loop control:
2340 // for (; count != 0; count--) {
2341 // copied_oop = load_heap_oop(from++);
2342 // ... generate_type_check ...;
2343 // store_heap_oop(to++, copied_oop);
2344 // }
2345 __ align(OptoLoopAlignment);
2346
2347 __ BIND(L_store_element);
2348 bs->copy_store_at(_masm, decorators, T_OBJECT, element_size,
2349 __ post(to, element_size), copied_oop, noreg,
2350 gct1, gct2, gct3);
2351 __ sub(count, count, 1);
2352 __ cbz(count, L_do_card_marks);
2353
2354 // ======== loop entry is here ========
2355 __ BIND(L_load_element);
2356 bs->copy_load_at(_masm, decorators, T_OBJECT, element_size,
2357 copied_oop, noreg, __ post(from, element_size),
2358 gct1);
2359 __ cbz(copied_oop, L_store_element);
2360
2361 __ load_klass(r19_klass, copied_oop, rscratch1);// query the object klass
2362
2363 BLOCK_COMMENT("type_check:");
2364 generate_type_check(/*sub_klass*/r19_klass,
2365 /*super_check_offset*/ckoff,
2366 /*super_klass*/ckval,
2367 /*r_array_base*/gct1,
2368 /*temp2*/gct2,
2369 /*result*/r10, L_store_element);
2370
2371 // Fall through on failure!
2372
2373 // ======== end loop ========
2374
2375 // It was a real error; we must depend on the caller to finish the job.
2376 // Register count = remaining oops, count_orig = total oops.
2377 // Emit GC store barriers for the oops we have copied and report
2378 // their number to the caller.
2379
2380 __ subs(count, count_save, count); // K = partially copied oop count
2381 __ eon(count, count, zr); // report (-1^K) to caller
2382 __ br(Assembler::EQ, L_done_pop);
2383
2384 __ BIND(L_do_card_marks);
2385 bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, count_save, rscratch1);
2386
2387 __ bind(L_done_pop);
2388 __ pop(RegSet::of(r19, r20, r21, r22), sp);
2389 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
2390
2391 __ bind(L_done);
2392 __ mov(r0, count);
2393 __ leave();
2394 __ ret(lr);
2395
2396 // record the stub entry and end plus any no_push entry
2397 store_archive_data(stub_id, start, __ pc() , entries_ptr);
2398 return start;
2399 }
2400
2401 // Perform range checks on the proposed arraycopy.
2402 // Kills temp, but nothing else.
2403 // Also, clean the sign bits of src_pos and dst_pos.
2404 void arraycopy_range_checks(Register src, // source array oop (c_rarg0)
2405 Register src_pos, // source position (c_rarg1)
2406 Register dst, // destination array oo (c_rarg2)
2407 Register dst_pos, // destination position (c_rarg3)
2408 Register length,
2409 Register temp,
2410 Label& L_failed) {
2411 BLOCK_COMMENT("arraycopy_range_checks:");
2412
2413 assert_different_registers(rscratch1, temp);
2414
2415 // if (src_pos + length > arrayOop(src)->length()) FAIL;
2416 __ ldrw(rscratch1, Address(src, arrayOopDesc::length_offset_in_bytes()));
2417 __ addw(temp, length, src_pos);
2418 __ cmpw(temp, rscratch1);
2419 __ br(Assembler::HI, L_failed);
2420
2421 // if (dst_pos + length > arrayOop(dst)->length()) FAIL;
2422 __ ldrw(rscratch1, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2423 __ addw(temp, length, dst_pos);
2424 __ cmpw(temp, rscratch1);
2425 __ br(Assembler::HI, L_failed);
2426
2427 // Have to clean up high 32 bits of 'src_pos' and 'dst_pos'.
2428 __ movw(src_pos, src_pos);
2429 __ movw(dst_pos, dst_pos);
2430
2431 BLOCK_COMMENT("arraycopy_range_checks done");
2432 }
2433
2434 // These stubs get called from some dumb test routine.
2435 // I'll write them properly when they're called from
2436 // something that's actually doing something.
2437 static void fake_arraycopy_stub(address src, address dst, int count) {
2438 assert(count == 0, "huh?");
2439 }
2440
2441
2442 //
2443 // Generate 'unsafe' array copy stub
2444 // Though just as safe as the other stubs, it takes an unscaled
2445 // size_t argument instead of an element count.
2446 //
2447 // Input:
2448 // c_rarg0 - source array address
2449 // c_rarg1 - destination array address
2450 // c_rarg2 - byte count, treated as ssize_t, can be zero
2451 //
2452 // Examines the alignment of the operands and dispatches
2453 // to a long, int, short, or byte copy loop.
2454 //
2455 address generate_unsafe_copy(address byte_copy_entry,
2456 address short_copy_entry,
2457 address int_copy_entry,
2458 address long_copy_entry) {
2459 StubId stub_id = StubId::stubgen_unsafe_arraycopy_id;
2460 int entry_count = StubInfo::entry_count(stub_id);
2461 assert(entry_count == 1, "sanity check");
2462 address start = load_archive_data(stub_id);
2463 if (start != nullptr) {
2464 return start;
2465 }
2466 Label L_long_aligned, L_int_aligned, L_short_aligned;
2467 Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
2468
2469 __ align(CodeEntryAlignment);
2470 StubCodeMark mark(this, stub_id);
2471 start = __ pc();
2472 __ enter(); // required for proper stackwalking of RuntimeStub frame
2473
2474 // bump this on entry, not on exit:
2475 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2476
2477 __ orr(rscratch1, s, d);
2478 __ orr(rscratch1, rscratch1, count);
2479
2480 __ andr(rscratch1, rscratch1, BytesPerLong-1);
2481 __ cbz(rscratch1, L_long_aligned);
2482 __ andr(rscratch1, rscratch1, BytesPerInt-1);
2483 __ cbz(rscratch1, L_int_aligned);
2484 __ tbz(rscratch1, 0, L_short_aligned);
2485 __ b(RuntimeAddress(byte_copy_entry));
2486
2487 __ BIND(L_short_aligned);
2488 __ lsr(count, count, LogBytesPerShort); // size => short_count
2489 __ b(RuntimeAddress(short_copy_entry));
2490 __ BIND(L_int_aligned);
2491 __ lsr(count, count, LogBytesPerInt); // size => int_count
2492 __ b(RuntimeAddress(int_copy_entry));
2493 __ BIND(L_long_aligned);
2494 __ lsr(count, count, LogBytesPerLong); // size => long_count
2495 __ b(RuntimeAddress(long_copy_entry));
2496
2497 // record the stub entry and end
2498 store_archive_data(stub_id, start, __ pc());
2499
2500 return start;
2501 }
2502
2503 //
2504 // Generate generic array copy stubs
2505 //
2506 // Input:
2507 // c_rarg0 - src oop
2508 // c_rarg1 - src_pos (32-bits)
2509 // c_rarg2 - dst oop
2510 // c_rarg3 - dst_pos (32-bits)
2511 // c_rarg4 - element count (32-bits)
2512 //
2513 // Output:
2514 // r0 == 0 - success
2515 // r0 == -1^K - failure, where K is partial transfer count
2516 //
2517 address generate_generic_copy(address byte_copy_entry, address short_copy_entry,
2518 address int_copy_entry, address oop_copy_entry,
2519 address long_copy_entry, address checkcast_copy_entry) {
2520 StubId stub_id = StubId::stubgen_generic_arraycopy_id;
2521 int entry_count = StubInfo::entry_count(stub_id);
2522 assert(entry_count == 1, "sanity check");
2523 address start = load_archive_data(stub_id);
2524 if (start != nullptr) {
2525 return start;
2526 }
2527 Label L_failed, L_objArray;
2528 Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
2529
2530 // Input registers
2531 const Register src = c_rarg0; // source array oop
2532 const Register src_pos = c_rarg1; // source position
2533 const Register dst = c_rarg2; // destination array oop
2534 const Register dst_pos = c_rarg3; // destination position
2535 const Register length = c_rarg4;
2536
2537
2538 // Registers used as temps
2539 const Register dst_klass = c_rarg5;
2540
2541 __ align(CodeEntryAlignment);
2542
2543 StubCodeMark mark(this, stub_id);
2544
2545 start = __ pc();
2546
2547 __ enter(); // required for proper stackwalking of RuntimeStub frame
2548
2549 // bump this on entry, not on exit:
2550 inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
2551
2552 //-----------------------------------------------------------------------
2553 // Assembler stub will be used for this call to arraycopy
2554 // if the following conditions are met:
2555 //
2556 // (1) src and dst must not be null.
2557 // (2) src_pos must not be negative.
2558 // (3) dst_pos must not be negative.
2559 // (4) length must not be negative.
2560 // (5) src klass and dst klass should be the same and not null.
2561 // (6) src and dst should be arrays.
2562 // (7) src_pos + length must not exceed length of src.
2563 // (8) dst_pos + length must not exceed length of dst.
2564 //
2565
2566 // if (src == nullptr) return -1;
2567 __ cbz(src, L_failed);
2568
2569 // if (src_pos < 0) return -1;
2570 __ tbnz(src_pos, 31, L_failed); // i.e. sign bit set
2571
2572 // if (dst == nullptr) return -1;
2573 __ cbz(dst, L_failed);
2574
2575 // if (dst_pos < 0) return -1;
2576 __ tbnz(dst_pos, 31, L_failed); // i.e. sign bit set
2577
2578 // registers used as temp
2579 const Register scratch_length = r16; // elements count to copy
2580 const Register scratch_src_klass = r17; // array klass
2581 const Register lh = r15; // layout helper
2582
2583 // if (length < 0) return -1;
2584 __ movw(scratch_length, length); // length (elements count, 32-bits value)
2585 __ tbnz(scratch_length, 31, L_failed); // i.e. sign bit set
2586
2587 __ load_narrow_klass(scratch_src_klass, src);
2588 #ifdef ASSERT
2589 // assert(src->klass() != nullptr);
2590 {
2591 BLOCK_COMMENT("assert klasses not null {");
2592 Label L1, L2;
2593 __ cbnz(scratch_src_klass, L2); // it is broken if klass is null
2594 __ bind(L1);
2595 __ stop("broken null klass");
2596 __ bind(L2);
2597 __ load_narrow_klass(rscratch1, dst);
2598 __ cbz(rscratch1, L1); // this would be broken also
2599 BLOCK_COMMENT("} assert klasses not null done");
2600 }
2601 #endif
2602 __ decode_klass_not_null(scratch_src_klass, scratch_src_klass, rscratch1);
2603
2604 // Load layout helper (32-bits)
2605 //
2606 // |array_tag| | header_size | element_type | |log2_element_size|
2607 // 32 30 24 16 8 2 0
2608 //
2609 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2610 //
2611
2612 const int lh_offset = in_bytes(Klass::layout_helper_offset());
2613
2614 // Handle objArrays completely differently...
2615 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2616 __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2617 __ movw(rscratch1, objArray_lh);
2618 __ eorw(rscratch2, lh, rscratch1);
2619 __ cbzw(rscratch2, L_objArray);
2620
2621 // if (src->klass() != dst->klass()) return -1;
2622 __ load_klass(rscratch2, dst, rscratch1);
2623 __ eor(rscratch2, rscratch2, scratch_src_klass);
2624 __ cbnz(rscratch2, L_failed);
2625
2626 // Check for flat inline type array -> return -1
2627 __ test_flat_array_oop(src, rscratch2, L_failed);
2628
2629 // Check for null-free (non-flat) inline type array -> handle as object array
2630 __ test_null_free_array_oop(src, rscratch2, L_objArray);
2631
2632 // if (!src->is_Array()) return -1;
2633 __ tbz(lh, 31, L_failed); // i.e. (lh >= 0)
2634
2635 // At this point, it is known to be a typeArray (array_tag 0x3).
2636 #ifdef ASSERT
2637 {
2638 BLOCK_COMMENT("assert primitive array {");
2639 Label L;
2640 __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2641 __ cmpw(lh, rscratch2);
2642 __ br(Assembler::GE, L);
2643 __ stop("must be a primitive array");
2644 __ bind(L);
2645 BLOCK_COMMENT("} assert primitive array done");
2646 }
2647 #endif
2648
2649 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2650 rscratch2, L_failed);
2651
2652 // TypeArrayKlass
2653 //
2654 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2655 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2656 //
2657
2658 const Register rscratch1_offset = rscratch1; // array offset
2659 const Register r15_elsize = lh; // element size
2660
2661 __ ubfx(rscratch1_offset, lh, Klass::_lh_header_size_shift,
2662 exact_log2(Klass::_lh_header_size_mask+1)); // array_offset
2663 __ add(src, src, rscratch1_offset); // src array offset
2664 __ add(dst, dst, rscratch1_offset); // dst array offset
2665 BLOCK_COMMENT("choose copy loop based on element size");
2666
2667 // next registers should be set before the jump to corresponding stub
2668 const Register from = c_rarg0; // source array address
2669 const Register to = c_rarg1; // destination array address
2670 const Register count = c_rarg2; // elements count
2671
2672 // 'from', 'to', 'count' registers should be set in such order
2673 // since they are the same as 'src', 'src_pos', 'dst'.
2674
2675 assert(Klass::_lh_log2_element_size_shift == 0, "fix this code");
2676
2677 // The possible values of elsize are 0-3, i.e. exact_log2(element
2678 // size in bytes). We do a simple bitwise binary search.
2679 __ BIND(L_copy_bytes);
2680 __ tbnz(r15_elsize, 1, L_copy_ints);
2681 __ tbnz(r15_elsize, 0, L_copy_shorts);
2682 __ lea(from, Address(src, src_pos));// src_addr
2683 __ lea(to, Address(dst, dst_pos));// dst_addr
2684 __ movw(count, scratch_length); // length
2685 __ b(RuntimeAddress(byte_copy_entry));
2686
2687 __ BIND(L_copy_shorts);
2688 __ lea(from, Address(src, src_pos, Address::lsl(1)));// src_addr
2689 __ lea(to, Address(dst, dst_pos, Address::lsl(1)));// dst_addr
2690 __ movw(count, scratch_length); // length
2691 __ b(RuntimeAddress(short_copy_entry));
2692
2693 __ BIND(L_copy_ints);
2694 __ tbnz(r15_elsize, 0, L_copy_longs);
2695 __ lea(from, Address(src, src_pos, Address::lsl(2)));// src_addr
2696 __ lea(to, Address(dst, dst_pos, Address::lsl(2)));// dst_addr
2697 __ movw(count, scratch_length); // length
2698 __ b(RuntimeAddress(int_copy_entry));
2699
2700 __ BIND(L_copy_longs);
2701 #ifdef ASSERT
2702 {
2703 BLOCK_COMMENT("assert long copy {");
2704 Label L;
2705 __ andw(lh, lh, Klass::_lh_log2_element_size_mask); // lh -> r15_elsize
2706 __ cmpw(r15_elsize, LogBytesPerLong);
2707 __ br(Assembler::EQ, L);
2708 __ stop("must be long copy, but elsize is wrong");
2709 __ bind(L);
2710 BLOCK_COMMENT("} assert long copy done");
2711 }
2712 #endif
2713 __ lea(from, Address(src, src_pos, Address::lsl(3)));// src_addr
2714 __ lea(to, Address(dst, dst_pos, Address::lsl(3)));// dst_addr
2715 __ movw(count, scratch_length); // length
2716 __ b(RuntimeAddress(long_copy_entry));
2717
2718 // ObjArrayKlass
2719 __ BIND(L_objArray);
2720 // live at this point: scratch_src_klass, scratch_length, src[_pos], dst[_pos]
2721
2722 Label L_plain_copy, L_checkcast_copy;
2723 // test array classes for subtyping
2724 __ load_klass(r15, dst, rscratch1);
2725 __ cmp(scratch_src_klass, r15); // usual case is exact equality
2726 __ br(Assembler::NE, L_checkcast_copy);
2727
2728 // Identically typed arrays can be copied without element-wise checks.
2729 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2730 rscratch2, L_failed);
2731
2732 __ lea(from, Address(src, src_pos, Address::lsl(LogBytesPerHeapOop)));
2733 __ add(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2734 __ lea(to, Address(dst, dst_pos, Address::lsl(LogBytesPerHeapOop)));
2735 __ add(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2736 __ movw(count, scratch_length); // length
2737 __ BIND(L_plain_copy);
2738 __ b(RuntimeAddress(oop_copy_entry));
2739
2740 __ BIND(L_checkcast_copy);
2741 // live at this point: scratch_src_klass, scratch_length, r15 (dst_klass)
2742 {
2743 // Before looking at dst.length, make sure dst is also an objArray.
2744 __ ldrw(rscratch1, Address(r15, lh_offset));
2745 __ movw(rscratch2, objArray_lh);
2746 __ eorw(rscratch1, rscratch1, rscratch2);
2747 __ cbnzw(rscratch1, L_failed);
2748
2749 // It is safe to examine both src.length and dst.length.
2750 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2751 r15, L_failed);
2752
2753 __ load_klass(dst_klass, dst, rscratch1); // reload
2754
2755 // Marshal the base address arguments now, freeing registers.
2756 __ lea(from, Address(src, src_pos, Address::lsl(LogBytesPerHeapOop)));
2757 __ add(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2758 __ lea(to, Address(dst, dst_pos, Address::lsl(LogBytesPerHeapOop)));
2759 __ add(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2760 __ movw(count, length); // length (reloaded)
2761 Register sco_temp = c_rarg3; // this register is free now
2762 assert_different_registers(from, to, count, sco_temp,
2763 dst_klass, scratch_src_klass);
2764 // assert_clean_int(count, sco_temp);
2765
2766 // Generate the type check.
2767 const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2768 __ ldrw(sco_temp, Address(dst_klass, sco_offset));
2769
2770 // Smashes rscratch1, rscratch2
2771 generate_type_check(scratch_src_klass, sco_temp, dst_klass, /*temps*/ noreg, noreg, noreg,
2772 L_plain_copy);
2773
2774 // Fetch destination element klass from the ObjArrayKlass header.
2775 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2776 __ ldr(dst_klass, Address(dst_klass, ek_offset));
2777 __ ldrw(sco_temp, Address(dst_klass, sco_offset));
2778
2779 // the checkcast_copy loop needs two extra arguments:
2780 assert(c_rarg3 == sco_temp, "#3 already in place");
2781 // Set up arguments for checkcast_copy_entry.
2782 __ mov(c_rarg4, dst_klass); // dst.klass.element_klass
2783 __ b(RuntimeAddress(checkcast_copy_entry));
2784 }
2785
2786 __ BIND(L_failed);
2787 __ mov(r0, -1);
2788 __ leave(); // required for proper stackwalking of RuntimeStub frame
2789 __ ret(lr);
2790
2791 // record the stub entry and end
2792 store_archive_data(stub_id, start, __ pc());
2793
2794 return start;
2795 }
2796
2797 //
2798 // Generate stub for array fill. If "aligned" is true, the
2799 // "to" address is assumed to be heapword aligned.
2800 //
2801 // Arguments for generated stub:
2802 // to: c_rarg0
2803 // value: c_rarg1
2804 // count: c_rarg2 treated as signed
2805 //
2806 address generate_fill(StubId stub_id) {
2807 BasicType t;
2808 bool aligned;
2809
2810 switch (stub_id) {
2811 case StubId::stubgen_jbyte_fill_id:
2812 t = T_BYTE;
2813 aligned = false;
2814 break;
2815 case StubId::stubgen_jshort_fill_id:
2816 t = T_SHORT;
2817 aligned = false;
2818 break;
2819 case StubId::stubgen_jint_fill_id:
2820 t = T_INT;
2821 aligned = false;
2822 break;
2823 case StubId::stubgen_arrayof_jbyte_fill_id:
2824 t = T_BYTE;
2825 aligned = true;
2826 break;
2827 case StubId::stubgen_arrayof_jshort_fill_id:
2828 t = T_SHORT;
2829 aligned = true;
2830 break;
2831 case StubId::stubgen_arrayof_jint_fill_id:
2832 t = T_INT;
2833 aligned = true;
2834 break;
2835 default:
2836 ShouldNotReachHere();
2837 };
2838 int entry_count = StubInfo::entry_count(stub_id);
2839 assert(entry_count == 1, "sanity check");
2840 address start = load_archive_data(stub_id);
2841 if (start != nullptr) {
2842 return start;
2843 }
2844 __ align(CodeEntryAlignment);
2845 StubCodeMark mark(this, stub_id);
2846 start = __ pc();
2847
2848 BLOCK_COMMENT("Entry:");
2849
2850 const Register to = c_rarg0; // source array address
2851 const Register value = c_rarg1; // value
2852 const Register count = c_rarg2; // elements count
2853
2854 const Register bz_base = r10; // base for block_zero routine
2855 const Register cnt_words = r11; // temp register
2856
2857 __ enter();
2858
2859 Label L_fill_elements, L_exit1;
2860
2861 int shift = -1;
2862 switch (t) {
2863 case T_BYTE:
2864 shift = 0;
2865 __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
2866 __ bfi(value, value, 8, 8); // 8 bit -> 16 bit
2867 __ bfi(value, value, 16, 16); // 16 bit -> 32 bit
2868 __ br(Assembler::LO, L_fill_elements);
2869 break;
2870 case T_SHORT:
2871 shift = 1;
2872 __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
2873 __ bfi(value, value, 16, 16); // 16 bit -> 32 bit
2874 __ br(Assembler::LO, L_fill_elements);
2875 break;
2876 case T_INT:
2877 shift = 2;
2878 __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
2879 __ br(Assembler::LO, L_fill_elements);
2880 break;
2881 default: ShouldNotReachHere();
2882 }
2883
2884 // Align source address at 8 bytes address boundary.
2885 Label L_skip_align1, L_skip_align2, L_skip_align4;
2886 if (!aligned) {
2887 switch (t) {
2888 case T_BYTE:
2889 // One byte misalignment happens only for byte arrays.
2890 __ tbz(to, 0, L_skip_align1);
2891 __ strb(value, Address(__ post(to, 1)));
2892 __ subw(count, count, 1);
2893 __ bind(L_skip_align1);
2894 // Fallthrough
2895 case T_SHORT:
2896 // Two bytes misalignment happens only for byte and short (char) arrays.
2897 __ tbz(to, 1, L_skip_align2);
2898 __ strh(value, Address(__ post(to, 2)));
2899 __ subw(count, count, 2 >> shift);
2900 __ bind(L_skip_align2);
2901 // Fallthrough
2902 case T_INT:
2903 // Align to 8 bytes, we know we are 4 byte aligned to start.
2904 __ tbz(to, 2, L_skip_align4);
2905 __ strw(value, Address(__ post(to, 4)));
2906 __ subw(count, count, 4 >> shift);
2907 __ bind(L_skip_align4);
2908 break;
2909 default: ShouldNotReachHere();
2910 }
2911 }
2912
2913 //
2914 // Fill large chunks
2915 //
2916 __ lsrw(cnt_words, count, 3 - shift); // number of words
2917 __ bfi(value, value, 32, 32); // 32 bit -> 64 bit
2918 __ subw(count, count, cnt_words, Assembler::LSL, 3 - shift);
2919 if (UseBlockZeroing) {
2920 Label non_block_zeroing, rest;
2921 // If the fill value is zero we can use the fast zero_words().
2922 __ cbnz(value, non_block_zeroing);
2923 __ mov(bz_base, to);
2924 __ add(to, to, cnt_words, Assembler::LSL, LogBytesPerWord);
2925 address tpc = __ zero_words(bz_base, cnt_words);
2926 if (tpc == nullptr) {
2927 fatal("CodeCache is full at generate_fill");
2928 }
2929 __ b(rest);
2930 __ bind(non_block_zeroing);
2931 __ fill_words(to, cnt_words, value);
2932 __ bind(rest);
2933 } else {
2934 __ fill_words(to, cnt_words, value);
2935 }
2936
2937 // Remaining count is less than 8 bytes. Fill it by a single store.
2938 // Note that the total length is no less than 8 bytes.
2939 if (t == T_BYTE || t == T_SHORT) {
2940 Label L_exit1;
2941 __ cbzw(count, L_exit1);
2942 __ add(to, to, count, Assembler::LSL, shift); // points to the end
2943 __ str(value, Address(to, -8)); // overwrite some elements
2944 __ bind(L_exit1);
2945 __ leave();
2946 __ ret(lr);
2947 }
2948
2949 // Handle copies less than 8 bytes.
2950 Label L_fill_2, L_fill_4, L_exit2;
2951 __ bind(L_fill_elements);
2952 switch (t) {
2953 case T_BYTE:
2954 __ tbz(count, 0, L_fill_2);
2955 __ strb(value, Address(__ post(to, 1)));
2956 __ bind(L_fill_2);
2957 __ tbz(count, 1, L_fill_4);
2958 __ strh(value, Address(__ post(to, 2)));
2959 __ bind(L_fill_4);
2960 __ tbz(count, 2, L_exit2);
2961 __ strw(value, Address(to));
2962 break;
2963 case T_SHORT:
2964 __ tbz(count, 0, L_fill_4);
2965 __ strh(value, Address(__ post(to, 2)));
2966 __ bind(L_fill_4);
2967 __ tbz(count, 1, L_exit2);
2968 __ strw(value, Address(to));
2969 break;
2970 case T_INT:
2971 __ cbzw(count, L_exit2);
2972 __ strw(value, Address(to));
2973 break;
2974 default: ShouldNotReachHere();
2975 }
2976 __ bind(L_exit2);
2977 __ leave();
2978 __ ret(lr);
2979
2980 // record the stub entry and end
2981 store_archive_data(stub_id, start, __ pc());
2982
2983 return start;
2984 }
2985
2986 address generate_unsafecopy_common_error_exit() {
2987 StubId stub_id = StubId::stubgen_unsafecopy_common_id;
2988 int entry_count = StubInfo::entry_count(stub_id);
2989 assert(entry_count == 1, "sanity check");
2990 address start = load_archive_data(stub_id);
2991 if (start != nullptr) {
2992 return start;
2993 }
2994 __ align(CodeEntryAlignment);
2995 StubCodeMark mark(this, stub_id);
2996 start = __ pc();
2997 __ leave();
2998 __ mov(r0, 0);
2999 __ ret(lr);
3000
3001 // record the stub entry and end
3002 store_archive_data(stub_id, start, __ pc());
3003
3004 return start;
3005 }
3006
3007 //
3008 // Generate 'unsafe' set memory stub
3009 // Though just as safe as the other stubs, it takes an unscaled
3010 // size_t (# bytes) argument instead of an element count.
3011 //
3012 // This fill operation is atomicity preserving: as long as the
3013 // address supplied is sufficiently aligned, all writes of up to 64
3014 // bits in size are single-copy atomic.
3015 //
3016 // Input:
3017 // c_rarg0 - destination array address
3018 // c_rarg1 - byte count (size_t)
3019 // c_rarg2 - byte value
3020 //
3021 address generate_unsafe_setmemory() {
3022 StubId stub_id = StubId::stubgen_unsafe_setmemory_id;
3023 int entry_count = StubInfo::entry_count(stub_id);
3024 assert(entry_count == 1, "sanity check");
3025 // we expect one set of extra unsafememory access handler entries
3026 GrowableArray<address> extras;
3027 int extra_count = 1 * UnsafeMemoryAccess::COLUMN_COUNT;
3028 address start = load_archive_data(stub_id, nullptr, &extras);
3029 if (start != nullptr) {
3030 assert(extras.length() == extra_count,
3031 "unexpected extra entry count %d", extras.length());
3032 register_unsafe_access_handlers(extras, 0, 1);
3033 return start;
3034 }
3035
3036 __ align(CodeEntryAlignment);
3037 StubCodeMark mark(this, stub_id);
3038 start = __ pc();
3039
3040 Register dest = c_rarg0, count = c_rarg1, value = c_rarg2;
3041 Label tail;
3042
3043 {
3044 UnsafeMemoryAccessMark umam(this, true, false);
3045
3046 __ enter(); // required for proper stackwalking of RuntimeStub frame
3047
3048 __ dup(v0, __ T16B, value);
3049
3050 if (AvoidUnalignedAccesses) {
3051 __ cmp(count, (u1)16);
3052 __ br(__ LO, tail);
3053
3054 __ mov(rscratch1, 16);
3055 __ andr(rscratch2, dest, 15);
3056 __ sub(rscratch1, rscratch1, rscratch2); // Bytes needed to 16-align dest
3057 __ strq(v0, Address(dest));
3058 __ sub(count, count, rscratch1);
3059 __ add(dest, dest, rscratch1);
3060 }
3061
3062 __ subs(count, count, (u1)64);
3063 __ br(__ LO, tail);
3064 {
3065 Label again;
3066 __ bind(again);
3067 __ stpq(v0, v0, Address(dest));
3068 __ stpq(v0, v0, Address(dest, 32));
3069
3070 __ subs(count, count, 64);
3071 __ add(dest, dest, 64);
3072 __ br(__ HS, again);
3073 }
3074
3075 __ bind(tail);
3076 // The count of bytes is off by 64, but we don't need to correct
3077 // it because we're only going to use the least-significant few
3078 // count bits from here on.
3079 // __ add(count, count, 64);
3080
3081 {
3082 Label dont;
3083 __ tbz(count, exact_log2(32), dont);
3084 __ stpq(v0, v0, __ post(dest, 32));
3085 __ bind(dont);
3086 }
3087 {
3088 Label dont;
3089 __ tbz(count, exact_log2(16), dont);
3090 __ strq(v0, __ post(dest, 16));
3091 __ bind(dont);
3092 }
3093 {
3094 Label dont;
3095 __ tbz(count, exact_log2(8), dont);
3096 __ strd(v0, __ post(dest, 8));
3097 __ bind(dont);
3098 }
3099
3100 Label finished;
3101 __ tst(count, 7);
3102 __ br(__ EQ, finished);
3103
3104 {
3105 Label dont;
3106 __ tbz(count, exact_log2(4), dont);
3107 __ strs(v0, __ post(dest, 4));
3108 __ bind(dont);
3109 }
3110 {
3111 Label dont;
3112 __ tbz(count, exact_log2(2), dont);
3113 __ bfi(value, value, 8, 8);
3114 __ strh(value, __ post(dest, 2));
3115 __ bind(dont);
3116 }
3117 {
3118 Label dont;
3119 __ tbz(count, exact_log2(1), dont);
3120 __ strb(value, Address(dest));
3121 __ bind(dont);
3122 }
3123
3124 __ bind(finished);
3125 __ leave();
3126 __ ret(lr);
3127 // have to exit the block and destroy the UnsafeMemoryAccessMark
3128 // in order to retrieve the handler end address
3129 }
3130
3131 // install saved handler addresses in extras
3132 address end = __ pc();
3133 retrieve_unsafe_access_handlers(start, end, extras);
3134 assert(extras.length() == extra_count,
3135 "incorrect handlers count %d", extras.length());
3136 // record the stub entry and end plus the extras
3137 store_archive_data(stub_id, start, end, nullptr, &extras);
3138
3139 return start;
3140 }
3141
3142 address generate_data_cache_writeback() {
3143 const Register line = c_rarg0; // address of line to write back
3144
3145 StubId stub_id = StubId::stubgen_data_cache_writeback_id;
3146 int entry_count = StubInfo::entry_count(stub_id);
3147 assert(entry_count == 1, "sanity check");
3148 address start = load_archive_data(stub_id);
3149 if (start != nullptr) {
3150 return start;
3151 }
3152 __ align(CodeEntryAlignment);
3153 StubCodeMark mark(this, stub_id);
3154
3155 start = __ pc();
3156 __ enter();
3157 __ cache_wb(Address(line, 0));
3158 __ leave();
3159 __ ret(lr);
3160
3161 // record the stub entry and end
3162 store_archive_data(stub_id, start, __ pc());
3163
3164 return start;
3165 }
3166
3167 address generate_data_cache_writeback_sync() {
3168 StubId stub_id = StubId::stubgen_data_cache_writeback_sync_id;
3169 int entry_count = StubInfo::entry_count(stub_id);
3170 assert(entry_count == 1, "sanity check");
3171 address start = load_archive_data(stub_id);
3172 if (start != nullptr) {
3173 return start;
3174 }
3175 const Register is_pre = c_rarg0; // pre or post sync
3176 __ align(CodeEntryAlignment);
3177 StubCodeMark mark(this, stub_id);
3178
3179 // pre wbsync is a no-op
3180 // post wbsync translates to an sfence
3181
3182 Label skip;
3183 start = __ pc();
3184 __ enter();
3185 __ cbnz(is_pre, skip);
3186 __ cache_wbsync(false);
3187 __ bind(skip);
3188 __ leave();
3189 __ ret(lr);
3190
3191 // record the stub entry and end
3192 store_archive_data(stub_id, start, __ pc());
3193
3194 return start;
3195 }
3196
3197 void generate_arraycopy_stubs() {
3198 // Some copy stubs publish a normal entry and then a 2nd 'fallback'
3199 // entry immediately following their stack push. This can be used
3200 // as a post-push branch target for compatible stubs when they
3201 // identify a special case that can be handled by the fallback
3202 // stub e.g a disjoint copy stub may be use as a special case
3203 // fallback for its compatible conjoint copy stub.
3204 //
3205 // A no push entry is always returned in the following local and
3206 // then published by assigning to the appropriate entry field in
3207 // class StubRoutines. The entry value is then passed to the
3208 // generator for the compatible stub. That means the entry must be
3209 // listed when saving to/restoring from the AOT cache, ensuring
3210 // that the inter-stub jumps are noted at AOT-cache save and
3211 // relocated at AOT cache load.
3212 address nopush_entry;
3213
3214 // generate the common exit first so later stubs can rely on it if
3215 // they want an UnsafeMemoryAccess exit non-local to the stub
3216 StubRoutines::_unsafecopy_common_exit = generate_unsafecopy_common_error_exit();
3217 // register the stub as the default exit with class UnsafeMemoryAccess
3218 UnsafeMemoryAccess::set_common_exit_stub_pc(StubRoutines::_unsafecopy_common_exit);
3219
3220 // generate and publish arch64-specific bulk copy routines first
3221 // so we can call them from other copy stubs
3222 StubRoutines::aarch64::_copy_byte_f = generate_copy_longs(StubId::stubgen_copy_byte_f_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
3223 StubRoutines::aarch64::_copy_byte_b = generate_copy_longs(StubId::stubgen_copy_byte_b_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
3224
3225 StubRoutines::aarch64::_copy_oop_f = generate_copy_longs(StubId::stubgen_copy_oop_f_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
3226 StubRoutines::aarch64::_copy_oop_b = generate_copy_longs(StubId::stubgen_copy_oop_b_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
3227
3228 StubRoutines::aarch64::_copy_oop_uninit_f = generate_copy_longs(StubId::stubgen_copy_oop_uninit_f_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, r0, r1, r15);
3229 StubRoutines::aarch64::_copy_oop_uninit_b = generate_copy_longs(StubId::stubgen_copy_oop_uninit_b_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, r0, r1, r15);
3230
3231 StubRoutines::aarch64::_zero_blocks = generate_zero_blocks();
3232
3233 //*** jbyte
3234 // Always need aligned and unaligned versions
3235 StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &nopush_entry);
3236 // disjoint nopush entry is needed by conjoint copy
3237 StubRoutines::_jbyte_disjoint_arraycopy_nopush = nopush_entry;
3238 StubRoutines::_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, StubRoutines::_jbyte_disjoint_arraycopy_nopush, &nopush_entry);
3239 // conjoint nopush entry is needed by generic/unsafe copy
3240 StubRoutines::_jbyte_arraycopy_nopush = nopush_entry;
3241 StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &nopush_entry);
3242 // disjoint arrayof nopush entry is needed by conjoint copy
3243 StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush = nopush_entry;
3244 StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush, nullptr);
3245
3246 //*** jshort
3247 // Always need aligned and unaligned versions
3248 StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &nopush_entry);
3249 // disjoint nopush entry is needed by conjoint copy
3250 StubRoutines::_jshort_disjoint_arraycopy_nopush = nopush_entry;
3251 StubRoutines::_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, StubRoutines::_jshort_disjoint_arraycopy_nopush, &nopush_entry);
3252 // conjoint nopush entry is used by generic/unsafe copy
3253 StubRoutines::_jshort_arraycopy_nopush = nopush_entry;
3254 StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &nopush_entry);
3255 // disjoint arrayof nopush entry is needed by conjoint copy
3256 StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush = nopush_entry;
3257 StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush, nullptr);
3258
3259 //*** jint
3260 // Aligned versions
3261 StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &nopush_entry);
3262 // disjoint arrayof nopush entry is needed by conjoint copy
3263 StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush = nopush_entry;
3264 StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush, nullptr);
3265 // In 64 bit we need both aligned and unaligned versions of jint arraycopy.
3266 // jint_arraycopy_nopush always points to the unaligned version
3267 StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &nopush_entry);
3268 // disjoint nopush entry is needed by conjoint copy
3269 StubRoutines::_jint_disjoint_arraycopy_nopush = nopush_entry;
3270 StubRoutines::_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, StubRoutines::_jint_disjoint_arraycopy_nopush, &nopush_entry);
3271 // conjoint nopush entry is needed by generic/unsafe copy
3272 StubRoutines::_jint_arraycopy_nopush = nopush_entry;
3273
3274 //*** jlong
3275 // It is always aligned
3276 StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &nopush_entry);
3277 // disjoint arrayof nopush entry is needed by conjoint copy
3278 StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush = nopush_entry;
3279 StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush, &nopush_entry);
3280 // conjoint nopush entry is needed by generic/unsafe copy
3281 StubRoutines::_jlong_arraycopy_nopush = nopush_entry;
3282 // disjoint normal/nopush and conjoint normal entries are not
3283 // generated since the arrayof versions are the same
3284 StubRoutines::_jlong_disjoint_arraycopy = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
3285 StubRoutines::_jlong_disjoint_arraycopy_nopush = StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush;
3286 StubRoutines::_jlong_arraycopy = StubRoutines::_arrayof_jlong_arraycopy;
3287
3288 //*** oops
3289 {
3290 StubRoutines::_arrayof_oop_disjoint_arraycopy
3291 = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &nopush_entry);
3292 // disjoint arrayof nopush entry is needed by conjoint copy
3293 StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush = nopush_entry;
3294 StubRoutines::_arrayof_oop_arraycopy
3295 = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush, &nopush_entry);
3296 // conjoint arrayof nopush entry is needed by generic/unsafe copy
3297 StubRoutines::_oop_arraycopy_nopush = nopush_entry;
3298 // Aligned versions without pre-barriers
3299 StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit
3300 = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
3301 // disjoint arrayof+uninit nopush entry is needed by conjoint copy
3302 StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush = nopush_entry;
3303 // note that we don't need a returned nopush entry because the
3304 // generic/unsafe copy does not cater for uninit arrays.
3305 StubRoutines::_arrayof_oop_arraycopy_uninit
3306 = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush, nullptr);
3307 }
3308
3309 // for oop copies reuse arrayof entries for non-arrayof cases
3310 StubRoutines::_oop_disjoint_arraycopy = StubRoutines::_arrayof_oop_disjoint_arraycopy;
3311 StubRoutines::_oop_disjoint_arraycopy_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush;
3312 StubRoutines::_oop_arraycopy = StubRoutines::_arrayof_oop_arraycopy;
3313 StubRoutines::_oop_disjoint_arraycopy_uninit = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
3314 StubRoutines::_oop_disjoint_arraycopy_uninit_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush;
3315 StubRoutines::_oop_arraycopy_uninit = StubRoutines::_arrayof_oop_arraycopy_uninit;
3316
3317 StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &nopush_entry);
3318 // checkcast nopush entry is needed by generic copy
3319 StubRoutines::_checkcast_arraycopy_nopush = nopush_entry;
3320 // note that we don't need a returned nopush entry because the
3321 // generic copy does not cater for uninit arrays.
3322 StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_uninit_id, nullptr);
3323
3324 // unsafe arraycopy may fallback on conjoint stubs
3325 StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(StubRoutines::_jbyte_arraycopy_nopush,
3326 StubRoutines::_jshort_arraycopy_nopush,
3327 StubRoutines::_jint_arraycopy_nopush,
3328 StubRoutines::_jlong_arraycopy_nopush);
3329
3330 // generic arraycopy may fallback on conjoint stubs
3331 StubRoutines::_generic_arraycopy = generate_generic_copy(StubRoutines::_jbyte_arraycopy_nopush,
3332 StubRoutines::_jshort_arraycopy_nopush,
3333 StubRoutines::_jint_arraycopy_nopush,
3334 StubRoutines::_oop_arraycopy_nopush,
3335 StubRoutines::_jlong_arraycopy_nopush,
3336 StubRoutines::_checkcast_arraycopy_nopush);
3337
3338 StubRoutines::_jbyte_fill = generate_fill(StubId::stubgen_jbyte_fill_id);
3339 StubRoutines::_jshort_fill = generate_fill(StubId::stubgen_jshort_fill_id);
3340 StubRoutines::_jint_fill = generate_fill(StubId::stubgen_jint_fill_id);
3341 StubRoutines::_arrayof_jbyte_fill = generate_fill(StubId::stubgen_arrayof_jbyte_fill_id);
3342 StubRoutines::_arrayof_jshort_fill = generate_fill(StubId::stubgen_arrayof_jshort_fill_id);
3343 StubRoutines::_arrayof_jint_fill = generate_fill(StubId::stubgen_arrayof_jint_fill_id);
3344 }
3345
3346 void generate_math_stubs() { Unimplemented(); }
3347
3348 // Arguments:
3349 //
3350 // Inputs:
3351 // c_rarg0 - source byte array address
3352 // c_rarg1 - destination byte array address
3353 // c_rarg2 - sessionKe (key) in little endian int array
3354 //
3355 address generate_aescrypt_encryptBlock() {
3356 assert(UseAES, "need AES cryptographic extension support");
3357 StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id;
3358 int entry_count = StubInfo::entry_count(stub_id);
3359 assert(entry_count == 1, "sanity check");
3360 address start = load_archive_data(stub_id);
3361 if (start != nullptr) {
3362 return start;
3363 }
3364 __ align(CodeEntryAlignment);
3365 StubCodeMark mark(this, stub_id);
3366
3367 const Register from = c_rarg0; // source array address
3368 const Register to = c_rarg1; // destination array address
3369 const Register key = c_rarg2; // key array address
3370 const Register keylen = rscratch1;
3371
3372 start = __ pc();
3373 __ enter();
3374
3375 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3376
3377 __ aesenc_loadkeys(key, keylen);
3378 __ aesecb_encrypt(from, to, keylen);
3379
3380 __ mov(r0, 0);
3381
3382 __ leave();
3383 __ ret(lr);
3384
3385 // record the stub entry and end
3386 store_archive_data(stub_id, start, __ pc());
3387
3388 return start;
3389 }
3390
3391 // Arguments:
3392 //
3393 // Inputs:
3394 // c_rarg0 - source byte array address
3395 // c_rarg1 - destination byte array address
3396 // c_rarg2 - sessionKd (key) in little endian int array
3397 //
3398 address generate_aescrypt_decryptBlock() {
3399 assert(UseAES, "need AES cryptographic extension support");
3400 StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id;
3401 int entry_count = StubInfo::entry_count(stub_id);
3402 assert(entry_count == 1, "sanity check");
3403 address start = load_archive_data(stub_id);
3404 if (start != nullptr) {
3405 return start;
3406 }
3407 __ align(CodeEntryAlignment);
3408 StubCodeMark mark(this, stub_id);
3409 Label L_doLast;
3410
3411 const Register from = c_rarg0; // source array address
3412 const Register to = c_rarg1; // destination array address
3413 const Register key = c_rarg2; // key array address
3414 const Register keylen = rscratch1;
3415
3416 start = __ pc();
3417 __ enter(); // required for proper stackwalking of RuntimeStub frame
3418
3419 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3420
3421 __ aesecb_decrypt(from, to, key, keylen);
3422
3423 __ mov(r0, 0);
3424
3425 __ leave();
3426 __ ret(lr);
3427
3428 // record the stub entry and end
3429 store_archive_data(stub_id, start, __ pc());
3430
3431 return start;
3432 }
3433
3434 // Arguments:
3435 //
3436 // Inputs:
3437 // c_rarg0 - source byte array address
3438 // c_rarg1 - destination byte array address
3439 // c_rarg2 - sessionKe (key) in little endian int array
3440 // c_rarg3 - r vector byte array address
3441 // c_rarg4 - input length
3442 //
3443 // Output:
3444 // x0 - input length
3445 //
3446 address generate_cipherBlockChaining_encryptAESCrypt() {
3447 assert(UseAES, "need AES cryptographic extension support");
3448 StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id;
3449 int entry_count = StubInfo::entry_count(stub_id);
3450 assert(entry_count == 1, "sanity check");
3451 address start = load_archive_data(stub_id);
3452 if (start != nullptr) {
3453 return start;
3454 }
3455 __ align(CodeEntryAlignment);
3456 StubCodeMark mark(this, stub_id);
3457
3458 Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52;
3459
3460 const Register from = c_rarg0; // source array address
3461 const Register to = c_rarg1; // destination array address
3462 const Register key = c_rarg2; // key array address
3463 const Register rvec = c_rarg3; // r byte array initialized from initvector array address
3464 // and left with the results of the last encryption block
3465 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
3466 const Register keylen = rscratch1;
3467
3468 start = __ pc();
3469
3470 __ enter();
3471
3472 __ movw(rscratch2, len_reg);
3473
3474 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3475
3476 __ ld1(v0, __ T16B, rvec);
3477
3478 __ cmpw(keylen, 52);
3479 __ br(Assembler::CC, L_loadkeys_44);
3480 __ br(Assembler::EQ, L_loadkeys_52);
3481
3482 __ ld1(v17, v18, __ T16B, __ post(key, 32));
3483 __ rev32(v17, __ T16B, v17);
3484 __ rev32(v18, __ T16B, v18);
3485 __ BIND(L_loadkeys_52);
3486 __ ld1(v19, v20, __ T16B, __ post(key, 32));
3487 __ rev32(v19, __ T16B, v19);
3488 __ rev32(v20, __ T16B, v20);
3489 __ BIND(L_loadkeys_44);
3490 __ ld1(v21, v22, v23, v24, __ T16B, __ post(key, 64));
3491 __ rev32(v21, __ T16B, v21);
3492 __ rev32(v22, __ T16B, v22);
3493 __ rev32(v23, __ T16B, v23);
3494 __ rev32(v24, __ T16B, v24);
3495 __ ld1(v25, v26, v27, v28, __ T16B, __ post(key, 64));
3496 __ rev32(v25, __ T16B, v25);
3497 __ rev32(v26, __ T16B, v26);
3498 __ rev32(v27, __ T16B, v27);
3499 __ rev32(v28, __ T16B, v28);
3500 __ ld1(v29, v30, v31, __ T16B, key);
3501 __ rev32(v29, __ T16B, v29);
3502 __ rev32(v30, __ T16B, v30);
3503 __ rev32(v31, __ T16B, v31);
3504
3505 __ BIND(L_aes_loop);
3506 __ ld1(v1, __ T16B, __ post(from, 16));
3507 __ eor(v0, __ T16B, v0, v1);
3508
3509 __ br(Assembler::CC, L_rounds_44);
3510 __ br(Assembler::EQ, L_rounds_52);
3511
3512 __ aese(v0, v17); __ aesmc(v0, v0);
3513 __ aese(v0, v18); __ aesmc(v0, v0);
3514 __ BIND(L_rounds_52);
3515 __ aese(v0, v19); __ aesmc(v0, v0);
3516 __ aese(v0, v20); __ aesmc(v0, v0);
3517 __ BIND(L_rounds_44);
3518 __ aese(v0, v21); __ aesmc(v0, v0);
3519 __ aese(v0, v22); __ aesmc(v0, v0);
3520 __ aese(v0, v23); __ aesmc(v0, v0);
3521 __ aese(v0, v24); __ aesmc(v0, v0);
3522 __ aese(v0, v25); __ aesmc(v0, v0);
3523 __ aese(v0, v26); __ aesmc(v0, v0);
3524 __ aese(v0, v27); __ aesmc(v0, v0);
3525 __ aese(v0, v28); __ aesmc(v0, v0);
3526 __ aese(v0, v29); __ aesmc(v0, v0);
3527 __ aese(v0, v30);
3528 __ eor(v0, __ T16B, v0, v31);
3529
3530 __ st1(v0, __ T16B, __ post(to, 16));
3531
3532 __ subw(len_reg, len_reg, 16);
3533 __ cbnzw(len_reg, L_aes_loop);
3534
3535 __ st1(v0, __ T16B, rvec);
3536
3537 __ mov(r0, rscratch2);
3538
3539 __ leave();
3540 __ ret(lr);
3541
3542 // record the stub entry and end
3543 store_archive_data(stub_id, start, __ pc());
3544
3545 return start;
3546 }
3547
3548 // Arguments:
3549 //
3550 // Inputs:
3551 // c_rarg0 - source byte array address
3552 // c_rarg1 - destination byte array address
3553 // c_rarg2 - sessionKd (key) in little endian int array
3554 // c_rarg3 - r vector byte array address
3555 // c_rarg4 - input length
3556 //
3557 // Output:
3558 // r0 - input length
3559 //
3560 address generate_cipherBlockChaining_decryptAESCrypt() {
3561 assert(UseAES, "need AES cryptographic extension support");
3562 StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
3563 int entry_count = StubInfo::entry_count(stub_id);
3564 assert(entry_count == 1, "sanity check");
3565 address start = load_archive_data(stub_id);
3566 if (start != nullptr) {
3567 return start;
3568 }
3569 __ align(CodeEntryAlignment);
3570 StubCodeMark mark(this, stub_id);
3571
3572 Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52;
3573
3574 const Register from = c_rarg0; // source array address
3575 const Register to = c_rarg1; // destination array address
3576 const Register key = c_rarg2; // key array address
3577 const Register rvec = c_rarg3; // r byte array initialized from initvector array address
3578 // and left with the results of the last encryption block
3579 const Register len_reg = c_rarg4; // src len (must be multiple of blocksize 16)
3580 const Register keylen = rscratch1;
3581
3582 start = __ pc();
3583
3584 __ enter();
3585
3586 __ movw(rscratch2, len_reg);
3587
3588 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3589
3590 __ ld1(v2, __ T16B, rvec);
3591
3592 __ ld1(v31, __ T16B, __ post(key, 16));
3593 __ rev32(v31, __ T16B, v31);
3594
3595 __ cmpw(keylen, 52);
3596 __ br(Assembler::CC, L_loadkeys_44);
3597 __ br(Assembler::EQ, L_loadkeys_52);
3598
3599 __ ld1(v17, v18, __ T16B, __ post(key, 32));
3600 __ rev32(v17, __ T16B, v17);
3601 __ rev32(v18, __ T16B, v18);
3602 __ BIND(L_loadkeys_52);
3603 __ ld1(v19, v20, __ T16B, __ post(key, 32));
3604 __ rev32(v19, __ T16B, v19);
3605 __ rev32(v20, __ T16B, v20);
3606 __ BIND(L_loadkeys_44);
3607 __ ld1(v21, v22, v23, v24, __ T16B, __ post(key, 64));
3608 __ rev32(v21, __ T16B, v21);
3609 __ rev32(v22, __ T16B, v22);
3610 __ rev32(v23, __ T16B, v23);
3611 __ rev32(v24, __ T16B, v24);
3612 __ ld1(v25, v26, v27, v28, __ T16B, __ post(key, 64));
3613 __ rev32(v25, __ T16B, v25);
3614 __ rev32(v26, __ T16B, v26);
3615 __ rev32(v27, __ T16B, v27);
3616 __ rev32(v28, __ T16B, v28);
3617 __ ld1(v29, v30, __ T16B, key);
3618 __ rev32(v29, __ T16B, v29);
3619 __ rev32(v30, __ T16B, v30);
3620
3621 __ BIND(L_aes_loop);
3622 __ ld1(v0, __ T16B, __ post(from, 16));
3623 __ orr(v1, __ T16B, v0, v0);
3624
3625 __ br(Assembler::CC, L_rounds_44);
3626 __ br(Assembler::EQ, L_rounds_52);
3627
3628 __ aesd(v0, v17); __ aesimc(v0, v0);
3629 __ aesd(v0, v18); __ aesimc(v0, v0);
3630 __ BIND(L_rounds_52);
3631 __ aesd(v0, v19); __ aesimc(v0, v0);
3632 __ aesd(v0, v20); __ aesimc(v0, v0);
3633 __ BIND(L_rounds_44);
3634 __ aesd(v0, v21); __ aesimc(v0, v0);
3635 __ aesd(v0, v22); __ aesimc(v0, v0);
3636 __ aesd(v0, v23); __ aesimc(v0, v0);
3637 __ aesd(v0, v24); __ aesimc(v0, v0);
3638 __ aesd(v0, v25); __ aesimc(v0, v0);
3639 __ aesd(v0, v26); __ aesimc(v0, v0);
3640 __ aesd(v0, v27); __ aesimc(v0, v0);
3641 __ aesd(v0, v28); __ aesimc(v0, v0);
3642 __ aesd(v0, v29); __ aesimc(v0, v0);
3643 __ aesd(v0, v30);
3644 __ eor(v0, __ T16B, v0, v31);
3645 __ eor(v0, __ T16B, v0, v2);
3646
3647 __ st1(v0, __ T16B, __ post(to, 16));
3648 __ orr(v2, __ T16B, v1, v1);
3649
3650 __ subw(len_reg, len_reg, 16);
3651 __ cbnzw(len_reg, L_aes_loop);
3652
3653 __ st1(v2, __ T16B, rvec);
3654
3655 __ mov(r0, rscratch2);
3656
3657 __ leave();
3658 __ ret(lr);
3659
3660 // record the stub entry and end
3661 store_archive_data(stub_id, start, __ pc());
3662
3663 return start;
3664 }
3665
3666 // Big-endian 128-bit + 64-bit -> 128-bit addition.
3667 // Inputs: 128-bits. in is preserved.
3668 // The least-significant 64-bit word is in the upper dword of each vector.
3669 // inc (the 64-bit increment) is preserved. Its lower dword must be zero.
3670 // Output: result
3671 void be_add_128_64(FloatRegister result, FloatRegister in,
3672 FloatRegister inc, FloatRegister tmp) {
3673 assert_different_registers(result, tmp, inc);
3674
3675 __ addv(result, __ T2D, in, inc); // Add inc to the least-significant dword of
3676 // input
3677 __ cm(__ HI, tmp, __ T2D, inc, result);// Check for result overflowing
3678 __ ext(tmp, __ T16B, tmp, tmp, 0x08); // Swap LSD of comparison result to MSD and
3679 // MSD == 0 (must be!) to LSD
3680 __ subv(result, __ T2D, result, tmp); // Subtract -1 from MSD if there was an overflow
3681 }
3682
3683 // CTR AES crypt.
3684 // Arguments:
3685 //
3686 // Inputs:
3687 // c_rarg0 - source byte array address
3688 // c_rarg1 - destination byte array address
3689 // c_rarg2 - sessionKe (key) in little endian int array
3690 // c_rarg3 - counter vector byte array address
3691 // c_rarg4 - input length
3692 // c_rarg5 - saved encryptedCounter start
3693 // c_rarg6 - saved used length
3694 //
3695 // Output:
3696 // r0 - input length
3697 //
3698 address generate_counterMode_AESCrypt() {
3699 StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
3700 int entry_count = StubInfo::entry_count(stub_id);
3701 assert(entry_count == 1, "sanity check");
3702 address start = load_archive_data(stub_id);
3703 if (start != nullptr) {
3704 return start;
3705 }
3706 const Register in = c_rarg0;
3707 const Register out = c_rarg1;
3708 const Register key = c_rarg2;
3709 const Register counter = c_rarg3;
3710 const Register saved_len = c_rarg4, len = r10;
3711 const Register saved_encrypted_ctr = c_rarg5;
3712 const Register used_ptr = c_rarg6, used = r12;
3713
3714 const Register offset = r7;
3715 const Register keylen = r11;
3716
3717 const unsigned char block_size = 16;
3718 const int bulk_width = 4;
3719 // NB: bulk_width can be 4 or 8. 8 gives slightly faster
3720 // performance with larger data sizes, but it also means that the
3721 // fast path isn't used until you have at least 8 blocks, and up
3722 // to 127 bytes of data will be executed on the slow path. For
3723 // that reason, and also so as not to blow away too much icache, 4
3724 // blocks seems like a sensible compromise.
3725
3726 // Algorithm:
3727 //
3728 // if (len == 0) {
3729 // goto DONE;
3730 // }
3731 // int result = len;
3732 // do {
3733 // if (used >= blockSize) {
3734 // if (len >= bulk_width * blockSize) {
3735 // CTR_large_block();
3736 // if (len == 0)
3737 // goto DONE;
3738 // }
3739 // for (;;) {
3740 // 16ByteVector v0 = counter;
3741 // embeddedCipher.encryptBlock(v0, 0, encryptedCounter, 0);
3742 // used = 0;
3743 // if (len < blockSize)
3744 // break; /* goto NEXT */
3745 // 16ByteVector v1 = load16Bytes(in, offset);
3746 // v1 = v1 ^ encryptedCounter;
3747 // store16Bytes(out, offset);
3748 // used = blockSize;
3749 // offset += blockSize;
3750 // len -= blockSize;
3751 // if (len == 0)
3752 // goto DONE;
3753 // }
3754 // }
3755 // NEXT:
3756 // out[outOff++] = (byte)(in[inOff++] ^ encryptedCounter[used++]);
3757 // len--;
3758 // } while (len != 0);
3759 // DONE:
3760 // return result;
3761 //
3762 // CTR_large_block()
3763 // Wide bulk encryption of whole blocks.
3764
3765 __ align(CodeEntryAlignment);
3766 StubCodeMark mark(this, stub_id);
3767 start = __ pc();
3768 __ enter();
3769
3770 Label DONE, CTR_large_block, large_block_return;
3771 __ ldrw(used, Address(used_ptr));
3772 __ cbzw(saved_len, DONE);
3773
3774 __ mov(len, saved_len);
3775 __ mov(offset, 0);
3776
3777 // Compute #rounds for AES based on the length of the key array
3778 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3779
3780 __ aesenc_loadkeys(key, keylen);
3781
3782 {
3783 Label L_CTR_loop, NEXT;
3784
3785 __ bind(L_CTR_loop);
3786
3787 __ cmp(used, block_size);
3788 __ br(__ LO, NEXT);
3789
3790 // Maybe we have a lot of data
3791 __ subsw(rscratch1, len, bulk_width * block_size);
3792 __ br(__ HS, CTR_large_block);
3793 __ BIND(large_block_return);
3794 __ cbzw(len, DONE);
3795
3796 // Setup the counter
3797 __ movi(v4, __ T4S, 0);
3798 __ movi(v5, __ T4S, 1);
3799 __ ins(v4, __ S, v5, 2, 2); // v4 contains { 0, 1 }
3800
3801 // 128-bit big-endian increment
3802 __ ld1(v0, __ T16B, counter);
3803 __ rev64(v16, __ T16B, v0);
3804 be_add_128_64(v16, v16, v4, /*tmp*/v5);
3805 __ rev64(v16, __ T16B, v16);
3806 __ st1(v16, __ T16B, counter);
3807 // Previous counter value is in v0
3808 // v4 contains { 0, 1 }
3809
3810 {
3811 // We have fewer than bulk_width blocks of data left. Encrypt
3812 // them one by one until there is less than a full block
3813 // remaining, being careful to save both the encrypted counter
3814 // and the counter.
3815
3816 Label inner_loop;
3817 __ bind(inner_loop);
3818 // Counter to encrypt is in v0
3819 __ aesecb_encrypt(noreg, noreg, keylen);
3820 __ st1(v0, __ T16B, saved_encrypted_ctr);
3821
3822 // Do we have a remaining full block?
3823
3824 __ mov(used, 0);
3825 __ cmp(len, block_size);
3826 __ br(__ LO, NEXT);
3827
3828 // Yes, we have a full block
3829 __ ldrq(v1, Address(in, offset));
3830 __ eor(v1, __ T16B, v1, v0);
3831 __ strq(v1, Address(out, offset));
3832 __ mov(used, block_size);
3833 __ add(offset, offset, block_size);
3834
3835 __ subw(len, len, block_size);
3836 __ cbzw(len, DONE);
3837
3838 // Increment the counter, store it back
3839 __ orr(v0, __ T16B, v16, v16);
3840 __ rev64(v16, __ T16B, v16);
3841 be_add_128_64(v16, v16, v4, /*tmp*/v5);
3842 __ rev64(v16, __ T16B, v16);
3843 __ st1(v16, __ T16B, counter); // Save the incremented counter back
3844
3845 __ b(inner_loop);
3846 }
3847
3848 __ BIND(NEXT);
3849
3850 // Encrypt a single byte, and loop.
3851 // We expect this to be a rare event.
3852 __ ldrb(rscratch1, Address(in, offset));
3853 __ ldrb(rscratch2, Address(saved_encrypted_ctr, used));
3854 __ eor(rscratch1, rscratch1, rscratch2);
3855 __ strb(rscratch1, Address(out, offset));
3856 __ add(offset, offset, 1);
3857 __ add(used, used, 1);
3858 __ subw(len, len,1);
3859 __ cbnzw(len, L_CTR_loop);
3860 }
3861
3862 __ bind(DONE);
3863 __ strw(used, Address(used_ptr));
3864 __ mov(r0, saved_len);
3865
3866 __ leave(); // required for proper stackwalking of RuntimeStub frame
3867 __ ret(lr);
3868
3869 // Bulk encryption
3870
3871 __ BIND (CTR_large_block);
3872 assert(bulk_width == 4 || bulk_width == 8, "must be");
3873
3874 if (bulk_width == 8) {
3875 __ sub(sp, sp, 4 * 16);
3876 __ st1(v12, v13, v14, v15, __ T16B, Address(sp));
3877 }
3878 __ sub(sp, sp, 4 * 16);
3879 __ st1(v8, v9, v10, v11, __ T16B, Address(sp));
3880 RegSet saved_regs = (RegSet::of(in, out, offset)
3881 + RegSet::of(saved_encrypted_ctr, used_ptr, len));
3882 __ push(saved_regs, sp);
3883 __ andr(len, len, -16 * bulk_width); // 8/4 encryptions, 16 bytes per encryption
3884 __ add(in, in, offset);
3885 __ add(out, out, offset);
3886
3887 // Keys should already be loaded into the correct registers
3888
3889 __ ld1(v0, __ T16B, counter); // v0 contains the first counter
3890 __ rev64(v16, __ T16B, v0); // v16 contains byte-reversed counter
3891
3892 // AES/CTR loop
3893 {
3894 Label L_CTR_loop;
3895 __ BIND(L_CTR_loop);
3896
3897 // Setup the counters
3898 __ movi(v8, __ T4S, 0);
3899 __ movi(v9, __ T4S, 1);
3900 __ ins(v8, __ S, v9, 2, 2); // v8 contains { 0, 1 }
3901
3902 for (int i = 0; i < bulk_width; i++) {
3903 FloatRegister v0_ofs = as_FloatRegister(v0->encoding() + i);
3904 __ rev64(v0_ofs, __ T16B, v16);
3905 be_add_128_64(v16, v16, v8, /*tmp*/v9);
3906 }
3907
3908 __ ld1(v8, v9, v10, v11, __ T16B, __ post(in, 4 * 16));
3909
3910 // Encrypt the counters
3911 __ aesecb_encrypt(noreg, noreg, keylen, v0, bulk_width);
3912
3913 if (bulk_width == 8) {
3914 __ ld1(v12, v13, v14, v15, __ T16B, __ post(in, 4 * 16));
3915 }
3916
3917 // XOR the encrypted counters with the inputs
3918 for (int i = 0; i < bulk_width; i++) {
3919 FloatRegister v0_ofs = as_FloatRegister(v0->encoding() + i);
3920 FloatRegister v8_ofs = as_FloatRegister(v8->encoding() + i);
3921 __ eor(v0_ofs, __ T16B, v0_ofs, v8_ofs);
3922 }
3923
3924 // Write the encrypted data
3925 __ st1(v0, v1, v2, v3, __ T16B, __ post(out, 4 * 16));
3926 if (bulk_width == 8) {
3927 __ st1(v4, v5, v6, v7, __ T16B, __ post(out, 4 * 16));
3928 }
3929
3930 __ subw(len, len, 16 * bulk_width);
3931 __ cbnzw(len, L_CTR_loop);
3932 }
3933
3934 // Save the counter back where it goes
3935 __ rev64(v16, __ T16B, v16);
3936 __ st1(v16, __ T16B, counter);
3937
3938 __ pop(saved_regs, sp);
3939
3940 __ ld1(v8, v9, v10, v11, __ T16B, __ post(sp, 4 * 16));
3941 if (bulk_width == 8) {
3942 __ ld1(v12, v13, v14, v15, __ T16B, __ post(sp, 4 * 16));
3943 }
3944
3945 __ andr(rscratch1, len, -16 * bulk_width);
3946 __ sub(len, len, rscratch1);
3947 __ add(offset, offset, rscratch1);
3948 __ mov(used, 16);
3949 __ strw(used, Address(used_ptr));
3950 __ b(large_block_return);
3951
3952 // record the stub entry and end
3953 store_archive_data(stub_id, start, __ pc());
3954
3955 return start;
3956 }
3957
3958 // Vector AES Galois Counter Mode implementation. Parameters:
3959 //
3960 // in = c_rarg0
3961 // len = c_rarg1
3962 // ct = c_rarg2 - ciphertext that ghash will read (in for encrypt, out for decrypt)
3963 // out = c_rarg3
3964 // key = c_rarg4
3965 // state = c_rarg5 - GHASH.state
3966 // subkeyHtbl = c_rarg6 - powers of H
3967 // counter = c_rarg7 - 16 bytes of CTR
3968 // return - number of processed bytes
3969 address generate_galoisCounterMode_AESCrypt() {
3970 Label ghash_polynomial; // local data generated after code
3971 StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
3972 int entry_count = StubInfo::entry_count(stub_id);
3973 assert(entry_count == 1, "sanity check");
3974 address start = load_archive_data(stub_id);
3975 if (start != nullptr) {
3976 return start;
3977 }
3978 __ align(CodeEntryAlignment);
3979 StubCodeMark mark(this, stub_id);
3980 start = __ pc();
3981 __ enter();
3982
3983 const Register in = c_rarg0;
3984 const Register len = c_rarg1;
3985 const Register ct = c_rarg2;
3986 const Register out = c_rarg3;
3987 // and updated with the incremented counter in the end
3988
3989 const Register key = c_rarg4;
3990 const Register state = c_rarg5;
3991
3992 const Register subkeyHtbl = c_rarg6;
3993
3994 const Register counter = c_rarg7;
3995
3996 const Register keylen = r10;
3997 // Save state before entering routine
3998 __ sub(sp, sp, 4 * 16);
3999 __ st1(v12, v13, v14, v15, __ T16B, Address(sp));
4000 __ sub(sp, sp, 4 * 16);
4001 __ st1(v8, v9, v10, v11, __ T16B, Address(sp));
4002
4003 // __ andr(len, len, -512);
4004 __ andr(len, len, -16 * 8); // 8 encryptions, 16 bytes per encryption
4005 __ str(len, __ pre(sp, -2 * wordSize));
4006
4007 Label DONE;
4008 __ cbz(len, DONE);
4009
4010 // Compute #rounds for AES based on the length of the key array
4011 __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
4012
4013 __ aesenc_loadkeys(key, keylen);
4014 __ ld1(v0, __ T16B, counter); // v0 contains the first counter
4015 __ rev32(v16, __ T16B, v0); // v16 contains byte-reversed counter
4016
4017 // AES/CTR loop
4018 {
4019 Label L_CTR_loop;
4020 __ BIND(L_CTR_loop);
4021
4022 // Setup the counters
4023 __ movi(v8, __ T4S, 0);
4024 __ movi(v9, __ T4S, 1);
4025 __ ins(v8, __ S, v9, 3, 3); // v8 contains { 0, 0, 0, 1 }
4026
4027 assert(v0->encoding() < v8->encoding(), "");
4028 for (int i = v0->encoding(); i < v8->encoding(); i++) {
4029 FloatRegister f = as_FloatRegister(i);
4030 __ rev32(f, __ T16B, v16);
4031 __ addv(v16, __ T4S, v16, v8);
4032 }
4033
4034 __ ld1(v8, v9, v10, v11, __ T16B, __ post(in, 4 * 16));
4035
4036 // Encrypt the counters
4037 __ aesecb_encrypt(noreg, noreg, keylen, v0, /*unrolls*/8);
4038
4039 __ ld1(v12, v13, v14, v15, __ T16B, __ post(in, 4 * 16));
4040
4041 // XOR the encrypted counters with the inputs
4042 for (int i = 0; i < 8; i++) {
4043 FloatRegister v0_ofs = as_FloatRegister(v0->encoding() + i);
4044 FloatRegister v8_ofs = as_FloatRegister(v8->encoding() + i);
4045 __ eor(v0_ofs, __ T16B, v0_ofs, v8_ofs);
4046 }
4047 __ st1(v0, v1, v2, v3, __ T16B, __ post(out, 4 * 16));
4048 __ st1(v4, v5, v6, v7, __ T16B, __ post(out, 4 * 16));
4049
4050 __ subw(len, len, 16 * 8);
4051 __ cbnzw(len, L_CTR_loop);
4052 }
4053
4054 __ rev32(v16, __ T16B, v16);
4055 __ st1(v16, __ T16B, counter);
4056
4057 __ ldr(len, Address(sp));
4058 __ lsr(len, len, exact_log2(16)); // We want the count of blocks
4059
4060 // GHASH/CTR loop
4061 __ ghash_processBlocks_wide(ghash_polynomial, state, subkeyHtbl, ct,
4062 len, /*unrolls*/4);
4063
4064 #ifdef ASSERT
4065 { Label L;
4066 __ cmp(len, (unsigned char)0);
4067 __ br(Assembler::EQ, L);
4068 __ stop("stubGenerator: abort");
4069 __ bind(L);
4070 }
4071 #endif
4072
4073 __ bind(DONE);
4074 // Return the number of bytes processed
4075 __ ldr(r0, __ post(sp, 2 * wordSize));
4076
4077 __ ld1(v8, v9, v10, v11, __ T16B, __ post(sp, 4 * 16));
4078 __ ld1(v12, v13, v14, v15, __ T16B, __ post(sp, 4 * 16));
4079
4080 __ leave(); // required for proper stackwalking of RuntimeStub frame
4081 __ ret(lr);
4082
4083 // bind label and generate polynomial data
4084 __ align(wordSize * 2);
4085 __ bind(ghash_polynomial);
4086 __ emit_int64(0x87); // The low-order bits of the field
4087 // polynomial (i.e. p = z^7+z^2+z+1)
4088 // repeated in the low and high parts of a
4089 // 128-bit vector
4090 __ emit_int64(0x87);
4091
4092 // record the stub entry and end
4093 store_archive_data(stub_id, start, __ pc());
4094
4095 return start;
4096 }
4097
4098 class Cached64Bytes {
4099 private:
4100 MacroAssembler *_masm;
4101 Register _regs[8];
4102
4103 public:
4104 Cached64Bytes(MacroAssembler *masm, RegSet rs): _masm(masm) {
4105 assert(rs.size() == 8, "%u registers are used to cache 16 4-byte data", rs.size());
4106 auto it = rs.begin();
4107 for (auto &r: _regs) {
4108 r = *it;
4109 ++it;
4110 }
4111 }
4112
4113 void gen_loads(Register base) {
4114 for (int i = 0; i < 8; i += 2) {
4115 __ ldp(_regs[i], _regs[i + 1], Address(base, 8 * i));
4116 }
4117 }
4118
4119 // Generate code extracting i-th unsigned word (4 bytes) from cached 64 bytes.
4120 void extract_u32(Register dest, int i) {
4121 __ ubfx(dest, _regs[i / 2], 32 * (i % 2), 32);
4122 }
4123 };
4124
4125 // Utility routines for md5.
4126 // Clobbers r10 and r11.
4127 void md5_FF(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
4128 int k, int s, int t) {
4129 Register rscratch3 = r10;
4130 Register rscratch4 = r11;
4131
4132 __ eorw(rscratch3, r3, r4);
4133 __ movw(rscratch2, t);
4134 __ andw(rscratch3, rscratch3, r2);
4135 __ addw(rscratch4, r1, rscratch2);
4136 reg_cache.extract_u32(rscratch1, k);
4137 __ eorw(rscratch3, rscratch3, r4);
4138 __ addw(rscratch4, rscratch4, rscratch1);
4139 __ addw(rscratch3, rscratch3, rscratch4);
4140 __ rorw(rscratch2, rscratch3, 32 - s);
4141 __ addw(r1, rscratch2, r2);
4142 }
4143
4144 void md5_GG(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
4145 int k, int s, int t) {
4146 Register rscratch3 = r10;
4147 Register rscratch4 = r11;
4148
4149 reg_cache.extract_u32(rscratch1, k);
4150 __ movw(rscratch2, t);
4151 __ addw(rscratch4, r1, rscratch2);
4152 __ addw(rscratch4, rscratch4, rscratch1);
4153 __ bicw(rscratch2, r3, r4);
4154 __ andw(rscratch3, r2, r4);
4155 __ addw(rscratch2, rscratch2, rscratch4);
4156 __ addw(rscratch2, rscratch2, rscratch3);
4157 __ rorw(rscratch2, rscratch2, 32 - s);
4158 __ addw(r1, rscratch2, r2);
4159 }
4160
4161 void md5_HH(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
4162 int k, int s, int t) {
4163 Register rscratch3 = r10;
4164 Register rscratch4 = r11;
4165
4166 __ eorw(rscratch3, r3, r4);
4167 __ movw(rscratch2, t);
4168 __ addw(rscratch4, r1, rscratch2);
4169 reg_cache.extract_u32(rscratch1, k);
4170 __ eorw(rscratch3, rscratch3, r2);
4171 __ addw(rscratch4, rscratch4, rscratch1);
4172 __ addw(rscratch3, rscratch3, rscratch4);
4173 __ rorw(rscratch2, rscratch3, 32 - s);
4174 __ addw(r1, rscratch2, r2);
4175 }
4176
4177 void md5_II(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
4178 int k, int s, int t) {
4179 Register rscratch3 = r10;
4180 Register rscratch4 = r11;
4181
4182 __ movw(rscratch3, t);
4183 __ ornw(rscratch2, r2, r4);
4184 __ addw(rscratch4, r1, rscratch3);
4185 reg_cache.extract_u32(rscratch1, k);
4186 __ eorw(rscratch3, rscratch2, r3);
4187 __ addw(rscratch4, rscratch4, rscratch1);
4188 __ addw(rscratch3, rscratch3, rscratch4);
4189 __ rorw(rscratch2, rscratch3, 32 - s);
4190 __ addw(r1, rscratch2, r2);
4191 }
4192
4193 // Arguments:
4194 //
4195 // Inputs:
4196 // c_rarg0 - byte[] source+offset
4197 // c_rarg1 - int[] SHA.state
4198 // c_rarg2 - int offset
4199 // c_rarg3 - int limit
4200 //
4201 address generate_md5_implCompress(StubId stub_id) {
4202 bool multi_block;
4203 switch (stub_id) {
4204 case StubId::stubgen_md5_implCompress_id:
4205 multi_block = false;
4206 break;
4207 case StubId::stubgen_md5_implCompressMB_id:
4208 multi_block = true;
4209 break;
4210 default:
4211 ShouldNotReachHere();
4212 }
4213 int entry_count = StubInfo::entry_count(stub_id);
4214 assert(entry_count == 1, "sanity check");
4215 address start = load_archive_data(stub_id);
4216 if (start != nullptr) {
4217 return start;
4218 }
4219 __ align(CodeEntryAlignment);
4220
4221 StubCodeMark mark(this, stub_id);
4222 start = __ pc();
4223
4224 Register buf = c_rarg0;
4225 Register state = c_rarg1;
4226 Register ofs = c_rarg2;
4227 Register limit = c_rarg3;
4228 Register a = r4;
4229 Register b = r5;
4230 Register c = r6;
4231 Register d = r7;
4232 Register rscratch3 = r10;
4233 Register rscratch4 = r11;
4234
4235 Register state_regs[2] = { r12, r13 };
4236 RegSet saved_regs = RegSet::range(r16, r22) - r18_tls;
4237 Cached64Bytes reg_cache(_masm, RegSet::of(r14, r15) + saved_regs); // using 8 registers
4238
4239 __ push(saved_regs, sp);
4240
4241 __ ldp(state_regs[0], state_regs[1], Address(state));
4242 __ ubfx(a, state_regs[0], 0, 32);
4243 __ ubfx(b, state_regs[0], 32, 32);
4244 __ ubfx(c, state_regs[1], 0, 32);
4245 __ ubfx(d, state_regs[1], 32, 32);
4246
4247 Label md5_loop;
4248 __ BIND(md5_loop);
4249
4250 reg_cache.gen_loads(buf);
4251
4252 // Round 1
4253 md5_FF(reg_cache, a, b, c, d, 0, 7, 0xd76aa478);
4254 md5_FF(reg_cache, d, a, b, c, 1, 12, 0xe8c7b756);
4255 md5_FF(reg_cache, c, d, a, b, 2, 17, 0x242070db);
4256 md5_FF(reg_cache, b, c, d, a, 3, 22, 0xc1bdceee);
4257 md5_FF(reg_cache, a, b, c, d, 4, 7, 0xf57c0faf);
4258 md5_FF(reg_cache, d, a, b, c, 5, 12, 0x4787c62a);
4259 md5_FF(reg_cache, c, d, a, b, 6, 17, 0xa8304613);
4260 md5_FF(reg_cache, b, c, d, a, 7, 22, 0xfd469501);
4261 md5_FF(reg_cache, a, b, c, d, 8, 7, 0x698098d8);
4262 md5_FF(reg_cache, d, a, b, c, 9, 12, 0x8b44f7af);
4263 md5_FF(reg_cache, c, d, a, b, 10, 17, 0xffff5bb1);
4264 md5_FF(reg_cache, b, c, d, a, 11, 22, 0x895cd7be);
4265 md5_FF(reg_cache, a, b, c, d, 12, 7, 0x6b901122);
4266 md5_FF(reg_cache, d, a, b, c, 13, 12, 0xfd987193);
4267 md5_FF(reg_cache, c, d, a, b, 14, 17, 0xa679438e);
4268 md5_FF(reg_cache, b, c, d, a, 15, 22, 0x49b40821);
4269
4270 // Round 2
4271 md5_GG(reg_cache, a, b, c, d, 1, 5, 0xf61e2562);
4272 md5_GG(reg_cache, d, a, b, c, 6, 9, 0xc040b340);
4273 md5_GG(reg_cache, c, d, a, b, 11, 14, 0x265e5a51);
4274 md5_GG(reg_cache, b, c, d, a, 0, 20, 0xe9b6c7aa);
4275 md5_GG(reg_cache, a, b, c, d, 5, 5, 0xd62f105d);
4276 md5_GG(reg_cache, d, a, b, c, 10, 9, 0x02441453);
4277 md5_GG(reg_cache, c, d, a, b, 15, 14, 0xd8a1e681);
4278 md5_GG(reg_cache, b, c, d, a, 4, 20, 0xe7d3fbc8);
4279 md5_GG(reg_cache, a, b, c, d, 9, 5, 0x21e1cde6);
4280 md5_GG(reg_cache, d, a, b, c, 14, 9, 0xc33707d6);
4281 md5_GG(reg_cache, c, d, a, b, 3, 14, 0xf4d50d87);
4282 md5_GG(reg_cache, b, c, d, a, 8, 20, 0x455a14ed);
4283 md5_GG(reg_cache, a, b, c, d, 13, 5, 0xa9e3e905);
4284 md5_GG(reg_cache, d, a, b, c, 2, 9, 0xfcefa3f8);
4285 md5_GG(reg_cache, c, d, a, b, 7, 14, 0x676f02d9);
4286 md5_GG(reg_cache, b, c, d, a, 12, 20, 0x8d2a4c8a);
4287
4288 // Round 3
4289 md5_HH(reg_cache, a, b, c, d, 5, 4, 0xfffa3942);
4290 md5_HH(reg_cache, d, a, b, c, 8, 11, 0x8771f681);
4291 md5_HH(reg_cache, c, d, a, b, 11, 16, 0x6d9d6122);
4292 md5_HH(reg_cache, b, c, d, a, 14, 23, 0xfde5380c);
4293 md5_HH(reg_cache, a, b, c, d, 1, 4, 0xa4beea44);
4294 md5_HH(reg_cache, d, a, b, c, 4, 11, 0x4bdecfa9);
4295 md5_HH(reg_cache, c, d, a, b, 7, 16, 0xf6bb4b60);
4296 md5_HH(reg_cache, b, c, d, a, 10, 23, 0xbebfbc70);
4297 md5_HH(reg_cache, a, b, c, d, 13, 4, 0x289b7ec6);
4298 md5_HH(reg_cache, d, a, b, c, 0, 11, 0xeaa127fa);
4299 md5_HH(reg_cache, c, d, a, b, 3, 16, 0xd4ef3085);
4300 md5_HH(reg_cache, b, c, d, a, 6, 23, 0x04881d05);
4301 md5_HH(reg_cache, a, b, c, d, 9, 4, 0xd9d4d039);
4302 md5_HH(reg_cache, d, a, b, c, 12, 11, 0xe6db99e5);
4303 md5_HH(reg_cache, c, d, a, b, 15, 16, 0x1fa27cf8);
4304 md5_HH(reg_cache, b, c, d, a, 2, 23, 0xc4ac5665);
4305
4306 // Round 4
4307 md5_II(reg_cache, a, b, c, d, 0, 6, 0xf4292244);
4308 md5_II(reg_cache, d, a, b, c, 7, 10, 0x432aff97);
4309 md5_II(reg_cache, c, d, a, b, 14, 15, 0xab9423a7);
4310 md5_II(reg_cache, b, c, d, a, 5, 21, 0xfc93a039);
4311 md5_II(reg_cache, a, b, c, d, 12, 6, 0x655b59c3);
4312 md5_II(reg_cache, d, a, b, c, 3, 10, 0x8f0ccc92);
4313 md5_II(reg_cache, c, d, a, b, 10, 15, 0xffeff47d);
4314 md5_II(reg_cache, b, c, d, a, 1, 21, 0x85845dd1);
4315 md5_II(reg_cache, a, b, c, d, 8, 6, 0x6fa87e4f);
4316 md5_II(reg_cache, d, a, b, c, 15, 10, 0xfe2ce6e0);
4317 md5_II(reg_cache, c, d, a, b, 6, 15, 0xa3014314);
4318 md5_II(reg_cache, b, c, d, a, 13, 21, 0x4e0811a1);
4319 md5_II(reg_cache, a, b, c, d, 4, 6, 0xf7537e82);
4320 md5_II(reg_cache, d, a, b, c, 11, 10, 0xbd3af235);
4321 md5_II(reg_cache, c, d, a, b, 2, 15, 0x2ad7d2bb);
4322 md5_II(reg_cache, b, c, d, a, 9, 21, 0xeb86d391);
4323
4324 __ addw(a, state_regs[0], a);
4325 __ ubfx(rscratch2, state_regs[0], 32, 32);
4326 __ addw(b, rscratch2, b);
4327 __ addw(c, state_regs[1], c);
4328 __ ubfx(rscratch4, state_regs[1], 32, 32);
4329 __ addw(d, rscratch4, d);
4330
4331 __ orr(state_regs[0], a, b, Assembler::LSL, 32);
4332 __ orr(state_regs[1], c, d, Assembler::LSL, 32);
4333
4334 if (multi_block) {
4335 __ add(buf, buf, 64);
4336 __ add(ofs, ofs, 64);
4337 __ cmp(ofs, limit);
4338 __ br(Assembler::LE, md5_loop);
4339 __ mov(c_rarg0, ofs); // return ofs
4340 }
4341
4342 // write hash values back in the correct order
4343 __ stp(state_regs[0], state_regs[1], Address(state));
4344
4345 __ pop(saved_regs, sp);
4346
4347 __ ret(lr);
4348
4349 // record the stub entry and end
4350 store_archive_data(stub_id, start, __ pc());
4351
4352 return start;
4353 }
4354
4355 // Arguments:
4356 //
4357 // Inputs:
4358 // c_rarg0 - byte[] source+offset
4359 // c_rarg1 - int[] SHA.state
4360 // c_rarg2 - int offset
4361 // c_rarg3 - int limit
4362 //
4363 address generate_sha1_implCompress(StubId stub_id) {
4364 bool multi_block;
4365 switch (stub_id) {
4366 case StubId::stubgen_sha1_implCompress_id:
4367 multi_block = false;
4368 break;
4369 case StubId::stubgen_sha1_implCompressMB_id:
4370 multi_block = true;
4371 break;
4372 default:
4373 ShouldNotReachHere();
4374 }
4375 int entry_count = StubInfo::entry_count(stub_id);
4376 assert(entry_count == 1, "sanity check");
4377 address start = load_archive_data(stub_id);
4378 if (start != nullptr) {
4379 return start;
4380 }
4381 __ align(CodeEntryAlignment);
4382
4383 StubCodeMark mark(this, stub_id);
4384 start = __ pc();
4385
4386 Register buf = c_rarg0;
4387 Register state = c_rarg1;
4388 Register ofs = c_rarg2;
4389 Register limit = c_rarg3;
4390
4391 Label keys;
4392 Label sha1_loop;
4393
4394 // load the keys into v0..v3
4395 __ adr(rscratch1, keys);
4396 __ ld4r(v0, v1, v2, v3, __ T4S, Address(rscratch1));
4397 // load 5 words state into v6, v7
4398 __ ldrq(v6, Address(state, 0));
4399 __ ldrs(v7, Address(state, 16));
4400
4401
4402 __ BIND(sha1_loop);
4403 // load 64 bytes of data into v16..v19
4404 __ ld1(v16, v17, v18, v19, __ T4S, multi_block ? __ post(buf, 64) : buf);
4405 __ rev32(v16, __ T16B, v16);
4406 __ rev32(v17, __ T16B, v17);
4407 __ rev32(v18, __ T16B, v18);
4408 __ rev32(v19, __ T16B, v19);
4409
4410 // do the sha1
4411 __ addv(v4, __ T4S, v16, v0);
4412 __ orr(v20, __ T16B, v6, v6);
4413
4414 FloatRegister d0 = v16;
4415 FloatRegister d1 = v17;
4416 FloatRegister d2 = v18;
4417 FloatRegister d3 = v19;
4418
4419 for (int round = 0; round < 20; round++) {
4420 FloatRegister tmp1 = (round & 1) ? v4 : v5;
4421 FloatRegister tmp2 = (round & 1) ? v21 : v22;
4422 FloatRegister tmp3 = round ? ((round & 1) ? v22 : v21) : v7;
4423 FloatRegister tmp4 = (round & 1) ? v5 : v4;
4424 FloatRegister key = (round < 4) ? v0 : ((round < 9) ? v1 : ((round < 14) ? v2 : v3));
4425
4426 if (round < 16) __ sha1su0(d0, __ T4S, d1, d2);
4427 if (round < 19) __ addv(tmp1, __ T4S, d1, key);
4428 __ sha1h(tmp2, __ T4S, v20);
4429 if (round < 5)
4430 __ sha1c(v20, __ T4S, tmp3, tmp4);
4431 else if (round < 10 || round >= 15)
4432 __ sha1p(v20, __ T4S, tmp3, tmp4);
4433 else
4434 __ sha1m(v20, __ T4S, tmp3, tmp4);
4435 if (round < 16) __ sha1su1(d0, __ T4S, d3);
4436
4437 tmp1 = d0; d0 = d1; d1 = d2; d2 = d3; d3 = tmp1;
4438 }
4439
4440 __ addv(v7, __ T2S, v7, v21);
4441 __ addv(v6, __ T4S, v6, v20);
4442
4443 if (multi_block) {
4444 __ add(ofs, ofs, 64);
4445 __ cmp(ofs, limit);
4446 __ br(Assembler::LE, sha1_loop);
4447 __ mov(c_rarg0, ofs); // return ofs
4448 }
4449
4450 __ strq(v6, Address(state, 0));
4451 __ strs(v7, Address(state, 16));
4452
4453 __ ret(lr);
4454
4455 __ bind(keys);
4456 __ emit_int32(0x5a827999);
4457 __ emit_int32(0x6ed9eba1);
4458 __ emit_int32(0x8f1bbcdc);
4459 __ emit_int32(0xca62c1d6);
4460
4461 // record the stub entry and end
4462 store_archive_data(stub_id, start, __ pc());
4463
4464 return start;
4465 }
4466
4467
4468 // Arguments:
4469 //
4470 // Inputs:
4471 // c_rarg0 - byte[] source+offset
4472 // c_rarg1 - int[] SHA.state
4473 // c_rarg2 - int offset
4474 // c_rarg3 - int limit
4475 //
4476 address generate_sha256_implCompress(StubId stub_id) {
4477 bool multi_block;
4478 switch (stub_id) {
4479 case StubId::stubgen_sha256_implCompress_id:
4480 multi_block = false;
4481 break;
4482 case StubId::stubgen_sha256_implCompressMB_id:
4483 multi_block = true;
4484 break;
4485 default:
4486 ShouldNotReachHere();
4487 }
4488 int entry_count = StubInfo::entry_count(stub_id);
4489 assert(entry_count == 1, "sanity check");
4490 address start = load_archive_data(stub_id);
4491 if (start != nullptr) {
4492 return start;
4493 }
4494 __ align(CodeEntryAlignment);
4495 StubCodeMark mark(this, stub_id);
4496 start = __ pc();
4497
4498 Register buf = c_rarg0;
4499 Register state = c_rarg1;
4500 Register ofs = c_rarg2;
4501 Register limit = c_rarg3;
4502
4503 Label sha1_loop;
4504
4505 __ stpd(v8, v9, __ pre(sp, -32));
4506 __ stpd(v10, v11, Address(sp, 16));
4507
4508 // dga == v0
4509 // dgb == v1
4510 // dg0 == v2
4511 // dg1 == v3
4512 // dg2 == v4
4513 // t0 == v6
4514 // t1 == v7
4515
4516 // load 16 keys to v16..v31
4517 __ lea(rscratch1, ExternalAddress((address)_sha256_round_consts));
4518 __ ld1(v16, v17, v18, v19, __ T4S, __ post(rscratch1, 64));
4519 __ ld1(v20, v21, v22, v23, __ T4S, __ post(rscratch1, 64));
4520 __ ld1(v24, v25, v26, v27, __ T4S, __ post(rscratch1, 64));
4521 __ ld1(v28, v29, v30, v31, __ T4S, rscratch1);
4522
4523 // load 8 words (256 bits) state
4524 __ ldpq(v0, v1, state);
4525
4526 __ BIND(sha1_loop);
4527 // load 64 bytes of data into v8..v11
4528 __ ld1(v8, v9, v10, v11, __ T4S, multi_block ? __ post(buf, 64) : buf);
4529 __ rev32(v8, __ T16B, v8);
4530 __ rev32(v9, __ T16B, v9);
4531 __ rev32(v10, __ T16B, v10);
4532 __ rev32(v11, __ T16B, v11);
4533
4534 __ addv(v6, __ T4S, v8, v16);
4535 __ orr(v2, __ T16B, v0, v0);
4536 __ orr(v3, __ T16B, v1, v1);
4537
4538 FloatRegister d0 = v8;
4539 FloatRegister d1 = v9;
4540 FloatRegister d2 = v10;
4541 FloatRegister d3 = v11;
4542
4543
4544 for (int round = 0; round < 16; round++) {
4545 FloatRegister tmp1 = (round & 1) ? v6 : v7;
4546 FloatRegister tmp2 = (round & 1) ? v7 : v6;
4547 FloatRegister tmp3 = (round & 1) ? v2 : v4;
4548 FloatRegister tmp4 = (round & 1) ? v4 : v2;
4549
4550 if (round < 12) __ sha256su0(d0, __ T4S, d1);
4551 __ orr(v4, __ T16B, v2, v2);
4552 if (round < 15)
4553 __ addv(tmp1, __ T4S, d1, as_FloatRegister(round + 17));
4554 __ sha256h(v2, __ T4S, v3, tmp2);
4555 __ sha256h2(v3, __ T4S, v4, tmp2);
4556 if (round < 12) __ sha256su1(d0, __ T4S, d2, d3);
4557
4558 tmp1 = d0; d0 = d1; d1 = d2; d2 = d3; d3 = tmp1;
4559 }
4560
4561 __ addv(v0, __ T4S, v0, v2);
4562 __ addv(v1, __ T4S, v1, v3);
4563
4564 if (multi_block) {
4565 __ add(ofs, ofs, 64);
4566 __ cmp(ofs, limit);
4567 __ br(Assembler::LE, sha1_loop);
4568 __ mov(c_rarg0, ofs); // return ofs
4569 }
4570
4571 __ ldpd(v10, v11, Address(sp, 16));
4572 __ ldpd(v8, v9, __ post(sp, 32));
4573
4574 __ stpq(v0, v1, state);
4575
4576 __ ret(lr);
4577
4578 // record the stub entry and end
4579 store_archive_data(stub_id, start, __ pc());
4580
4581 return start;
4582 }
4583
4584 // Double rounds for sha512.
4585 void sha512_dround(int dr,
4586 FloatRegister vi0, FloatRegister vi1,
4587 FloatRegister vi2, FloatRegister vi3,
4588 FloatRegister vi4, FloatRegister vrc0,
4589 FloatRegister vrc1, FloatRegister vin0,
4590 FloatRegister vin1, FloatRegister vin2,
4591 FloatRegister vin3, FloatRegister vin4) {
4592 if (dr < 36) {
4593 __ ld1(vrc1, __ T2D, __ post(rscratch2, 16));
4594 }
4595 __ addv(v5, __ T2D, vrc0, vin0);
4596 __ ext(v6, __ T16B, vi2, vi3, 8);
4597 __ ext(v5, __ T16B, v5, v5, 8);
4598 __ ext(v7, __ T16B, vi1, vi2, 8);
4599 __ addv(vi3, __ T2D, vi3, v5);
4600 if (dr < 32) {
4601 __ ext(v5, __ T16B, vin3, vin4, 8);
4602 __ sha512su0(vin0, __ T2D, vin1);
4603 }
4604 __ sha512h(vi3, __ T2D, v6, v7);
4605 if (dr < 32) {
4606 __ sha512su1(vin0, __ T2D, vin2, v5);
4607 }
4608 __ addv(vi4, __ T2D, vi1, vi3);
4609 __ sha512h2(vi3, __ T2D, vi1, vi0);
4610 }
4611
4612 // Arguments:
4613 //
4614 // Inputs:
4615 // c_rarg0 - byte[] source+offset
4616 // c_rarg1 - int[] SHA.state
4617 // c_rarg2 - int offset
4618 // c_rarg3 - int limit
4619 //
4620 address generate_sha512_implCompress(StubId stub_id) {
4621 bool multi_block;
4622 switch (stub_id) {
4623 case StubId::stubgen_sha512_implCompress_id:
4624 multi_block = false;
4625 break;
4626 case StubId::stubgen_sha512_implCompressMB_id:
4627 multi_block = true;
4628 break;
4629 default:
4630 ShouldNotReachHere();
4631 }
4632 int entry_count = StubInfo::entry_count(stub_id);
4633 assert(entry_count == 1, "sanity check");
4634 address start = load_archive_data(stub_id);
4635 if (start != nullptr) {
4636 return start;
4637 }
4638 __ align(CodeEntryAlignment);
4639 StubCodeMark mark(this, stub_id);
4640 start = __ pc();
4641
4642 Register buf = c_rarg0;
4643 Register state = c_rarg1;
4644 Register ofs = c_rarg2;
4645 Register limit = c_rarg3;
4646
4647 __ stpd(v8, v9, __ pre(sp, -64));
4648 __ stpd(v10, v11, Address(sp, 16));
4649 __ stpd(v12, v13, Address(sp, 32));
4650 __ stpd(v14, v15, Address(sp, 48));
4651
4652 Label sha512_loop;
4653
4654 // load state
4655 __ ld1(v8, v9, v10, v11, __ T2D, state);
4656
4657 // load first 4 round constants
4658 __ lea(rscratch1, ExternalAddress((address)_sha512_round_consts));
4659 __ ld1(v20, v21, v22, v23, __ T2D, __ post(rscratch1, 64));
4660
4661 __ BIND(sha512_loop);
4662 // load 128B of data into v12..v19
4663 __ ld1(v12, v13, v14, v15, __ T2D, __ post(buf, 64));
4664 __ ld1(v16, v17, v18, v19, __ T2D, __ post(buf, 64));
4665 __ rev64(v12, __ T16B, v12);
4666 __ rev64(v13, __ T16B, v13);
4667 __ rev64(v14, __ T16B, v14);
4668 __ rev64(v15, __ T16B, v15);
4669 __ rev64(v16, __ T16B, v16);
4670 __ rev64(v17, __ T16B, v17);
4671 __ rev64(v18, __ T16B, v18);
4672 __ rev64(v19, __ T16B, v19);
4673
4674 __ mov(rscratch2, rscratch1);
4675
4676 __ mov(v0, __ T16B, v8);
4677 __ mov(v1, __ T16B, v9);
4678 __ mov(v2, __ T16B, v10);
4679 __ mov(v3, __ T16B, v11);
4680
4681 sha512_dround( 0, v0, v1, v2, v3, v4, v20, v24, v12, v13, v19, v16, v17);
4682 sha512_dround( 1, v3, v0, v4, v2, v1, v21, v25, v13, v14, v12, v17, v18);
4683 sha512_dround( 2, v2, v3, v1, v4, v0, v22, v26, v14, v15, v13, v18, v19);
4684 sha512_dround( 3, v4, v2, v0, v1, v3, v23, v27, v15, v16, v14, v19, v12);
4685 sha512_dround( 4, v1, v4, v3, v0, v2, v24, v28, v16, v17, v15, v12, v13);
4686 sha512_dround( 5, v0, v1, v2, v3, v4, v25, v29, v17, v18, v16, v13, v14);
4687 sha512_dround( 6, v3, v0, v4, v2, v1, v26, v30, v18, v19, v17, v14, v15);
4688 sha512_dround( 7, v2, v3, v1, v4, v0, v27, v31, v19, v12, v18, v15, v16);
4689 sha512_dround( 8, v4, v2, v0, v1, v3, v28, v24, v12, v13, v19, v16, v17);
4690 sha512_dround( 9, v1, v4, v3, v0, v2, v29, v25, v13, v14, v12, v17, v18);
4691 sha512_dround(10, v0, v1, v2, v3, v4, v30, v26, v14, v15, v13, v18, v19);
4692 sha512_dround(11, v3, v0, v4, v2, v1, v31, v27, v15, v16, v14, v19, v12);
4693 sha512_dround(12, v2, v3, v1, v4, v0, v24, v28, v16, v17, v15, v12, v13);
4694 sha512_dround(13, v4, v2, v0, v1, v3, v25, v29, v17, v18, v16, v13, v14);
4695 sha512_dround(14, v1, v4, v3, v0, v2, v26, v30, v18, v19, v17, v14, v15);
4696 sha512_dround(15, v0, v1, v2, v3, v4, v27, v31, v19, v12, v18, v15, v16);
4697 sha512_dround(16, v3, v0, v4, v2, v1, v28, v24, v12, v13, v19, v16, v17);
4698 sha512_dround(17, v2, v3, v1, v4, v0, v29, v25, v13, v14, v12, v17, v18);
4699 sha512_dround(18, v4, v2, v0, v1, v3, v30, v26, v14, v15, v13, v18, v19);
4700 sha512_dround(19, v1, v4, v3, v0, v2, v31, v27, v15, v16, v14, v19, v12);
4701 sha512_dround(20, v0, v1, v2, v3, v4, v24, v28, v16, v17, v15, v12, v13);
4702 sha512_dround(21, v3, v0, v4, v2, v1, v25, v29, v17, v18, v16, v13, v14);
4703 sha512_dround(22, v2, v3, v1, v4, v0, v26, v30, v18, v19, v17, v14, v15);
4704 sha512_dround(23, v4, v2, v0, v1, v3, v27, v31, v19, v12, v18, v15, v16);
4705 sha512_dround(24, v1, v4, v3, v0, v2, v28, v24, v12, v13, v19, v16, v17);
4706 sha512_dround(25, v0, v1, v2, v3, v4, v29, v25, v13, v14, v12, v17, v18);
4707 sha512_dround(26, v3, v0, v4, v2, v1, v30, v26, v14, v15, v13, v18, v19);
4708 sha512_dround(27, v2, v3, v1, v4, v0, v31, v27, v15, v16, v14, v19, v12);
4709 sha512_dround(28, v4, v2, v0, v1, v3, v24, v28, v16, v17, v15, v12, v13);
4710 sha512_dround(29, v1, v4, v3, v0, v2, v25, v29, v17, v18, v16, v13, v14);
4711 sha512_dround(30, v0, v1, v2, v3, v4, v26, v30, v18, v19, v17, v14, v15);
4712 sha512_dround(31, v3, v0, v4, v2, v1, v27, v31, v19, v12, v18, v15, v16);
4713 sha512_dround(32, v2, v3, v1, v4, v0, v28, v24, v12, v0, v0, v0, v0);
4714 sha512_dround(33, v4, v2, v0, v1, v3, v29, v25, v13, v0, v0, v0, v0);
4715 sha512_dround(34, v1, v4, v3, v0, v2, v30, v26, v14, v0, v0, v0, v0);
4716 sha512_dround(35, v0, v1, v2, v3, v4, v31, v27, v15, v0, v0, v0, v0);
4717 sha512_dround(36, v3, v0, v4, v2, v1, v24, v0, v16, v0, v0, v0, v0);
4718 sha512_dround(37, v2, v3, v1, v4, v0, v25, v0, v17, v0, v0, v0, v0);
4719 sha512_dround(38, v4, v2, v0, v1, v3, v26, v0, v18, v0, v0, v0, v0);
4720 sha512_dround(39, v1, v4, v3, v0, v2, v27, v0, v19, v0, v0, v0, v0);
4721
4722 __ addv(v8, __ T2D, v8, v0);
4723 __ addv(v9, __ T2D, v9, v1);
4724 __ addv(v10, __ T2D, v10, v2);
4725 __ addv(v11, __ T2D, v11, v3);
4726
4727 if (multi_block) {
4728 __ add(ofs, ofs, 128);
4729 __ cmp(ofs, limit);
4730 __ br(Assembler::LE, sha512_loop);
4731 __ mov(c_rarg0, ofs); // return ofs
4732 }
4733
4734 __ st1(v8, v9, v10, v11, __ T2D, state);
4735
4736 __ ldpd(v14, v15, Address(sp, 48));
4737 __ ldpd(v12, v13, Address(sp, 32));
4738 __ ldpd(v10, v11, Address(sp, 16));
4739 __ ldpd(v8, v9, __ post(sp, 64));
4740
4741 __ ret(lr);
4742
4743 // record the stub entry and end
4744 store_archive_data(stub_id, start, __ pc());
4745
4746 return start;
4747 }
4748
4749 // Execute one round of keccak of two computations in parallel.
4750 // One of the states should be loaded into the lower halves of
4751 // the vector registers v0-v24, the other should be loaded into
4752 // the upper halves of those registers. The ld1r instruction loads
4753 // the round constant into both halves of register v31.
4754 // Intermediate results c0...c5 and d0...d5 are computed
4755 // in registers v25...v30.
4756 // All vector instructions that are used operate on both register
4757 // halves in parallel.
4758 // If only a single computation is needed, one can only load the lower halves.
4759 void keccak_round(Register rscratch1) {
4760 __ eor3(v29, __ T16B, v4, v9, v14); // c4 = a4 ^ a9 ^ a14
4761 __ eor3(v26, __ T16B, v1, v6, v11); // c1 = a1 ^ a16 ^ a11
4762 __ eor3(v28, __ T16B, v3, v8, v13); // c3 = a3 ^ a8 ^a13
4763 __ eor3(v25, __ T16B, v0, v5, v10); // c0 = a0 ^ a5 ^ a10
4764 __ eor3(v27, __ T16B, v2, v7, v12); // c2 = a2 ^ a7 ^ a12
4765 __ eor3(v29, __ T16B, v29, v19, v24); // c4 ^= a19 ^ a24
4766 __ eor3(v26, __ T16B, v26, v16, v21); // c1 ^= a16 ^ a21
4767 __ eor3(v28, __ T16B, v28, v18, v23); // c3 ^= a18 ^ a23
4768 __ eor3(v25, __ T16B, v25, v15, v20); // c0 ^= a15 ^ a20
4769 __ eor3(v27, __ T16B, v27, v17, v22); // c2 ^= a17 ^ a22
4770
4771 __ rax1(v30, __ T2D, v29, v26); // d0 = c4 ^ rol(c1, 1)
4772 __ rax1(v26, __ T2D, v26, v28); // d2 = c1 ^ rol(c3, 1)
4773 __ rax1(v28, __ T2D, v28, v25); // d4 = c3 ^ rol(c0, 1)
4774 __ rax1(v25, __ T2D, v25, v27); // d1 = c0 ^ rol(c2, 1)
4775 __ rax1(v27, __ T2D, v27, v29); // d3 = c2 ^ rol(c4, 1)
4776
4777 __ eor(v0, __ T16B, v0, v30); // a0 = a0 ^ d0
4778 __ xar(v29, __ T2D, v1, v25, (64 - 1)); // a10' = rol((a1^d1), 1)
4779 __ xar(v1, __ T2D, v6, v25, (64 - 44)); // a1 = rol(a6^d1), 44)
4780 __ xar(v6, __ T2D, v9, v28, (64 - 20)); // a6 = rol((a9^d4), 20)
4781 __ xar(v9, __ T2D, v22, v26, (64 - 61)); // a9 = rol((a22^d2), 61)
4782 __ xar(v22, __ T2D, v14, v28, (64 - 39)); // a22 = rol((a14^d4), 39)
4783 __ xar(v14, __ T2D, v20, v30, (64 - 18)); // a14 = rol((a20^d0), 18)
4784 __ xar(v31, __ T2D, v2, v26, (64 - 62)); // a20' = rol((a2^d2), 62)
4785 __ xar(v2, __ T2D, v12, v26, (64 - 43)); // a2 = rol((a12^d2), 43)
4786 __ xar(v12, __ T2D, v13, v27, (64 - 25)); // a12 = rol((a13^d3), 25)
4787 __ xar(v13, __ T2D, v19, v28, (64 - 8)); // a13 = rol((a19^d4), 8)
4788 __ xar(v19, __ T2D, v23, v27, (64 - 56)); // a19 = rol((a23^d3), 56)
4789 __ xar(v23, __ T2D, v15, v30, (64 - 41)); // a23 = rol((a15^d0), 41)
4790 __ xar(v15, __ T2D, v4, v28, (64 - 27)); // a15 = rol((a4^d4), 27)
4791 __ xar(v28, __ T2D, v24, v28, (64 - 14)); // a4' = rol((a24^d4), 14)
4792 __ xar(v24, __ T2D, v21, v25, (64 - 2)); // a24 = rol((a21^d1), 2)
4793 __ xar(v8, __ T2D, v8, v27, (64 - 55)); // a21' = rol((a8^d3), 55)
4794 __ xar(v4, __ T2D, v16, v25, (64 - 45)); // a8' = rol((a16^d1), 45)
4795 __ xar(v16, __ T2D, v5, v30, (64 - 36)); // a16 = rol((a5^d0), 36)
4796 __ xar(v5, __ T2D, v3, v27, (64 - 28)); // a5 = rol((a3^d3), 28)
4797 __ xar(v27, __ T2D, v18, v27, (64 - 21)); // a3' = rol((a18^d3), 21)
4798 __ xar(v3, __ T2D, v17, v26, (64 - 15)); // a18' = rol((a17^d2), 15)
4799 __ xar(v25, __ T2D, v11, v25, (64 - 10)); // a17' = rol((a11^d1), 10)
4800 __ xar(v26, __ T2D, v7, v26, (64 - 6)); // a11' = rol((a7^d2), 6)
4801 __ xar(v30, __ T2D, v10, v30, (64 - 3)); // a7' = rol((a10^d0), 3)
4802
4803 __ bcax(v20, __ T16B, v31, v22, v8); // a20 = a20' ^ (~a21 & a22')
4804 __ bcax(v21, __ T16B, v8, v23, v22); // a21 = a21' ^ (~a22 & a23)
4805 __ bcax(v22, __ T16B, v22, v24, v23); // a22 = a22 ^ (~a23 & a24)
4806 __ bcax(v23, __ T16B, v23, v31, v24); // a23 = a23 ^ (~a24 & a20')
4807 __ bcax(v24, __ T16B, v24, v8, v31); // a24 = a24 ^ (~a20' & a21')
4808
4809 __ ld1r(v31, __ T2D, __ post(rscratch1, 8)); // rc = round_constants[i]
4810
4811 __ bcax(v17, __ T16B, v25, v19, v3); // a17 = a17' ^ (~a18' & a19)
4812 __ bcax(v18, __ T16B, v3, v15, v19); // a18 = a18' ^ (~a19 & a15')
4813 __ bcax(v19, __ T16B, v19, v16, v15); // a19 = a19 ^ (~a15 & a16)
4814 __ bcax(v15, __ T16B, v15, v25, v16); // a15 = a15 ^ (~a16 & a17')
4815 __ bcax(v16, __ T16B, v16, v3, v25); // a16 = a16 ^ (~a17' & a18')
4816
4817 __ bcax(v10, __ T16B, v29, v12, v26); // a10 = a10' ^ (~a11' & a12)
4818 __ bcax(v11, __ T16B, v26, v13, v12); // a11 = a11' ^ (~a12 & a13)
4819 __ bcax(v12, __ T16B, v12, v14, v13); // a12 = a12 ^ (~a13 & a14)
4820 __ bcax(v13, __ T16B, v13, v29, v14); // a13 = a13 ^ (~a14 & a10')
4821 __ bcax(v14, __ T16B, v14, v26, v29); // a14 = a14 ^ (~a10' & a11')
4822
4823 __ bcax(v7, __ T16B, v30, v9, v4); // a7 = a7' ^ (~a8' & a9)
4824 __ bcax(v8, __ T16B, v4, v5, v9); // a8 = a8' ^ (~a9 & a5)
4825 __ bcax(v9, __ T16B, v9, v6, v5); // a9 = a9 ^ (~a5 & a6)
4826 __ bcax(v5, __ T16B, v5, v30, v6); // a5 = a5 ^ (~a6 & a7)
4827 __ bcax(v6, __ T16B, v6, v4, v30); // a6 = a6 ^ (~a7 & a8')
4828
4829 __ bcax(v3, __ T16B, v27, v0, v28); // a3 = a3' ^ (~a4' & a0)
4830 __ bcax(v4, __ T16B, v28, v1, v0); // a4 = a4' ^ (~a0 & a1)
4831 __ bcax(v0, __ T16B, v0, v2, v1); // a0 = a0 ^ (~a1 & a2)
4832 __ bcax(v1, __ T16B, v1, v27, v2); // a1 = a1 ^ (~a2 & a3)
4833 __ bcax(v2, __ T16B, v2, v28, v27); // a2 = a2 ^ (~a3 & a4')
4834
4835 __ eor(v0, __ T16B, v0, v31); // a0 = a0 ^ rc
4836 }
4837
4838 // Arguments:
4839 //
4840 // Inputs:
4841 // c_rarg0 - byte[] source+offset
4842 // c_rarg1 - byte[] SHA.state
4843 // c_rarg2 - int block_size
4844 // c_rarg3 - int offset
4845 // c_rarg4 - int limit
4846 //
4847 address generate_sha3_implCompress(StubId stub_id) {
4848 bool multi_block;
4849 switch (stub_id) {
4850 case StubId::stubgen_sha3_implCompress_id:
4851 multi_block = false;
4852 break;
4853 case StubId::stubgen_sha3_implCompressMB_id:
4854 multi_block = true;
4855 break;
4856 default:
4857 ShouldNotReachHere();
4858 }
4859 int entry_count = StubInfo::entry_count(stub_id);
4860 assert(entry_count == 1, "sanity check");
4861 address start = load_archive_data(stub_id);
4862 if (start != nullptr) {
4863 return start;
4864 }
4865 __ align(CodeEntryAlignment);
4866 StubCodeMark mark(this, stub_id);
4867 start = __ pc();
4868
4869 Register buf = c_rarg0;
4870 Register state = c_rarg1;
4871 Register block_size = c_rarg2;
4872 Register ofs = c_rarg3;
4873 Register limit = c_rarg4;
4874
4875 Label sha3_loop, rounds24_loop;
4876 Label sha3_512_or_sha3_384, shake128;
4877
4878 __ stpd(v8, v9, __ pre(sp, -64));
4879 __ stpd(v10, v11, Address(sp, 16));
4880 __ stpd(v12, v13, Address(sp, 32));
4881 __ stpd(v14, v15, Address(sp, 48));
4882
4883 // load state
4884 __ add(rscratch1, state, 32);
4885 __ ld1(v0, v1, v2, v3, __ T1D, state);
4886 __ ld1(v4, v5, v6, v7, __ T1D, __ post(rscratch1, 32));
4887 __ ld1(v8, v9, v10, v11, __ T1D, __ post(rscratch1, 32));
4888 __ ld1(v12, v13, v14, v15, __ T1D, __ post(rscratch1, 32));
4889 __ ld1(v16, v17, v18, v19, __ T1D, __ post(rscratch1, 32));
4890 __ ld1(v20, v21, v22, v23, __ T1D, __ post(rscratch1, 32));
4891 __ ld1(v24, __ T1D, rscratch1);
4892
4893 __ BIND(sha3_loop);
4894
4895 // 24 keccak rounds
4896 __ movw(rscratch2, 24);
4897
4898 // load round_constants base
4899 __ lea(rscratch1, ExternalAddress((address) _sha3_round_consts));
4900
4901 // load input
4902 __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32));
4903 __ ld1(v29, v30, v31, __ T8B, __ post(buf, 24));
4904 __ eor(v0, __ T8B, v0, v25);
4905 __ eor(v1, __ T8B, v1, v26);
4906 __ eor(v2, __ T8B, v2, v27);
4907 __ eor(v3, __ T8B, v3, v28);
4908 __ eor(v4, __ T8B, v4, v29);
4909 __ eor(v5, __ T8B, v5, v30);
4910 __ eor(v6, __ T8B, v6, v31);
4911
4912 // block_size == 72, SHA3-512; block_size == 104, SHA3-384
4913 __ tbz(block_size, 7, sha3_512_or_sha3_384);
4914
4915 __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32));
4916 __ ld1(v29, v30, v31, __ T8B, __ post(buf, 24));
4917 __ eor(v7, __ T8B, v7, v25);
4918 __ eor(v8, __ T8B, v8, v26);
4919 __ eor(v9, __ T8B, v9, v27);
4920 __ eor(v10, __ T8B, v10, v28);
4921 __ eor(v11, __ T8B, v11, v29);
4922 __ eor(v12, __ T8B, v12, v30);
4923 __ eor(v13, __ T8B, v13, v31);
4924
4925 __ ld1(v25, v26, v27, __ T8B, __ post(buf, 24));
4926 __ eor(v14, __ T8B, v14, v25);
4927 __ eor(v15, __ T8B, v15, v26);
4928 __ eor(v16, __ T8B, v16, v27);
4929
4930 // block_size == 136, bit4 == 0 and bit5 == 0, SHA3-256 or SHAKE256
4931 __ andw(c_rarg5, block_size, 48);
4932 __ cbzw(c_rarg5, rounds24_loop);
4933
4934 __ tbnz(block_size, 5, shake128);
4935 // block_size == 144, bit5 == 0, SHA3-224
4936 __ ldrd(v28, __ post(buf, 8));
4937 __ eor(v17, __ T8B, v17, v28);
4938 __ b(rounds24_loop);
4939
4940 __ BIND(shake128);
4941 __ ld1(v28, v29, v30, v31, __ T8B, __ post(buf, 32));
4942 __ eor(v17, __ T8B, v17, v28);
4943 __ eor(v18, __ T8B, v18, v29);
4944 __ eor(v19, __ T8B, v19, v30);
4945 __ eor(v20, __ T8B, v20, v31);
4946 __ b(rounds24_loop); // block_size == 168, SHAKE128
4947
4948 __ BIND(sha3_512_or_sha3_384);
4949 __ ld1(v25, v26, __ T8B, __ post(buf, 16));
4950 __ eor(v7, __ T8B, v7, v25);
4951 __ eor(v8, __ T8B, v8, v26);
4952 __ tbz(block_size, 5, rounds24_loop); // SHA3-512
4953
4954 // SHA3-384
4955 __ ld1(v27, v28, v29, v30, __ T8B, __ post(buf, 32));
4956 __ eor(v9, __ T8B, v9, v27);
4957 __ eor(v10, __ T8B, v10, v28);
4958 __ eor(v11, __ T8B, v11, v29);
4959 __ eor(v12, __ T8B, v12, v30);
4960
4961 __ BIND(rounds24_loop);
4962 __ subw(rscratch2, rscratch2, 1);
4963
4964 keccak_round(rscratch1);
4965
4966 __ cbnzw(rscratch2, rounds24_loop);
4967
4968 if (multi_block) {
4969 __ add(ofs, ofs, block_size);
4970 __ cmp(ofs, limit);
4971 __ br(Assembler::LE, sha3_loop);
4972 __ mov(c_rarg0, ofs); // return ofs
4973 }
4974
4975 __ st1(v0, v1, v2, v3, __ T1D, __ post(state, 32));
4976 __ st1(v4, v5, v6, v7, __ T1D, __ post(state, 32));
4977 __ st1(v8, v9, v10, v11, __ T1D, __ post(state, 32));
4978 __ st1(v12, v13, v14, v15, __ T1D, __ post(state, 32));
4979 __ st1(v16, v17, v18, v19, __ T1D, __ post(state, 32));
4980 __ st1(v20, v21, v22, v23, __ T1D, __ post(state, 32));
4981 __ st1(v24, __ T1D, state);
4982
4983 // restore callee-saved registers
4984 __ ldpd(v14, v15, Address(sp, 48));
4985 __ ldpd(v12, v13, Address(sp, 32));
4986 __ ldpd(v10, v11, Address(sp, 16));
4987 __ ldpd(v8, v9, __ post(sp, 64));
4988
4989 __ ret(lr);
4990
4991 // record the stub entry and end
4992 store_archive_data(stub_id, start, __ pc());
4993
4994 return start;
4995 }
4996
4997 // Inputs:
4998 // c_rarg0 - long[] state0
4999 // c_rarg1 - long[] state1
5000 address generate_double_keccak() {
5001 StubId stub_id = StubId::stubgen_double_keccak_id;
5002 int entry_count = StubInfo::entry_count(stub_id);
5003 assert(entry_count == 1, "sanity check");
5004 address start = load_archive_data(stub_id);
5005 if (start != nullptr) {
5006 return start;
5007 }
5008 // Implements the double_keccak() method of the
5009 // sun.security.provider.SHA3Parallel class
5010 __ align(CodeEntryAlignment);
5011 StubCodeMark mark(this, stub_id);
5012 start = __ pc();
5013 __ enter();
5014
5015 Register state0 = c_rarg0;
5016 Register state1 = c_rarg1;
5017
5018 Label rounds24_loop;
5019
5020 // save callee-saved registers
5021 __ stpd(v8, v9, __ pre(sp, -64));
5022 __ stpd(v10, v11, Address(sp, 16));
5023 __ stpd(v12, v13, Address(sp, 32));
5024 __ stpd(v14, v15, Address(sp, 48));
5025
5026 // load states
5027 __ add(rscratch1, state0, 32);
5028 __ ld4(v0, v1, v2, v3, __ D, 0, state0);
5029 __ ld4(v4, v5, v6, v7, __ D, 0, __ post(rscratch1, 32));
5030 __ ld4(v8, v9, v10, v11, __ D, 0, __ post(rscratch1, 32));
5031 __ ld4(v12, v13, v14, v15, __ D, 0, __ post(rscratch1, 32));
5032 __ ld4(v16, v17, v18, v19, __ D, 0, __ post(rscratch1, 32));
5033 __ ld4(v20, v21, v22, v23, __ D, 0, __ post(rscratch1, 32));
5034 __ ld1(v24, __ D, 0, rscratch1);
5035 __ add(rscratch1, state1, 32);
5036 __ ld4(v0, v1, v2, v3, __ D, 1, state1);
5037 __ ld4(v4, v5, v6, v7, __ D, 1, __ post(rscratch1, 32));
5038 __ ld4(v8, v9, v10, v11, __ D, 1, __ post(rscratch1, 32));
5039 __ ld4(v12, v13, v14, v15, __ D, 1, __ post(rscratch1, 32));
5040 __ ld4(v16, v17, v18, v19, __ D, 1, __ post(rscratch1, 32));
5041 __ ld4(v20, v21, v22, v23, __ D, 1, __ post(rscratch1, 32));
5042 __ ld1(v24, __ D, 1, rscratch1);
5043
5044 // 24 keccak rounds
5045 __ movw(rscratch2, 24);
5046
5047 // load round_constants base
5048 __ lea(rscratch1, ExternalAddress((address) _double_keccak_round_consts));
5049
5050 __ BIND(rounds24_loop);
5051 __ subw(rscratch2, rscratch2, 1);
5052 keccak_round(rscratch1);
5053 __ cbnzw(rscratch2, rounds24_loop);
5054
5055 __ st4(v0, v1, v2, v3, __ D, 0, __ post(state0, 32));
5056 __ st4(v4, v5, v6, v7, __ D, 0, __ post(state0, 32));
5057 __ st4(v8, v9, v10, v11, __ D, 0, __ post(state0, 32));
5058 __ st4(v12, v13, v14, v15, __ D, 0, __ post(state0, 32));
5059 __ st4(v16, v17, v18, v19, __ D, 0, __ post(state0, 32));
5060 __ st4(v20, v21, v22, v23, __ D, 0, __ post(state0, 32));
5061 __ st1(v24, __ D, 0, state0);
5062 __ st4(v0, v1, v2, v3, __ D, 1, __ post(state1, 32));
5063 __ st4(v4, v5, v6, v7, __ D, 1, __ post(state1, 32));
5064 __ st4(v8, v9, v10, v11, __ D, 1, __ post(state1, 32));
5065 __ st4(v12, v13, v14, v15, __ D, 1, __ post(state1, 32));
5066 __ st4(v16, v17, v18, v19, __ D, 1, __ post(state1, 32));
5067 __ st4(v20, v21, v22, v23, __ D, 1, __ post(state1, 32));
5068 __ st1(v24, __ D, 1, state1);
5069
5070 // restore callee-saved vector registers
5071 __ ldpd(v14, v15, Address(sp, 48));
5072 __ ldpd(v12, v13, Address(sp, 32));
5073 __ ldpd(v10, v11, Address(sp, 16));
5074 __ ldpd(v8, v9, __ post(sp, 64));
5075
5076 __ leave(); // required for proper stackwalking of RuntimeStub frame
5077
5078 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
5079 __ ret(lr);
5080
5081 // record the stub entry and end
5082 store_archive_data(stub_id, start, __ pc());
5083
5084 return start;
5085 }
5086
5087 // ChaCha20 block function. This version parallelizes the 32-bit
5088 // state elements on each of 16 vectors, producing 4 blocks of
5089 // keystream at a time.
5090 //
5091 // state (int[16]) = c_rarg0
5092 // keystream (byte[256]) = c_rarg1
5093 // return - number of bytes of produced keystream (always 256)
5094 //
5095 // This implementation takes each 32-bit integer from the state
5096 // array and broadcasts it across all 4 32-bit lanes of a vector register
5097 // (e.g. state[0] is replicated on all 4 lanes of v4, state[1] to all 4 lanes
5098 // of v5, etc.). Once all 16 elements have been broadcast onto 16 vectors,
5099 // the quarter round schedule is implemented as outlined in RFC 7539 section
5100 // 2.3. However, instead of sequentially processing the 3 quarter round
5101 // operations represented by one QUARTERROUND function, we instead stack all
5102 // the adds, xors and left-rotations from the first 4 quarter rounds together
5103 // and then do the same for the second set of 4 quarter rounds. This removes
5104 // some latency that would otherwise be incurred by waiting for an add to
5105 // complete before performing an xor (which depends on the result of the
5106 // add), etc. An adjustment happens between the first and second groups of 4
5107 // quarter rounds, but this is done only in the inputs to the macro functions
5108 // that generate the assembly instructions - these adjustments themselves are
5109 // not part of the resulting assembly.
5110 // The 4 registers v0-v3 are used during the quarter round operations as
5111 // scratch registers. Once the 20 rounds are complete, these 4 scratch
5112 // registers become the vectors involved in adding the start state back onto
5113 // the post-QR working state. After the adds are complete, each of the 16
5114 // vectors write their first lane back to the keystream buffer, followed
5115 // by the second lane from all vectors and so on.
5116 address generate_chacha20Block_blockpar() {
5117 StubId stub_id = StubId::stubgen_chacha20Block_id;
5118 int entry_count = StubInfo::entry_count(stub_id);
5119 assert(entry_count == 1, "sanity check");
5120 address start = load_archive_data(stub_id);
5121 if (start != nullptr) {
5122 return start;
5123 }
5124 Label L_twoRounds, L_cc20_const;
5125 __ align(CodeEntryAlignment);
5126 StubCodeMark mark(this, stub_id);
5127 start = __ pc();
5128 __ enter();
5129
5130 int i, j;
5131 const Register state = c_rarg0;
5132 const Register keystream = c_rarg1;
5133 const Register loopCtr = r10;
5134 const Register tmpAddr = r11;
5135 const FloatRegister ctrAddOverlay = v28;
5136 const FloatRegister lrot8Tbl = v29;
5137
5138 // Organize SIMD registers in an array that facilitates
5139 // putting repetitive opcodes into loop structures. It is
5140 // important that each grouping of 4 registers is monotonically
5141 // increasing to support the requirements of multi-register
5142 // instructions (e.g. ld4r, st4, etc.)
5143 const FloatRegister workSt[16] = {
5144 v4, v5, v6, v7, v16, v17, v18, v19,
5145 v20, v21, v22, v23, v24, v25, v26, v27
5146 };
5147
5148 // Pull in constant data. The first 16 bytes are the add overlay
5149 // which is applied to the vector holding the counter (state[12]).
5150 // The second 16 bytes is the index register for the 8-bit left
5151 // rotation tbl instruction.
5152 __ adr(tmpAddr, L_cc20_const);
5153 __ ldpq(ctrAddOverlay, lrot8Tbl, Address(tmpAddr));
5154
5155 // Load from memory and interlace across 16 SIMD registers,
5156 // With each word from memory being broadcast to all lanes of
5157 // each successive SIMD register.
5158 // Addr(0) -> All lanes in workSt[i]
5159 // Addr(4) -> All lanes workSt[i + 1], etc.
5160 __ mov(tmpAddr, state);
5161 for (i = 0; i < 16; i += 4) {
5162 __ ld4r(workSt[i], workSt[i + 1], workSt[i + 2], workSt[i + 3], __ T4S,
5163 __ post(tmpAddr, 16));
5164 }
5165 __ addv(workSt[12], __ T4S, workSt[12], ctrAddOverlay); // Add ctr overlay
5166
5167 // Before entering the loop, create 5 4-register arrays. These
5168 // will hold the 4 registers that represent the a/b/c/d fields
5169 // in the quarter round operation. For instance the "b" field
5170 // for the first 4 quarter round operations is the set of v16/v17/v18/v19,
5171 // but in the second 4 quarter rounds it gets adjusted to v17/v18/v19/v16
5172 // since it is part of a diagonal organization. The aSet and scratch
5173 // register sets are defined at declaration time because they do not change
5174 // organization at any point during the 20-round processing.
5175 FloatRegister aSet[4] = { v4, v5, v6, v7 };
5176 FloatRegister bSet[4];
5177 FloatRegister cSet[4];
5178 FloatRegister dSet[4];
5179 FloatRegister scratch[4] = { v0, v1, v2, v3 };
5180
5181 // Set up the 10 iteration loop and perform all 8 quarter round ops
5182 __ mov(loopCtr, 10);
5183 __ BIND(L_twoRounds);
5184
5185 // Set to columnar organization and do the following 4 quarter-rounds:
5186 // QUARTERROUND(0, 4, 8, 12)
5187 // QUARTERROUND(1, 5, 9, 13)
5188 // QUARTERROUND(2, 6, 10, 14)
5189 // QUARTERROUND(3, 7, 11, 15)
5190 __ cc20_set_qr_registers(bSet, workSt, 4, 5, 6, 7);
5191 __ cc20_set_qr_registers(cSet, workSt, 8, 9, 10, 11);
5192 __ cc20_set_qr_registers(dSet, workSt, 12, 13, 14, 15);
5193
5194 __ cc20_qr_add4(aSet, bSet); // a += b
5195 __ cc20_qr_xor4(dSet, aSet, dSet); // d ^= a
5196 __ cc20_qr_lrot4(dSet, dSet, 16, lrot8Tbl); // d <<<= 16
5197
5198 __ cc20_qr_add4(cSet, dSet); // c += d
5199 __ cc20_qr_xor4(bSet, cSet, scratch); // b ^= c (scratch)
5200 __ cc20_qr_lrot4(scratch, bSet, 12, lrot8Tbl); // b <<<= 12
5201
5202 __ cc20_qr_add4(aSet, bSet); // a += b
5203 __ cc20_qr_xor4(dSet, aSet, dSet); // d ^= a
5204 __ cc20_qr_lrot4(dSet, dSet, 8, lrot8Tbl); // d <<<= 8
5205
5206 __ cc20_qr_add4(cSet, dSet); // c += d
5207 __ cc20_qr_xor4(bSet, cSet, scratch); // b ^= c (scratch)
5208 __ cc20_qr_lrot4(scratch, bSet, 7, lrot8Tbl); // b <<<= 12
5209
5210 // Set to diagonal organization and do the next 4 quarter-rounds:
5211 // QUARTERROUND(0, 5, 10, 15)
5212 // QUARTERROUND(1, 6, 11, 12)
5213 // QUARTERROUND(2, 7, 8, 13)
5214 // QUARTERROUND(3, 4, 9, 14)
5215 __ cc20_set_qr_registers(bSet, workSt, 5, 6, 7, 4);
5216 __ cc20_set_qr_registers(cSet, workSt, 10, 11, 8, 9);
5217 __ cc20_set_qr_registers(dSet, workSt, 15, 12, 13, 14);
5218
5219 __ cc20_qr_add4(aSet, bSet); // a += b
5220 __ cc20_qr_xor4(dSet, aSet, dSet); // d ^= a
5221 __ cc20_qr_lrot4(dSet, dSet, 16, lrot8Tbl); // d <<<= 16
5222
5223 __ cc20_qr_add4(cSet, dSet); // c += d
5224 __ cc20_qr_xor4(bSet, cSet, scratch); // b ^= c (scratch)
5225 __ cc20_qr_lrot4(scratch, bSet, 12, lrot8Tbl); // b <<<= 12
5226
5227 __ cc20_qr_add4(aSet, bSet); // a += b
5228 __ cc20_qr_xor4(dSet, aSet, dSet); // d ^= a
5229 __ cc20_qr_lrot4(dSet, dSet, 8, lrot8Tbl); // d <<<= 8
5230
5231 __ cc20_qr_add4(cSet, dSet); // c += d
5232 __ cc20_qr_xor4(bSet, cSet, scratch); // b ^= c (scratch)
5233 __ cc20_qr_lrot4(scratch, bSet, 7, lrot8Tbl); // b <<<= 12
5234
5235 // Decrement and iterate
5236 __ sub(loopCtr, loopCtr, 1);
5237 __ cbnz(loopCtr, L_twoRounds);
5238
5239 __ mov(tmpAddr, state);
5240
5241 // Add the starting state back to the post-loop keystream
5242 // state. We read/interlace the state array from memory into
5243 // 4 registers similar to what we did in the beginning. Then
5244 // add the counter overlay onto workSt[12] at the end.
5245 for (i = 0; i < 16; i += 4) {
5246 __ ld4r(v0, v1, v2, v3, __ T4S, __ post(tmpAddr, 16));
5247 __ addv(workSt[i], __ T4S, workSt[i], v0);
5248 __ addv(workSt[i + 1], __ T4S, workSt[i + 1], v1);
5249 __ addv(workSt[i + 2], __ T4S, workSt[i + 2], v2);
5250 __ addv(workSt[i + 3], __ T4S, workSt[i + 3], v3);
5251 }
5252 __ addv(workSt[12], __ T4S, workSt[12], ctrAddOverlay); // Add ctr overlay
5253
5254 // Write working state into the keystream buffer. This is accomplished
5255 // by taking the lane "i" from each of the four vectors and writing
5256 // it to consecutive 4-byte offsets, then post-incrementing by 16 and
5257 // repeating with the next 4 vectors until all 16 vectors have been used.
5258 // Then move to the next lane and repeat the process until all lanes have
5259 // been written.
5260 for (i = 0; i < 4; i++) {
5261 for (j = 0; j < 16; j += 4) {
5262 __ st4(workSt[j], workSt[j + 1], workSt[j + 2], workSt[j + 3], __ S, i,
5263 __ post(keystream, 16));
5264 }
5265 }
5266
5267 __ mov(r0, 256); // Return length of output keystream
5268 __ leave();
5269 __ ret(lr);
5270
5271 // bind label and generate local constant data used by this stub
5272 // The constant data is broken into two 128-bit segments to be loaded
5273 // onto FloatRegisters. The first 128 bits are a counter add overlay
5274 // that adds +0/+1/+2/+3 to the vector holding replicated state[12].
5275 // The second 128-bits is a table constant used for 8-bit left rotations.
5276 __ BIND(L_cc20_const);
5277 __ emit_int64(0x0000000100000000UL);
5278 __ emit_int64(0x0000000300000002UL);
5279 __ emit_int64(0x0605040702010003UL);
5280 __ emit_int64(0x0E0D0C0F0A09080BUL);
5281
5282 // record the stub entry and end
5283 store_archive_data(stub_id, start, __ pc());
5284
5285 return start;
5286 }
5287
5288 // Helpers to schedule parallel operation bundles across vector
5289 // register sequences of size 2, 4 or 8.
5290
5291 // Implement various primitive computations across vector sequences
5292
5293 template<int N>
5294 void vs_addv(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5295 const VSeq<N>& v1, const VSeq<N>& v2) {
5296 // output must not be constant
5297 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5298 // output cannot overwrite pending inputs
5299 assert(!vs_write_before_read(v, v1), "output overwrites input");
5300 assert(!vs_write_before_read(v, v2), "output overwrites input");
5301 for (int i = 0; i < N; i++) {
5302 __ addv(v[i], T, v1[i], v2[i]);
5303 }
5304 }
5305
5306 template<int N>
5307 void vs_subv(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5308 const VSeq<N>& v1, const VSeq<N>& v2) {
5309 // output must not be constant
5310 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5311 // output cannot overwrite pending inputs
5312 assert(!vs_write_before_read(v, v1), "output overwrites input");
5313 assert(!vs_write_before_read(v, v2), "output overwrites input");
5314 for (int i = 0; i < N; i++) {
5315 __ subv(v[i], T, v1[i], v2[i]);
5316 }
5317 }
5318
5319 template<int N>
5320 void vs_mulv(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5321 const VSeq<N>& v1, const VSeq<N>& v2) {
5322 // output must not be constant
5323 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5324 // output cannot overwrite pending inputs
5325 assert(!vs_write_before_read(v, v1), "output overwrites input");
5326 assert(!vs_write_before_read(v, v2), "output overwrites input");
5327 for (int i = 0; i < N; i++) {
5328 __ mulv(v[i], T, v1[i], v2[i]);
5329 }
5330 }
5331
5332 template<int N>
5333 void vs_negr(const VSeq<N>& v, Assembler::SIMD_Arrangement T, const VSeq<N>& v1) {
5334 // output must not be constant
5335 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5336 // output cannot overwrite pending inputs
5337 assert(!vs_write_before_read(v, v1), "output overwrites input");
5338 for (int i = 0; i < N; i++) {
5339 __ negr(v[i], T, v1[i]);
5340 }
5341 }
5342
5343 template<int N>
5344 void vs_shl(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5345 const VSeq<N>& v1, int shift) {
5346 // output must not be constant
5347 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5348 // output cannot overwrite pending inputs
5349 assert(!vs_write_before_read(v, v1), "output overwrites input");
5350
5351 for (int i = 0; i < N; i++) {
5352 __ shl(v[i], T, v1[i], shift);
5353 }
5354 }
5355
5356 template<int N>
5357 void vs_ushr(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5358 const VSeq<N>& v1, int shift) {
5359 // output must not be constant
5360 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5361 // output cannot overwrite pending inputs
5362 assert(!vs_write_before_read(v, v1), "output overwrites input");
5363
5364 for (int i = 0; i < N; i++) {
5365 __ ushr(v[i], T, v1[i], shift);
5366 }
5367 }
5368
5369 template<int N>
5370 void vs_sshr(const VSeq<N>& v, Assembler::SIMD_Arrangement T,
5371 const VSeq<N>& v1, int shift) {
5372 // output must not be constant
5373 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5374 // output cannot overwrite pending inputs
5375 assert(!vs_write_before_read(v, v1), "output overwrites input");
5376 for (int i = 0; i < N; i++) {
5377 __ sshr(v[i], T, v1[i], shift);
5378 }
5379 }
5380
5381 template<int N>
5382 void vs_andr(const VSeq<N>& v, const VSeq<N>& v1, const VSeq<N>& v2) {
5383 // output must not be constant
5384 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5385 // output cannot overwrite pending inputs
5386 assert(!vs_write_before_read(v, v1), "output overwrites input");
5387 assert(!vs_write_before_read(v, v2), "output overwrites input");
5388 for (int i = 0; i < N; i++) {
5389 __ andr(v[i], __ T16B, v1[i], v2[i]);
5390 }
5391 }
5392
5393 template<int N>
5394 void vs_andr(const VSeq<N>& v, const VSeq<N>& v1, const FloatRegister v2) {
5395 // output must not be constant
5396 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5397 // output cannot overwrite pending inputs
5398 assert(!vs_write_before_read(v, v1), "output overwrites input");
5399 for (int i = 0; i < N; i++) {
5400 __ andr(v[i], __ T16B, v1[i], v2);
5401 }
5402 }
5403
5404 template<int N>
5405 void vs_eor(const VSeq<N>& v, const VSeq<N>& v1, const VSeq<N>& v2) {
5406 // output must not be constant
5407 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5408 // output cannot overwrite pending inputs
5409 assert(!vs_write_before_read(v, v1), "output overwrites input");
5410 assert(!vs_write_before_read(v, v2), "output overwrites input");
5411 for (int i = 0; i < N; i++) {
5412 __ eor(v[i], __ T16B, v1[i], v2[i]);
5413 }
5414 }
5415
5416 template<int N>
5417 void vs_orr(const VSeq<N>& v, const VSeq<N>& v1, const VSeq<N>& v2) {
5418 // output must not be constant
5419 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5420 // output cannot overwrite pending inputs
5421 assert(!vs_write_before_read(v, v1), "output overwrites input");
5422 assert(!vs_write_before_read(v, v2), "output overwrites input");
5423 for (int i = 0; i < N; i++) {
5424 __ orr(v[i], __ T16B, v1[i], v2[i]);
5425 }
5426 }
5427
5428 template<int N>
5429 void vs_notr(const VSeq<N>& v, const VSeq<N>& v1) {
5430 // output must not be constant
5431 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5432 // output cannot overwrite pending inputs
5433 assert(!vs_write_before_read(v, v1), "output overwrites input");
5434 for (int i = 0; i < N; i++) {
5435 __ notr(v[i], __ T16B, v1[i]);
5436 }
5437 }
5438
5439 template<int N>
5440 void vs_sqdmulh(const VSeq<N>& v, Assembler::SIMD_Arrangement T, const VSeq<N>& v1, const VSeq<N>& v2) {
5441 // output must not be constant
5442 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5443 // output cannot overwrite pending inputs
5444 assert(!vs_write_before_read(v, v1), "output overwrites input");
5445 assert(!vs_write_before_read(v, v2), "output overwrites input");
5446 for (int i = 0; i < N; i++) {
5447 __ sqdmulh(v[i], T, v1[i], v2[i]);
5448 }
5449 }
5450
5451 template<int N>
5452 void vs_mlsv(const VSeq<N>& v, Assembler::SIMD_Arrangement T, const VSeq<N>& v1, VSeq<N>& v2) {
5453 // output must not be constant
5454 assert(N == 1 || !v.is_constant(), "cannot output multiple values to a constant vector");
5455 // output cannot overwrite pending inputs
5456 assert(!vs_write_before_read(v, v1), "output overwrites input");
5457 assert(!vs_write_before_read(v, v2), "output overwrites input");
5458 for (int i = 0; i < N; i++) {
5459 __ mlsv(v[i], T, v1[i], v2[i]);
5460 }
5461 }
5462
5463 // load N/2 successive pairs of quadword values from memory in order
5464 // into N successive vector registers of the sequence via the
5465 // address supplied in base.
5466 template<int N>
5467 void vs_ldpq(const VSeq<N>& v, Register base) {
5468 static_assert(N > 0 && is_even(N), "sequence length must be even");
5469 for (int i = 0; i < N; i += 2) {
5470 __ ldpq(v[i], v[i+1], Address(base, 16 * i));
5471 }
5472 }
5473
5474 // load N/2 successive pairs of quadword values from memory in order
5475 // into N vector registers of the sequence via the address supplied
5476 // in base using post-increment addressing
5477 template<int N>
5478 void vs_ldpq_post(const VSeq<N>& v, Register base) {
5479 static_assert(N > 0 && is_even(N), "sequence length must be even");
5480 for (int i = 0; i < N; i += 2) {
5481 __ ldpq(v[i], v[i+1], __ post(base, 32));
5482 }
5483 }
5484
5485 // store N successive vector registers of the sequence into N/2
5486 // successive pairs of quadword memory locations via the address
5487 // supplied in base using post-increment addressing
5488 template<int N>
5489 void vs_stpq_post(const VSeq<N>& v, Register base) {
5490 static_assert(N > 0 && is_even(N), "sequence length must be even");
5491 for (int i = 0; i < N; i += 2) {
5492 __ stpq(v[i], v[i+1], __ post(base, 32));
5493 }
5494 }
5495
5496 // load N/2 pairs of quadword values from memory de-interleaved into
5497 // N vector registers 2 at a time via the address supplied in base
5498 // using post-increment addressing.
5499 template<int N>
5500 void vs_ld2_post(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base) {
5501 static_assert(N > 0 && is_even(N), "sequence length must be even");
5502 for (int i = 0; i < N; i += 2) {
5503 __ ld2(v[i], v[i+1], T, __ post(base, 32));
5504 }
5505 }
5506
5507 // store N vector registers interleaved into N/2 pairs of quadword
5508 // memory locations via the address supplied in base using
5509 // post-increment addressing.
5510 template<int N>
5511 void vs_st2_post(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base) {
5512 static_assert(N > 0 && is_even(N), "sequence length must be even");
5513 for (int i = 0; i < N; i += 2) {
5514 __ st2(v[i], v[i+1], T, __ post(base, 32));
5515 }
5516 }
5517
5518 // store two vector register sequences of length N
5519 // interleaved into N pairs of quadword memory locations
5520 // starting at the address supplied in dest using
5521 // post-increment addressing.
5522 template<int N>
5523 void vs_st1_interleaved(VSeq<N> A, VSeq<N> B, Register dest) {
5524 for (int i = 0; i < N; i++) {
5525 __ st1(A[i], __ T2D, __ post(dest, 16));
5526 __ st1(B[i], __ T2D, __ post(dest, 16));
5527 }
5528 }
5529
5530 // load N quadword values from memory de-interleaved into N vector
5531 // registers 3 elements at a time via the address supplied in base.
5532 template<int N>
5533 void vs_ld3(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base) {
5534 static_assert(N == ((N / 3) * 3), "sequence length must be multiple of 3");
5535 for (int i = 0; i < N; i += 3) {
5536 __ ld3(v[i], v[i+1], v[i+2], T, base);
5537 }
5538 }
5539
5540 // load N quadword values from memory de-interleaved into N vector
5541 // registers 3 elements at a time via the address supplied in base
5542 // using post-increment addressing.
5543 template<int N>
5544 void vs_ld3_post(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base) {
5545 static_assert(N == ((N / 3) * 3), "sequence length must be multiple of 3");
5546 for (int i = 0; i < N; i += 3) {
5547 __ ld3(v[i], v[i+1], v[i+2], T, __ post(base, 48));
5548 }
5549 }
5550
5551 // load N/2 pairs of quadword values from memory into N vector
5552 // registers via the address supplied in base with each pair indexed
5553 // using the start offset plus the corresponding entry in the
5554 // offsets array
5555 template<int N>
5556 void vs_ldpq_indexed(const VSeq<N>& v, Register base, int start, int (&offsets)[N/2]) {
5557 static_assert(N > 0 && is_even(N), "sequence length must be even");
5558 for (int i = 0; i < N/2; i++) {
5559 __ ldpq(v[2*i], v[2*i+1], Address(base, start + offsets[i]));
5560 }
5561 }
5562
5563 // store N vector registers into N/2 pairs of quadword memory
5564 // locations via the address supplied in base with each pair indexed
5565 // using the start offset plus the corresponding entry in the
5566 // offsets array
5567 template<int N>
5568 void vs_stpq_indexed(const VSeq<N>& v, Register base, int start, int offsets[N/2]) {
5569 for (int i = 0; i < N/2; i++) {
5570 __ stpq(v[2*i], v[2*i+1], Address(base, start + offsets[i]));
5571 }
5572 }
5573
5574 // load N single quadword values from memory into N vector registers
5575 // via the address supplied in base with each value indexed using
5576 // the start offset plus the corresponding entry in the offsets
5577 // array
5578 template<int N>
5579 void vs_ldr_indexed(const VSeq<N>& v, Assembler::SIMD_RegVariant T, Register base,
5580 int start, int (&offsets)[N]) {
5581 for (int i = 0; i < N; i++) {
5582 __ ldr(v[i], T, Address(base, start + offsets[i]));
5583 }
5584 }
5585
5586 // store N vector registers into N single quadword memory locations
5587 // via the address supplied in base with each value indexed using
5588 // the start offset plus the corresponding entry in the offsets
5589 // array
5590 template<int N>
5591 void vs_str_indexed(const VSeq<N>& v, Assembler::SIMD_RegVariant T, Register base,
5592 int start, int (&offsets)[N]) {
5593 for (int i = 0; i < N; i++) {
5594 __ str(v[i], T, Address(base, start + offsets[i]));
5595 }
5596 }
5597
5598 // load N/2 pairs of quadword values from memory de-interleaved into
5599 // N vector registers 2 at a time via the address supplied in base
5600 // with each pair indexed using the start offset plus the
5601 // corresponding entry in the offsets array
5602 template<int N>
5603 void vs_ld2_indexed(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base,
5604 Register tmp, int start, int (&offsets)[N/2]) {
5605 static_assert(N > 0 && is_even(N), "sequence length must be even");
5606 for (int i = 0; i < N/2; i++) {
5607 __ add(tmp, base, start + offsets[i]);
5608 __ ld2(v[2*i], v[2*i+1], T, tmp);
5609 }
5610 }
5611
5612 // store N vector registers 2 at a time interleaved into N/2 pairs
5613 // of quadword memory locations via the address supplied in base
5614 // with each pair indexed using the start offset plus the
5615 // corresponding entry in the offsets array
5616 template<int N>
5617 void vs_st2_indexed(const VSeq<N>& v, Assembler::SIMD_Arrangement T, Register base,
5618 Register tmp, int start, int (&offsets)[N/2]) {
5619 static_assert(N > 0 && is_even(N), "sequence length must be even");
5620 for (int i = 0; i < N/2; i++) {
5621 __ add(tmp, base, start + offsets[i]);
5622 __ st2(v[2*i], v[2*i+1], T, tmp);
5623 }
5624 }
5625
5626 // Helper routines for various flavours of Montgomery multiply
5627
5628 // Perform 16 32-bit (4x4S) or 32 16-bit (4 x 8H) Montgomery
5629 // multiplications in parallel
5630 //
5631
5632 // See the montMul() method of the sun.security.provider.ML_DSA
5633 // class.
5634 //
5635 // Computes 4x4S results or 8x8H results
5636 // a = b * c * 2^MONT_R_BITS mod MONT_Q
5637 // Inputs: vb, vc - 4x4S or 4x8H vector register sequences
5638 // vq - 2x4S or 2x8H constants <MONT_Q, MONT_Q_INV_MOD_R>
5639 // Temps: vtmp - 4x4S or 4x8H vector sequence trashed after call
5640 // Outputs: va - 4x4S or 4x8H vector register sequences
5641 // vb, vc, vtmp and vq must all be disjoint
5642 // va must be disjoint from all other inputs/temps or must equal vc
5643 // va must have a non-zero delta i.e. it must not be a constant vseq.
5644 // n.b. MONT_R_BITS is 16 or 32, so the right shift by it is implicit.
5645 void vs_montmul4(const VSeq<4>& va, const VSeq<4>& vb, const VSeq<4>& vc,
5646 Assembler::SIMD_Arrangement T,
5647 const VSeq<4>& vtmp, const VSeq<2>& vq) {
5648 assert (T == __ T4S || T == __ T8H, "invalid arrangement for montmul");
5649 assert(vs_disjoint(vb, vc), "vb and vc overlap");
5650 assert(vs_disjoint(vb, vq), "vb and vq overlap");
5651 assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap");
5652
5653 assert(vs_disjoint(vc, vq), "vc and vq overlap");
5654 assert(vs_disjoint(vc, vtmp), "vc and vtmp overlap");
5655
5656 assert(vs_disjoint(vq, vtmp), "vq and vtmp overlap");
5657
5658 assert(vs_disjoint(va, vc) || vs_same(va, vc), "va and vc neither disjoint nor equal");
5659 assert(vs_disjoint(va, vb), "va and vb overlap");
5660 assert(vs_disjoint(va, vq), "va and vq overlap");
5661 assert(vs_disjoint(va, vtmp), "va and vtmp overlap");
5662 assert(!va.is_constant(), "output vector must identify 4 different registers");
5663
5664 // schedule 4 streams of instructions across the vector sequences
5665 for (int i = 0; i < 4; i++) {
5666 __ sqdmulh(vtmp[i], T, vb[i], vc[i]); // aHigh = hi32(2 * b * c)
5667 __ mulv(va[i], T, vb[i], vc[i]); // aLow = lo32(b * c)
5668 }
5669
5670 for (int i = 0; i < 4; i++) {
5671 __ mulv(va[i], T, va[i], vq[0]); // m = aLow * qinv
5672 }
5673
5674 for (int i = 0; i < 4; i++) {
5675 __ sqdmulh(va[i], T, va[i], vq[1]); // n = hi32(2 * m * q)
5676 }
5677
5678 for (int i = 0; i < 4; i++) {
5679 __ shsubv(va[i], T, vtmp[i], va[i]); // a = (aHigh - n) / 2
5680 }
5681 }
5682
5683 // Perform 8 32-bit (4x4S) or 16 16-bit (2 x 8H) Montgomery
5684 // multiplications in parallel
5685 //
5686
5687 // See the montMul() method of the sun.security.provider.ML_DSA
5688 // class.
5689 //
5690 // Computes 4x4S results or 8x8H results
5691 // a = b * c * 2^MONT_R_BITS mod MONT_Q
5692 // Inputs: vb, vc - 4x4S or 4x8H vector register sequences
5693 // vq - 2x4S or 2x8H constants <MONT_Q, MONT_Q_INV_MOD_R>
5694 // Temps: vtmp - 4x4S or 4x8H vector sequence trashed after call
5695 // Outputs: va - 4x4S or 4x8H vector register sequences
5696 // vb, vc, vtmp and vq must all be disjoint
5697 // va must be disjoint from all other inputs/temps or must equal vc
5698 // va must have a non-zero delta i.e. it must not be a constant vseq.
5699 // n.b. MONT_R_BITS is 16 or 32, so the right shift by it is implicit.
5700 void vs_montmul2(const VSeq<2>& va, const VSeq<2>& vb, const VSeq<2>& vc,
5701 Assembler::SIMD_Arrangement T,
5702 const VSeq<2>& vtmp, const VSeq<2>& vq) {
5703 assert (T == __ T4S || T == __ T8H, "invalid arrangement for montmul");
5704 assert(vs_disjoint(vb, vc), "vb and vc overlap");
5705 assert(vs_disjoint(vb, vq), "vb and vq overlap");
5706 assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap");
5707
5708 assert(vs_disjoint(vc, vq), "vc and vq overlap");
5709 assert(vs_disjoint(vc, vtmp), "vc and vtmp overlap");
5710
5711 assert(vs_disjoint(vq, vtmp), "vq and vtmp overlap");
5712
5713 assert(vs_disjoint(va, vc) || vs_same(va, vc), "va and vc neither disjoint nor equal");
5714 assert(vs_disjoint(va, vb), "va and vb overlap");
5715 assert(vs_disjoint(va, vq), "va and vq overlap");
5716 assert(vs_disjoint(va, vtmp), "va and vtmp overlap");
5717 assert(!va.is_constant(), "output vector must identify 2 different registers");
5718
5719 // schedule 2 streams of instructions across the vector sequences
5720 for (int i = 0; i < 2; i++) {
5721 __ sqdmulh(vtmp[i], T, vb[i], vc[i]); // aHigh = hi32(2 * b * c)
5722 __ mulv(va[i], T, vb[i], vc[i]); // aLow = lo32(b * c)
5723 }
5724
5725 for (int i = 0; i < 2; i++) {
5726 __ mulv(va[i], T, va[i], vq[0]); // m = aLow * qinv
5727 }
5728
5729 for (int i = 0; i < 2; i++) {
5730 __ sqdmulh(va[i], T, va[i], vq[1]); // n = hi32(2 * m * q)
5731 }
5732
5733 for (int i = 0; i < 2; i++) {
5734 __ shsubv(va[i], T, vtmp[i], va[i]); // a = (aHigh - n) / 2
5735 }
5736 }
5737
5738 // Perform 16 16-bit Montgomery multiplications in parallel.
5739 void kyber_montmul16(const VSeq<2>& va, const VSeq<2>& vb, const VSeq<2>& vc,
5740 const VSeq<2>& vtmp, const VSeq<2>& vq) {
5741 // Use the helper routine to schedule a 2x8H Montgomery multiply.
5742 // It will assert that the register use is valid
5743 vs_montmul2(va, vb, vc, __ T8H, vtmp, vq);
5744 }
5745
5746 // Perform 32 16-bit Montgomery multiplications in parallel.
5747 void kyber_montmul32(const VSeq<4>& va, const VSeq<4>& vb, const VSeq<4>& vc,
5748 const VSeq<4>& vtmp, const VSeq<2>& vq) {
5749 // Use the helper routine to schedule a 4x8H Montgomery multiply.
5750 // It will assert that the register use is valid
5751 vs_montmul4(va, vb, vc, __ T8H, vtmp, vq);
5752 }
5753
5754 // Perform 64 16-bit Montgomery multiplications in parallel.
5755 void kyber_montmul64(const VSeq<8>& va, const VSeq<8>& vb, const VSeq<8>& vc,
5756 const VSeq<4>& vtmp, const VSeq<2>& vq) {
5757 // Schedule two successive 4x8H multiplies via the montmul helper
5758 // on the front and back halves of va, vb and vc. The helper will
5759 // assert that the register use has no overlap conflicts on each
5760 // individual call but we also need to ensure that the necessary
5761 // disjoint/equality constraints are met across both calls.
5762
5763 // vb, vc, vtmp and vq must be disjoint. va must either be
5764 // disjoint from all other registers or equal vc
5765
5766 assert(vs_disjoint(vb, vc), "vb and vc overlap");
5767 assert(vs_disjoint(vb, vq), "vb and vq overlap");
5768 assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap");
5769
5770 assert(vs_disjoint(vc, vq), "vc and vq overlap");
5771 assert(vs_disjoint(vc, vtmp), "vc and vtmp overlap");
5772
5773 assert(vs_disjoint(vq, vtmp), "vq and vtmp overlap");
5774
5775 assert(vs_disjoint(va, vc) || vs_same(va, vc), "va and vc neither disjoint nor equal");
5776 assert(vs_disjoint(va, vb), "va and vb overlap");
5777 assert(vs_disjoint(va, vq), "va and vq overlap");
5778 assert(vs_disjoint(va, vtmp), "va and vtmp overlap");
5779
5780 // we multiply the front and back halves of each sequence 4 at a
5781 // time because
5782 //
5783 // 1) we are currently only able to get 4-way instruction
5784 // parallelism at best
5785 //
5786 // 2) we need registers for the constants in vq and temporary
5787 // scratch registers to hold intermediate results so vtmp can only
5788 // be a VSeq<4> which means we only have 4 scratch slots
5789
5790 vs_montmul4(vs_front(va), vs_front(vb), vs_front(vc), __ T8H, vtmp, vq);
5791 vs_montmul4(vs_back(va), vs_back(vb), vs_back(vc), __ T8H, vtmp, vq);
5792 }
5793
5794 void kyber_montmul32_sub_add(const VSeq<4>& va0, const VSeq<4>& va1,
5795 const VSeq<4>& vc,
5796 const VSeq<4>& vtmp,
5797 const VSeq<2>& vq) {
5798 // compute a = montmul(a1, c)
5799 kyber_montmul32(vc, va1, vc, vtmp, vq);
5800 // ouptut a1 = a0 - a
5801 vs_subv(va1, __ T8H, va0, vc);
5802 // and a0 = a0 + a
5803 vs_addv(va0, __ T8H, va0, vc);
5804 }
5805
5806 void kyber_sub_add_montmul32(const VSeq<4>& va0, const VSeq<4>& va1,
5807 const VSeq<4>& vb,
5808 const VSeq<4>& vtmp1,
5809 const VSeq<4>& vtmp2,
5810 const VSeq<2>& vq) {
5811 // compute c = a0 - a1
5812 vs_subv(vtmp1, __ T8H, va0, va1);
5813 // output a0 = a0 + a1
5814 vs_addv(va0, __ T8H, va0, va1);
5815 // output a1 = b montmul c
5816 kyber_montmul32(va1, vtmp1, vb, vtmp2, vq);
5817 }
5818
5819 void load64shorts(const VSeq<8>& v, Register shorts) {
5820 vs_ldpq_post(v, shorts);
5821 }
5822
5823 void load32shorts(const VSeq<4>& v, Register shorts) {
5824 vs_ldpq_post(v, shorts);
5825 }
5826
5827 void store64shorts(VSeq<8> v, Register tmpAddr) {
5828 vs_stpq_post(v, tmpAddr);
5829 }
5830
5831 // Kyber NTT function.
5832 // Implements
5833 // static int implKyberNtt(short[] poly, short[] ntt_zetas) {}
5834 //
5835 // coeffs (short[256]) = c_rarg0
5836 // ntt_zetas (short[256]) = c_rarg1
5837 address generate_kyberNtt() {
5838 StubId stub_id = StubId::stubgen_kyberNtt_id;
5839 int entry_count = StubInfo::entry_count(stub_id);
5840 assert(entry_count == 1, "sanity check");
5841 address start = load_archive_data(stub_id);
5842 if (start != nullptr) {
5843 return start;
5844 }
5845 __ align(CodeEntryAlignment);
5846 StubCodeMark mark(this, stub_id);
5847 start = __ pc();
5848 __ enter();
5849
5850 const Register coeffs = c_rarg0;
5851 const Register zetas = c_rarg1;
5852
5853 const Register kyberConsts = r10;
5854 const Register tmpAddr = r11;
5855
5856 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x8H inputs/outputs
5857 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
5858 VSeq<2> vq(30); // n.b. constants overlap vs3
5859
5860 __ lea(kyberConsts, ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
5861 // load the montmul constants
5862 vs_ldpq(vq, kyberConsts);
5863
5864 // Each level corresponds to an iteration of the outermost loop of the
5865 // Java method seilerNTT(int[] coeffs). There are some differences
5866 // from what is done in the seilerNTT() method, though:
5867 // 1. The computation is using 16-bit signed values, we do not convert them
5868 // to ints here.
5869 // 2. The zetas are delivered in a bigger array, 128 zetas are stored in
5870 // this array for each level, it is easier that way to fill up the vector
5871 // registers.
5872 // 3. In the seilerNTT() method we use R = 2^20 for the Montgomery
5873 // multiplications (this is because that way there should not be any
5874 // overflow during the inverse NTT computation), here we use R = 2^16 so
5875 // that we can use the 16-bit arithmetic in the vector unit.
5876 //
5877 // On each level, we fill up the vector registers in such a way that the
5878 // array elements that need to be multiplied by the zetas go into one
5879 // set of vector registers while the corresponding ones that don't need to
5880 // be multiplied, go into another set.
5881 // We can do 32 Montgomery multiplications in parallel, using 12 vector
5882 // registers interleaving the steps of 4 identical computations,
5883 // each done on 8 16-bit values per register.
5884
5885 // At levels 0-3 the coefficients multiplied by or added/subtracted
5886 // to the zetas occur in discrete blocks whose size is some multiple
5887 // of 32.
5888
5889 // level 0
5890 __ add(tmpAddr, coeffs, 256);
5891 load64shorts(vs1, tmpAddr);
5892 load64shorts(vs2, zetas);
5893 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5894 __ add(tmpAddr, coeffs, 0);
5895 load64shorts(vs1, tmpAddr);
5896 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5897 vs_addv(vs1, __ T8H, vs1, vs2);
5898 __ add(tmpAddr, coeffs, 0);
5899 vs_stpq_post(vs1, tmpAddr);
5900 __ add(tmpAddr, coeffs, 256);
5901 vs_stpq_post(vs3, tmpAddr);
5902 // restore montmul constants
5903 vs_ldpq(vq, kyberConsts);
5904 load64shorts(vs1, tmpAddr);
5905 load64shorts(vs2, zetas);
5906 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5907 __ add(tmpAddr, coeffs, 128);
5908 load64shorts(vs1, tmpAddr);
5909 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5910 vs_addv(vs1, __ T8H, vs1, vs2);
5911 __ add(tmpAddr, coeffs, 128);
5912 store64shorts(vs1, tmpAddr);
5913 __ add(tmpAddr, coeffs, 384);
5914 store64shorts(vs3, tmpAddr);
5915
5916 // level 1
5917 // restore montmul constants
5918 vs_ldpq(vq, kyberConsts);
5919 __ add(tmpAddr, coeffs, 128);
5920 load64shorts(vs1, tmpAddr);
5921 load64shorts(vs2, zetas);
5922 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5923 __ add(tmpAddr, coeffs, 0);
5924 load64shorts(vs1, tmpAddr);
5925 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5926 vs_addv(vs1, __ T8H, vs1, vs2);
5927 __ add(tmpAddr, coeffs, 0);
5928 store64shorts(vs1, tmpAddr);
5929 store64shorts(vs3, tmpAddr);
5930 vs_ldpq(vq, kyberConsts);
5931 __ add(tmpAddr, coeffs, 384);
5932 load64shorts(vs1, tmpAddr);
5933 load64shorts(vs2, zetas);
5934 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5935 __ add(tmpAddr, coeffs, 256);
5936 load64shorts(vs1, tmpAddr);
5937 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5938 vs_addv(vs1, __ T8H, vs1, vs2);
5939 __ add(tmpAddr, coeffs, 256);
5940 store64shorts(vs1, tmpAddr);
5941 store64shorts(vs3, tmpAddr);
5942
5943 // level 2
5944 vs_ldpq(vq, kyberConsts);
5945 int offsets1[4] = { 0, 32, 128, 160 };
5946 vs_ldpq_indexed(vs1, coeffs, 64, offsets1);
5947 load64shorts(vs2, zetas);
5948 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5949 vs_ldpq_indexed(vs1, coeffs, 0, offsets1);
5950 // kyber_subv_addv64();
5951 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5952 vs_addv(vs1, __ T8H, vs1, vs2);
5953 __ add(tmpAddr, coeffs, 0);
5954 vs_stpq_post(vs_front(vs1), tmpAddr);
5955 vs_stpq_post(vs_front(vs3), tmpAddr);
5956 vs_stpq_post(vs_back(vs1), tmpAddr);
5957 vs_stpq_post(vs_back(vs3), tmpAddr);
5958 vs_ldpq(vq, kyberConsts);
5959 vs_ldpq_indexed(vs1, tmpAddr, 64, offsets1);
5960 load64shorts(vs2, zetas);
5961 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5962 vs_ldpq_indexed(vs1, coeffs, 256, offsets1);
5963 // kyber_subv_addv64();
5964 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5965 vs_addv(vs1, __ T8H, vs1, vs2);
5966 __ add(tmpAddr, coeffs, 256);
5967 vs_stpq_post(vs_front(vs1), tmpAddr);
5968 vs_stpq_post(vs_front(vs3), tmpAddr);
5969 vs_stpq_post(vs_back(vs1), tmpAddr);
5970 vs_stpq_post(vs_back(vs3), tmpAddr);
5971
5972 // level 3
5973 vs_ldpq(vq, kyberConsts);
5974 int offsets2[4] = { 0, 64, 128, 192 };
5975 vs_ldpq_indexed(vs1, coeffs, 32, offsets2);
5976 load64shorts(vs2, zetas);
5977 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5978 vs_ldpq_indexed(vs1, coeffs, 0, offsets2);
5979 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5980 vs_addv(vs1, __ T8H, vs1, vs2);
5981 vs_stpq_indexed(vs1, coeffs, 0, offsets2);
5982 vs_stpq_indexed(vs3, coeffs, 32, offsets2);
5983
5984 vs_ldpq(vq, kyberConsts);
5985 vs_ldpq_indexed(vs1, coeffs, 256 + 32, offsets2);
5986 load64shorts(vs2, zetas);
5987 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
5988 vs_ldpq_indexed(vs1, coeffs, 256, offsets2);
5989 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
5990 vs_addv(vs1, __ T8H, vs1, vs2);
5991 vs_stpq_indexed(vs1, coeffs, 256, offsets2);
5992 vs_stpq_indexed(vs3, coeffs, 256 + 32, offsets2);
5993
5994 // level 4
5995 // At level 4 coefficients occur in 8 discrete blocks of size 16
5996 // so they are loaded by employing an ldr at 8 distinct offsets.
5997
5998 vs_ldpq(vq, kyberConsts);
5999 int offsets3[8] = { 0, 32, 64, 96, 128, 160, 192, 224 };
6000 vs_ldr_indexed(vs1, __ Q, coeffs, 16, offsets3);
6001 load64shorts(vs2, zetas);
6002 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6003 vs_ldr_indexed(vs1, __ Q, coeffs, 0, offsets3);
6004 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6005 vs_addv(vs1, __ T8H, vs1, vs2);
6006 vs_str_indexed(vs1, __ Q, coeffs, 0, offsets3);
6007 vs_str_indexed(vs3, __ Q, coeffs, 16, offsets3);
6008
6009 vs_ldpq(vq, kyberConsts);
6010 vs_ldr_indexed(vs1, __ Q, coeffs, 256 + 16, offsets3);
6011 load64shorts(vs2, zetas);
6012 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6013 vs_ldr_indexed(vs1, __ Q, coeffs, 256, offsets3);
6014 vs_subv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6015 vs_addv(vs1, __ T8H, vs1, vs2);
6016 vs_str_indexed(vs1, __ Q, coeffs, 256, offsets3);
6017 vs_str_indexed(vs3, __ Q, coeffs, 256 + 16, offsets3);
6018
6019 // level 5
6020 // At level 5 related coefficients occur in discrete blocks of size 8 so
6021 // need to be loaded interleaved using an ld2 operation with arrangement 2D.
6022
6023 vs_ldpq(vq, kyberConsts);
6024 int offsets4[4] = { 0, 32, 64, 96 };
6025 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4);
6026 load32shorts(vs_front(vs2), zetas);
6027 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6028 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4);
6029 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4);
6030 load32shorts(vs_front(vs2), zetas);
6031 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6032 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4);
6033 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4);
6034 load32shorts(vs_front(vs2), zetas);
6035 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6036 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4);
6037
6038 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4);
6039 load32shorts(vs_front(vs2), zetas);
6040 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6041 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4);
6042
6043 // level 6
6044 // At level 6 related coefficients occur in discrete blocks of size 4 so
6045 // need to be loaded interleaved using an ld2 operation with arrangement 4S.
6046
6047 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4);
6048 load32shorts(vs_front(vs2), zetas);
6049 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6050 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4);
6051 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4);
6052 load32shorts(vs_front(vs2), zetas);
6053 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6054 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4);
6055
6056 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4);
6057 load32shorts(vs_front(vs2), zetas);
6058 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6059 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4);
6060
6061 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4);
6062 load32shorts(vs_front(vs2), zetas);
6063 kyber_montmul32_sub_add(vs_even(vs1), vs_odd(vs1), vs_front(vs2), vtmp, vq);
6064 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4);
6065
6066 __ leave(); // required for proper stackwalking of RuntimeStub frame
6067 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6068 __ ret(lr);
6069
6070 // record the stub entry and end
6071 store_archive_data(stub_id, start, __ pc());
6072
6073 return start;
6074 }
6075
6076 // Kyber Inverse NTT function
6077 // Implements
6078 // static int implKyberInverseNtt(short[] poly, short[] zetas) {}
6079 //
6080 // coeffs (short[256]) = c_rarg0
6081 // ntt_zetas (short[256]) = c_rarg1
6082 address generate_kyberInverseNtt() {
6083 StubId stub_id = StubId::stubgen_kyberInverseNtt_id;
6084 int entry_count = StubInfo::entry_count(stub_id);
6085 assert(entry_count == 1, "sanity check");
6086 address start = load_archive_data(stub_id);
6087 if (start != nullptr) {
6088 return start;
6089 }
6090 __ align(CodeEntryAlignment);
6091 StubCodeMark mark(this, stub_id);
6092 start = __ pc();
6093 __ enter();
6094
6095 const Register coeffs = c_rarg0;
6096 const Register zetas = c_rarg1;
6097
6098 const Register kyberConsts = r10;
6099 const Register tmpAddr = r11;
6100 const Register tmpAddr2 = c_rarg2;
6101
6102 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x8H inputs/outputs
6103 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
6104 VSeq<2> vq(30); // n.b. constants overlap vs3
6105
6106 __ lea(kyberConsts,
6107 ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
6108
6109 // level 0
6110 // At level 0 related coefficients occur in discrete blocks of size 4 so
6111 // need to be loaded interleaved using an ld2 operation with arrangement 4S.
6112
6113 vs_ldpq(vq, kyberConsts);
6114 int offsets4[4] = { 0, 32, 64, 96 };
6115 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4);
6116 load32shorts(vs_front(vs2), zetas);
6117 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6118 vs_front(vs2), vs_back(vs2), vtmp, vq);
6119 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 0, offsets4);
6120 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4);
6121 load32shorts(vs_front(vs2), zetas);
6122 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6123 vs_front(vs2), vs_back(vs2), vtmp, vq);
6124 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 128, offsets4);
6125 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4);
6126 load32shorts(vs_front(vs2), zetas);
6127 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6128 vs_front(vs2), vs_back(vs2), vtmp, vq);
6129 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 256, offsets4);
6130 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4);
6131 load32shorts(vs_front(vs2), zetas);
6132 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6133 vs_front(vs2), vs_back(vs2), vtmp, vq);
6134 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, 384, offsets4);
6135
6136 // level 1
6137 // At level 1 related coefficients occur in discrete blocks of size 8 so
6138 // need to be loaded interleaved using an ld2 operation with arrangement 2D.
6139
6140 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4);
6141 load32shorts(vs_front(vs2), zetas);
6142 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6143 vs_front(vs2), vs_back(vs2), vtmp, vq);
6144 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 0, offsets4);
6145 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4);
6146 load32shorts(vs_front(vs2), zetas);
6147 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6148 vs_front(vs2), vs_back(vs2), vtmp, vq);
6149 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 128, offsets4);
6150
6151 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4);
6152 load32shorts(vs_front(vs2), zetas);
6153 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6154 vs_front(vs2), vs_back(vs2), vtmp, vq);
6155 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 256, offsets4);
6156 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4);
6157 load32shorts(vs_front(vs2), zetas);
6158 kyber_sub_add_montmul32(vs_even(vs1), vs_odd(vs1),
6159 vs_front(vs2), vs_back(vs2), vtmp, vq);
6160 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, 384, offsets4);
6161
6162 // level 2
6163 // At level 2 coefficients occur in 8 discrete blocks of size 16
6164 // so they are loaded by employing an ldr at 8 distinct offsets.
6165
6166 int offsets3[8] = { 0, 32, 64, 96, 128, 160, 192, 224 };
6167 vs_ldr_indexed(vs1, __ Q, coeffs, 0, offsets3);
6168 vs_ldr_indexed(vs2, __ Q, coeffs, 16, offsets3);
6169 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6170 vs_subv(vs1, __ T8H, vs1, vs2);
6171 vs_str_indexed(vs3, __ Q, coeffs, 0, offsets3);
6172 load64shorts(vs2, zetas);
6173 vs_ldpq(vq, kyberConsts);
6174 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6175 vs_str_indexed(vs2, __ Q, coeffs, 16, offsets3);
6176
6177 vs_ldr_indexed(vs1, __ Q, coeffs, 256, offsets3);
6178 vs_ldr_indexed(vs2, __ Q, coeffs, 256 + 16, offsets3);
6179 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6180 vs_subv(vs1, __ T8H, vs1, vs2);
6181 vs_str_indexed(vs3, __ Q, coeffs, 256, offsets3);
6182 load64shorts(vs2, zetas);
6183 vs_ldpq(vq, kyberConsts);
6184 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6185 vs_str_indexed(vs2, __ Q, coeffs, 256 + 16, offsets3);
6186
6187 // Barrett reduction at indexes where overflow may happen
6188
6189 // load q and the multiplier for the Barrett reduction
6190 __ add(tmpAddr, kyberConsts, 16);
6191 vs_ldpq(vq, tmpAddr);
6192
6193 VSeq<8> vq1 = VSeq<8>(vq[0], 0); // 2 constant 8 sequences
6194 VSeq<8> vq2 = VSeq<8>(vq[1], 0); // for above two kyber constants
6195 VSeq<8> vq3 = VSeq<8>(v29, 0); // 3rd sequence for const montmul
6196 vs_ldr_indexed(vs1, __ Q, coeffs, 0, offsets3);
6197 vs_sqdmulh(vs2, __ T8H, vs1, vq2);
6198 vs_sshr(vs2, __ T8H, vs2, 11);
6199 vs_mlsv(vs1, __ T8H, vs2, vq1);
6200 vs_str_indexed(vs1, __ Q, coeffs, 0, offsets3);
6201 vs_ldr_indexed(vs1, __ Q, coeffs, 256, offsets3);
6202 vs_sqdmulh(vs2, __ T8H, vs1, vq2);
6203 vs_sshr(vs2, __ T8H, vs2, 11);
6204 vs_mlsv(vs1, __ T8H, vs2, vq1);
6205 vs_str_indexed(vs1, __ Q, coeffs, 256, offsets3);
6206
6207 // level 3
6208 // From level 3 upwards coefficients occur in discrete blocks whose size is
6209 // some multiple of 32 so can be loaded using ldpq and suitable indexes.
6210
6211 int offsets2[4] = { 0, 64, 128, 192 };
6212 vs_ldpq_indexed(vs1, coeffs, 0, offsets2);
6213 vs_ldpq_indexed(vs2, coeffs, 32, offsets2);
6214 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6215 vs_subv(vs1, __ T8H, vs1, vs2);
6216 vs_stpq_indexed(vs3, coeffs, 0, offsets2);
6217 load64shorts(vs2, zetas);
6218 vs_ldpq(vq, kyberConsts);
6219 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6220 vs_stpq_indexed(vs2, coeffs, 32, offsets2);
6221
6222 vs_ldpq_indexed(vs1, coeffs, 256, offsets2);
6223 vs_ldpq_indexed(vs2, coeffs, 256 + 32, offsets2);
6224 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6225 vs_subv(vs1, __ T8H, vs1, vs2);
6226 vs_stpq_indexed(vs3, coeffs, 256, offsets2);
6227 load64shorts(vs2, zetas);
6228 vs_ldpq(vq, kyberConsts);
6229 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6230 vs_stpq_indexed(vs2, coeffs, 256 + 32, offsets2);
6231
6232 // level 4
6233
6234 int offsets1[4] = { 0, 32, 128, 160 };
6235 vs_ldpq_indexed(vs1, coeffs, 0, offsets1);
6236 vs_ldpq_indexed(vs2, coeffs, 64, offsets1);
6237 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6238 vs_subv(vs1, __ T8H, vs1, vs2);
6239 vs_stpq_indexed(vs3, coeffs, 0, offsets1);
6240 load64shorts(vs2, zetas);
6241 vs_ldpq(vq, kyberConsts);
6242 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6243 vs_stpq_indexed(vs2, coeffs, 64, offsets1);
6244
6245 vs_ldpq_indexed(vs1, coeffs, 256, offsets1);
6246 vs_ldpq_indexed(vs2, coeffs, 256 + 64, offsets1);
6247 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6248 vs_subv(vs1, __ T8H, vs1, vs2);
6249 vs_stpq_indexed(vs3, coeffs, 256, offsets1);
6250 load64shorts(vs2, zetas);
6251 vs_ldpq(vq, kyberConsts);
6252 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6253 vs_stpq_indexed(vs2, coeffs, 256 + 64, offsets1);
6254
6255 // level 5
6256
6257 __ add(tmpAddr, coeffs, 0);
6258 load64shorts(vs1, tmpAddr);
6259 __ add(tmpAddr, coeffs, 128);
6260 load64shorts(vs2, tmpAddr);
6261 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6262 vs_subv(vs1, __ T8H, vs1, vs2);
6263 __ add(tmpAddr, coeffs, 0);
6264 store64shorts(vs3, tmpAddr);
6265 load64shorts(vs2, zetas);
6266 vs_ldpq(vq, kyberConsts);
6267 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6268 __ add(tmpAddr, coeffs, 128);
6269 store64shorts(vs2, tmpAddr);
6270
6271 load64shorts(vs1, tmpAddr);
6272 __ add(tmpAddr, coeffs, 384);
6273 load64shorts(vs2, tmpAddr);
6274 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6275 vs_subv(vs1, __ T8H, vs1, vs2);
6276 __ add(tmpAddr, coeffs, 256);
6277 store64shorts(vs3, tmpAddr);
6278 load64shorts(vs2, zetas);
6279 vs_ldpq(vq, kyberConsts);
6280 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6281 __ add(tmpAddr, coeffs, 384);
6282 store64shorts(vs2, tmpAddr);
6283
6284 // Barrett reduction at indexes where overflow may happen
6285
6286 // load q and the multiplier for the Barrett reduction
6287 __ add(tmpAddr, kyberConsts, 16);
6288 vs_ldpq(vq, tmpAddr);
6289
6290 int offsets0[2] = { 0, 256 };
6291 vs_ldpq_indexed(vs_front(vs1), coeffs, 0, offsets0);
6292 vs_sqdmulh(vs2, __ T8H, vs1, vq2);
6293 vs_sshr(vs2, __ T8H, vs2, 11);
6294 vs_mlsv(vs1, __ T8H, vs2, vq1);
6295 vs_stpq_indexed(vs_front(vs1), coeffs, 0, offsets0);
6296
6297 // level 6
6298
6299 __ add(tmpAddr, coeffs, 0);
6300 load64shorts(vs1, tmpAddr);
6301 __ add(tmpAddr, coeffs, 256);
6302 load64shorts(vs2, tmpAddr);
6303 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6304 vs_subv(vs1, __ T8H, vs1, vs2);
6305 __ add(tmpAddr, coeffs, 0);
6306 store64shorts(vs3, tmpAddr);
6307 load64shorts(vs2, zetas);
6308 vs_ldpq(vq, kyberConsts);
6309 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6310 __ add(tmpAddr, coeffs, 256);
6311 store64shorts(vs2, tmpAddr);
6312
6313 __ add(tmpAddr, coeffs, 128);
6314 load64shorts(vs1, tmpAddr);
6315 __ add(tmpAddr, coeffs, 384);
6316 load64shorts(vs2, tmpAddr);
6317 vs_addv(vs3, __ T8H, vs1, vs2); // n.b. trashes vq
6318 vs_subv(vs1, __ T8H, vs1, vs2);
6319 __ add(tmpAddr, coeffs, 128);
6320 store64shorts(vs3, tmpAddr);
6321 load64shorts(vs2, zetas);
6322 vs_ldpq(vq, kyberConsts);
6323 kyber_montmul64(vs2, vs1, vs2, vtmp, vq);
6324 __ add(tmpAddr, coeffs, 384);
6325 store64shorts(vs2, tmpAddr);
6326
6327 // multiply by 2^-n
6328
6329 // load toMont(2^-n mod q)
6330 __ add(tmpAddr, kyberConsts, 48);
6331 __ ldr(v29, __ Q, tmpAddr);
6332
6333 vs_ldpq(vq, kyberConsts);
6334 __ add(tmpAddr, coeffs, 0);
6335 load64shorts(vs1, tmpAddr);
6336 kyber_montmul64(vs2, vs1, vq3, vtmp, vq);
6337 __ add(tmpAddr, coeffs, 0);
6338 store64shorts(vs2, tmpAddr);
6339
6340 // now tmpAddr contains coeffs + 128 because store64shorts adjusted it so
6341 load64shorts(vs1, tmpAddr);
6342 kyber_montmul64(vs2, vs1, vq3, vtmp, vq);
6343 __ add(tmpAddr, coeffs, 128);
6344 store64shorts(vs2, tmpAddr);
6345
6346 // now tmpAddr contains coeffs + 256
6347 load64shorts(vs1, tmpAddr);
6348 kyber_montmul64(vs2, vs1, vq3, vtmp, vq);
6349 __ add(tmpAddr, coeffs, 256);
6350 store64shorts(vs2, tmpAddr);
6351
6352 // now tmpAddr contains coeffs + 384
6353 load64shorts(vs1, tmpAddr);
6354 kyber_montmul64(vs2, vs1, vq3, vtmp, vq);
6355 __ add(tmpAddr, coeffs, 384);
6356 store64shorts(vs2, tmpAddr);
6357
6358 __ leave(); // required for proper stackwalking of RuntimeStub frame
6359 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6360 __ ret(lr);
6361
6362 // record the stub entry and end
6363 store_archive_data(stub_id, start, __ pc());
6364
6365 return start;
6366 }
6367
6368 // Kyber multiply polynomials in the NTT domain.
6369 // Implements
6370 // static int implKyberNttMult(
6371 // short[] result, short[] ntta, short[] nttb, short[] zetas) {}
6372 //
6373 // The actual algorithm that is used here differs from the one in the Java
6374 // implementation, it uses Montgomery multiplications instead of Barrett
6375 // reduction, but the end result modulo MLKEM_Q is the same. This is the
6376 // Java equivalent of this intrinsic implementation:
6377 // static void implKyberNttMultJava(short[] result, short[] ntta, short[] nttb) {
6378 // for (int m = 0; m < ML_KEM_N / 2; m++) {
6379 // int a0 = ntta[2 * m];
6380 // int a1 = ntta[2 * m + 1];
6381 // int b0 = nttb[2 * m];
6382 // int b1 = nttb[2 * m + 1];
6383 // int r = montMul(a0, b0) +
6384 // montMul(montMul(a1, b1), MONT_ZETAS_FOR_NTT_MULT[m]);
6385 // result[2 * m] = (short) montMul(r, MONT_R_SQUARE_MOD_Q);
6386 // result[2 * m + 1] = (short) montMul(
6387 // (montMul(a0, b1) + montMul(a1, b0)), MONT_R_SQUARE_MOD_Q);
6388 // }
6389 // }
6390 //
6391 // result (short[256]) = c_rarg0
6392 // ntta (short[256]) = c_rarg1
6393 // nttb (short[256]) = c_rarg2
6394 // zetas (short[128]) = c_rarg3
6395 address generate_kyberNttMult() {
6396 StubId stub_id = StubId::stubgen_kyberNttMult_id;
6397 int entry_count = StubInfo::entry_count(stub_id);
6398 assert(entry_count == 1, "sanity check");
6399 address start = load_archive_data(stub_id);
6400 if (start != nullptr) {
6401 return start;
6402 }
6403 __ align(CodeEntryAlignment);
6404 StubCodeMark mark(this, stub_id);
6405 start = __ pc();
6406 __ enter();
6407
6408 const Register result = c_rarg0;
6409 const Register ntta = c_rarg1;
6410 const Register nttb = c_rarg2;
6411 const Register zetas = c_rarg3;
6412
6413 const Register kyberConsts = r10;
6414 const Register limit = r11;
6415
6416 VSeq<4> vs1(0), vs2(4); // 4 sets of 8x8H inputs/outputs/tmps
6417 VSeq<4> vs3(16), vs4(20);
6418 VSeq<2> vq(30); // pair of constants for montmul: q, qinv
6419 VSeq<2> vz(28); // pair of zetas
6420 VSeq<4> vc(27, 0); // constant sequence for montmul: montRSquareModQ
6421
6422 __ lea(kyberConsts,
6423 ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
6424
6425 Label kyberNttMult_loop;
6426
6427 __ add(limit, result, 512);
6428
6429 // load q and qinv
6430 vs_ldpq(vq, kyberConsts);
6431
6432 // load R^2 mod q (to convert back from Montgomery representation)
6433 __ add(kyberConsts, kyberConsts, 64);
6434 __ ldr(v27, __ Q, kyberConsts);
6435
6436 __ BIND(kyberNttMult_loop);
6437
6438 // load 16 zetas
6439 vs_ldpq_post(vz, zetas);
6440
6441 // load 2 sets of 32 coefficients from the two input arrays
6442 // interleaved as shorts. i.e. pairs of shorts adjacent in memory
6443 // are striped across pairs of vector registers
6444 vs_ld2_post(vs_front(vs1), __ T8H, ntta); // <a0, a1> x 8H
6445 vs_ld2_post(vs_back(vs1), __ T8H, nttb); // <b0, b1> x 8H
6446 vs_ld2_post(vs_front(vs4), __ T8H, ntta); // <a2, a3> x 8H
6447 vs_ld2_post(vs_back(vs4), __ T8H, nttb); // <b2, b3> x 8H
6448
6449 // compute 4 montmul cross-products for pairs (a0,a1) and (b0,b1)
6450 // i.e. montmul the first and second halves of vs1 in order and
6451 // then with one sequence reversed storing the two results in vs3
6452 //
6453 // vs3[0] <- montmul(a0, b0)
6454 // vs3[1] <- montmul(a1, b1)
6455 // vs3[2] <- montmul(a0, b1)
6456 // vs3[3] <- montmul(a1, b0)
6457 kyber_montmul16(vs_front(vs3), vs_front(vs1), vs_back(vs1), vs_front(vs2), vq);
6458 kyber_montmul16(vs_back(vs3),
6459 vs_front(vs1), vs_reverse(vs_back(vs1)), vs_back(vs2), vq);
6460
6461 // compute 4 montmul cross-products for pairs (a2,a3) and (b2,b3)
6462 // i.e. montmul the first and second halves of vs4 in order and
6463 // then with one sequence reversed storing the two results in vs1
6464 //
6465 // vs1[0] <- montmul(a2, b2)
6466 // vs1[1] <- montmul(a3, b3)
6467 // vs1[2] <- montmul(a2, b3)
6468 // vs1[3] <- montmul(a3, b2)
6469 kyber_montmul16(vs_front(vs1), vs_front(vs4), vs_back(vs4), vs_front(vs2), vq);
6470 kyber_montmul16(vs_back(vs1),
6471 vs_front(vs4), vs_reverse(vs_back(vs4)), vs_back(vs2), vq);
6472
6473 // montmul result 2 of each cross-product i.e. (a1*b1, a3*b3) by a zeta.
6474 // We can schedule two montmuls at a time if we use a suitable vector
6475 // sequence <vs3[1], vs1[1]>.
6476 int delta = vs1[1]->encoding() - vs3[1]->encoding();
6477 VSeq<2> vs5(vs3[1], delta);
6478
6479 // vs3[1] <- montmul(montmul(a1, b1), z0)
6480 // vs1[1] <- montmul(montmul(a3, b3), z1)
6481 kyber_montmul16(vs5, vz, vs5, vs_front(vs2), vq);
6482
6483 // add results in pairs storing in vs3
6484 // vs3[0] <- montmul(a0, b0) + montmul(montmul(a1, b1), z0);
6485 // vs3[1] <- montmul(a0, b1) + montmul(a1, b0);
6486 vs_addv(vs_front(vs3), __ T8H, vs_even(vs3), vs_odd(vs3));
6487
6488 // vs3[2] <- montmul(a2, b2) + montmul(montmul(a3, b3), z1);
6489 // vs3[3] <- montmul(a2, b3) + montmul(a3, b2);
6490 vs_addv(vs_back(vs3), __ T8H, vs_even(vs1), vs_odd(vs1));
6491
6492 // vs1 <- montmul(vs3, montRSquareModQ)
6493 kyber_montmul32(vs1, vs3, vc, vs2, vq);
6494
6495 // store back the two pairs of result vectors de-interleaved as 8H elements
6496 // i.e. storing each pairs of shorts striped across a register pair adjacent
6497 // in memory
6498 vs_st2_post(vs1, __ T8H, result);
6499
6500 __ cmp(result, limit);
6501 __ br(Assembler::NE, kyberNttMult_loop);
6502
6503 __ leave(); // required for proper stackwalking of RuntimeStub frame
6504 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6505 __ ret(lr);
6506
6507 // record the stub entry and end
6508 store_archive_data(stub_id, start, __ pc());
6509
6510 return start;
6511 }
6512
6513 // Kyber add 2 polynomials.
6514 // Implements
6515 // static int implKyberAddPoly(short[] result, short[] a, short[] b) {}
6516 //
6517 // result (short[256]) = c_rarg0
6518 // a (short[256]) = c_rarg1
6519 // b (short[256]) = c_rarg2
6520 address generate_kyberAddPoly_2() {
6521 StubId stub_id = StubId::stubgen_kyberAddPoly_2_id;
6522 int entry_count = StubInfo::entry_count(stub_id);
6523 assert(entry_count == 1, "sanity check");
6524 address start = load_archive_data(stub_id);
6525 if (start != nullptr) {
6526 return start;
6527 }
6528 __ align(CodeEntryAlignment);
6529 StubCodeMark mark(this, stub_id);
6530 start = __ pc();
6531 __ enter();
6532
6533 const Register result = c_rarg0;
6534 const Register a = c_rarg1;
6535 const Register b = c_rarg2;
6536
6537 const Register kyberConsts = r11;
6538
6539 // We sum 256 sets of values in total i.e. 32 x 8H quadwords.
6540 // So, we can load, add and store the data in 3 groups of 11,
6541 // 11 and 10 at a time i.e. we need to map sets of 10 or 11
6542 // registers. A further constraint is that the mapping needs
6543 // to skip callee saves. So, we allocate the register
6544 // sequences using two 8 sequences, two 2 sequences and two
6545 // single registers.
6546 VSeq<8> vs1_1(0);
6547 VSeq<2> vs1_2(16);
6548 FloatRegister vs1_3 = v28;
6549 VSeq<8> vs2_1(18);
6550 VSeq<2> vs2_2(26);
6551 FloatRegister vs2_3 = v29;
6552
6553 // two constant vector sequences
6554 VSeq<8> vc_1(31, 0);
6555 VSeq<2> vc_2(31, 0);
6556
6557 FloatRegister vc_3 = v31;
6558 __ lea(kyberConsts,
6559 ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
6560
6561 __ ldr(vc_3, __ Q, Address(kyberConsts, 16)); // q
6562 for (int i = 0; i < 3; i++) {
6563 // load 80 or 88 values from a into vs1_1/2/3
6564 vs_ldpq_post(vs1_1, a);
6565 vs_ldpq_post(vs1_2, a);
6566 if (i < 2) {
6567 __ ldr(vs1_3, __ Q, __ post(a, 16));
6568 }
6569 // load 80 or 88 values from b into vs2_1/2/3
6570 vs_ldpq_post(vs2_1, b);
6571 vs_ldpq_post(vs2_2, b);
6572 if (i < 2) {
6573 __ ldr(vs2_3, __ Q, __ post(b, 16));
6574 }
6575 // sum 80 or 88 values across vs1 and vs2 into vs1
6576 vs_addv(vs1_1, __ T8H, vs1_1, vs2_1);
6577 vs_addv(vs1_2, __ T8H, vs1_2, vs2_2);
6578 if (i < 2) {
6579 __ addv(vs1_3, __ T8H, vs1_3, vs2_3);
6580 }
6581 // add constant to all 80 or 88 results
6582 vs_addv(vs1_1, __ T8H, vs1_1, vc_1);
6583 vs_addv(vs1_2, __ T8H, vs1_2, vc_2);
6584 if (i < 2) {
6585 __ addv(vs1_3, __ T8H, vs1_3, vc_3);
6586 }
6587 // store 80 or 88 values
6588 vs_stpq_post(vs1_1, result);
6589 vs_stpq_post(vs1_2, result);
6590 if (i < 2) {
6591 __ str(vs1_3, __ Q, __ post(result, 16));
6592 }
6593 }
6594
6595 __ leave(); // required for proper stackwalking of RuntimeStub frame
6596 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6597 __ ret(lr);
6598
6599 // record the stub entry and end
6600 store_archive_data(stub_id, start, __ pc());
6601
6602 return start;
6603 }
6604
6605 // Kyber add 3 polynomials.
6606 // Implements
6607 // static int implKyberAddPoly(short[] result, short[] a, short[] b, short[] c) {}
6608 //
6609 // result (short[256]) = c_rarg0
6610 // a (short[256]) = c_rarg1
6611 // b (short[256]) = c_rarg2
6612 // c (short[256]) = c_rarg3
6613 address generate_kyberAddPoly_3() {
6614 StubId stub_id = StubId::stubgen_kyberAddPoly_3_id;
6615 int entry_count = StubInfo::entry_count(stub_id);
6616 assert(entry_count == 1, "sanity check");
6617 address start = load_archive_data(stub_id);
6618 if (start != nullptr) {
6619 return start;
6620 }
6621 __ align(CodeEntryAlignment);
6622 StubCodeMark mark(this, stub_id);
6623 start = __ pc();
6624 __ enter();
6625
6626 const Register result = c_rarg0;
6627 const Register a = c_rarg1;
6628 const Register b = c_rarg2;
6629 const Register c = c_rarg3;
6630
6631 const Register kyberConsts = r11;
6632
6633 // As above we sum 256 sets of values in total i.e. 32 x 8H
6634 // quadwords. So, we can load, add and store the data in 3
6635 // groups of 11, 11 and 10 at a time i.e. we need to map sets
6636 // of 10 or 11 registers. A further constraint is that the
6637 // mapping needs to skip callee saves. So, we allocate the
6638 // register sequences using two 8 sequences, two 2 sequences
6639 // and two single registers.
6640 VSeq<8> vs1_1(0);
6641 VSeq<2> vs1_2(16);
6642 FloatRegister vs1_3 = v28;
6643 VSeq<8> vs2_1(18);
6644 VSeq<2> vs2_2(26);
6645 FloatRegister vs2_3 = v29;
6646
6647 // two constant vector sequences
6648 VSeq<8> vc_1(31, 0);
6649 VSeq<2> vc_2(31, 0);
6650
6651 FloatRegister vc_3 = v31;
6652
6653 __ lea(kyberConsts,
6654 ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
6655
6656 __ ldr(vc_3, __ Q, Address(kyberConsts, 16)); // q
6657 for (int i = 0; i < 3; i++) {
6658 // load 80 or 88 values from a into vs1_1/2/3
6659 vs_ldpq_post(vs1_1, a);
6660 vs_ldpq_post(vs1_2, a);
6661 if (i < 2) {
6662 __ ldr(vs1_3, __ Q, __ post(a, 16));
6663 }
6664 // load 80 or 88 values from b into vs2_1/2/3
6665 vs_ldpq_post(vs2_1, b);
6666 vs_ldpq_post(vs2_2, b);
6667 if (i < 2) {
6668 __ ldr(vs2_3, __ Q, __ post(b, 16));
6669 }
6670 // sum 80 or 88 values across vs1 and vs2 into vs1
6671 vs_addv(vs1_1, __ T8H, vs1_1, vs2_1);
6672 vs_addv(vs1_2, __ T8H, vs1_2, vs2_2);
6673 if (i < 2) {
6674 __ addv(vs1_3, __ T8H, vs1_3, vs2_3);
6675 }
6676 // load 80 or 88 values from c into vs2_1/2/3
6677 vs_ldpq_post(vs2_1, c);
6678 vs_ldpq_post(vs2_2, c);
6679 if (i < 2) {
6680 __ ldr(vs2_3, __ Q, __ post(c, 16));
6681 }
6682 // sum 80 or 88 values across vs1 and vs2 into vs1
6683 vs_addv(vs1_1, __ T8H, vs1_1, vs2_1);
6684 vs_addv(vs1_2, __ T8H, vs1_2, vs2_2);
6685 if (i < 2) {
6686 __ addv(vs1_3, __ T8H, vs1_3, vs2_3);
6687 }
6688 // add constant to all 80 or 88 results
6689 vs_addv(vs1_1, __ T8H, vs1_1, vc_1);
6690 vs_addv(vs1_2, __ T8H, vs1_2, vc_2);
6691 if (i < 2) {
6692 __ addv(vs1_3, __ T8H, vs1_3, vc_3);
6693 }
6694 // store 80 or 88 values
6695 vs_stpq_post(vs1_1, result);
6696 vs_stpq_post(vs1_2, result);
6697 if (i < 2) {
6698 __ str(vs1_3, __ Q, __ post(result, 16));
6699 }
6700 }
6701
6702 __ leave(); // required for proper stackwalking of RuntimeStub frame
6703 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6704 __ ret(lr);
6705
6706 // record the stub entry and end
6707 store_archive_data(stub_id, start, __ pc());
6708
6709 return start;
6710 }
6711
6712 // Kyber parse XOF output to polynomial coefficient candidates
6713 // or decodePoly(12, ...).
6714 // Implements
6715 // static int implKyber12To16(
6716 // byte[] condensed, int index, short[] parsed, int parsedLength) {}
6717 //
6718 // we assume that parsed and condensed are allocated such that for
6719 // n = (parsedLength + 63) / 64
6720 // n blocks of 96 bytes of input can be processed, i.e.
6721 // index + n * 96 <= condensed.length and
6722 // n * 64 <= parsed.length
6723 //
6724 // condensed (byte[]) = c_rarg0
6725 // condensedIndex = c_rarg1
6726 // parsed (short[]) = c_rarg2
6727 // parsedLength = c_rarg3
6728 address generate_kyber12To16() {
6729 StubId stub_id = StubId::stubgen_kyber12To16_id;
6730 int entry_count = StubInfo::entry_count(stub_id);
6731 assert(entry_count == 1, "sanity check");
6732 address start = load_archive_data(stub_id);
6733 if (start != nullptr) {
6734 return start;
6735 }
6736 Label L_F00, L_loop;
6737
6738 __ align(CodeEntryAlignment);
6739 StubCodeMark mark(this, stub_id);
6740 start = __ pc();
6741 __ enter();
6742
6743 const Register condensed = c_rarg0;
6744 const Register condensedOffs = c_rarg1;
6745 const Register parsed = c_rarg2;
6746 const Register parsedLength = c_rarg3;
6747
6748 const Register tmpAddr = r11;
6749
6750 // Data is input 96 bytes at a time i.e. in groups of 6 x 16B
6751 // quadwords so we need a 6 vector sequence for the inputs.
6752 // Parsing produces 64 shorts, employing two 8 vector
6753 // sequences to store and combine the intermediate data.
6754 VSeq<6> vin(24);
6755 VSeq<8> va(0), vb(16);
6756
6757 __ adr(tmpAddr, L_F00);
6758 __ ldr(v31, __ Q, tmpAddr); // 8H times 0x0f00
6759 __ add(condensed, condensed, condensedOffs);
6760
6761 __ BIND(L_loop);
6762 // load 96 (6 x 16B) byte values
6763 vs_ld3_post(vin, __ T16B, condensed);
6764
6765 // The front half of sequence vin (vin[0], vin[1] and vin[2])
6766 // holds 48 (16x3) contiguous bytes from memory striped
6767 // horizontally across each of the 16 byte lanes. Equivalently,
6768 // that is 16 pairs of 12-bit integers. Likewise the back half
6769 // holds the next 48 bytes in the same arrangement.
6770
6771 // Each vector in the front half can also be viewed as a vertical
6772 // strip across the 16 pairs of 12 bit integers. Each byte in
6773 // vin[0] stores the low 8 bits of the first int in a pair. Each
6774 // byte in vin[1] stores the high 4 bits of the first int and the
6775 // low 4 bits of the second int. Each byte in vin[2] stores the
6776 // high 8 bits of the second int. Likewise the vectors in second
6777 // half.
6778
6779 // Converting the data to 16-bit shorts requires first of all
6780 // expanding each of the 6 x 16B vectors into 6 corresponding
6781 // pairs of 8H vectors. Mask, shift and add operations on the
6782 // resulting vector pairs can be used to combine 4 and 8 bit
6783 // parts of related 8H vector elements.
6784 //
6785 // The middle vectors (vin[2] and vin[5]) are actually expanded
6786 // twice, one copy manipulated to provide the lower 4 bits
6787 // belonging to the first short in a pair and another copy
6788 // manipulated to provide the higher 4 bits belonging to the
6789 // second short in a pair. This is why the vector sequences va
6790 // and vb are used to hold the expanded 8H elements are of length 8.
6791
6792 // Expand vin[0] into va[0:1], and vin[1] into va[2:3] and va[4:5]
6793 // n.b. target elements 2 and 3 duplicate elements 4 and 5
6794 __ ushll(va[0], __ T8H, vin[0], __ T8B, 0);
6795 __ ushll2(va[1], __ T8H, vin[0], __ T16B, 0);
6796 __ ushll(va[2], __ T8H, vin[1], __ T8B, 0);
6797 __ ushll2(va[3], __ T8H, vin[1], __ T16B, 0);
6798 __ ushll(va[4], __ T8H, vin[1], __ T8B, 0);
6799 __ ushll2(va[5], __ T8H, vin[1], __ T16B, 0);
6800
6801 // likewise expand vin[3] into vb[0:1], and vin[4] into vb[2:3]
6802 // and vb[4:5]
6803 __ ushll(vb[0], __ T8H, vin[3], __ T8B, 0);
6804 __ ushll2(vb[1], __ T8H, vin[3], __ T16B, 0);
6805 __ ushll(vb[2], __ T8H, vin[4], __ T8B, 0);
6806 __ ushll2(vb[3], __ T8H, vin[4], __ T16B, 0);
6807 __ ushll(vb[4], __ T8H, vin[4], __ T8B, 0);
6808 __ ushll2(vb[5], __ T8H, vin[4], __ T16B, 0);
6809
6810 // shift lo byte of copy 1 of the middle stripe into the high byte
6811 __ shl(va[2], __ T8H, va[2], 8);
6812 __ shl(va[3], __ T8H, va[3], 8);
6813 __ shl(vb[2], __ T8H, vb[2], 8);
6814 __ shl(vb[3], __ T8H, vb[3], 8);
6815
6816 // expand vin[2] into va[6:7] and vin[5] into vb[6:7] but this
6817 // time pre-shifted by 4 to ensure top bits of input 12-bit int
6818 // are in bit positions [4..11].
6819 __ ushll(va[6], __ T8H, vin[2], __ T8B, 4);
6820 __ ushll2(va[7], __ T8H, vin[2], __ T16B, 4);
6821 __ ushll(vb[6], __ T8H, vin[5], __ T8B, 4);
6822 __ ushll2(vb[7], __ T8H, vin[5], __ T16B, 4);
6823
6824 // mask hi 4 bits of the 1st 12-bit int in a pair from copy1 and
6825 // shift lo 4 bits of the 2nd 12-bit int in a pair to the bottom of
6826 // copy2
6827 __ andr(va[2], __ T16B, va[2], v31);
6828 __ andr(va[3], __ T16B, va[3], v31);
6829 __ ushr(va[4], __ T8H, va[4], 4);
6830 __ ushr(va[5], __ T8H, va[5], 4);
6831 __ andr(vb[2], __ T16B, vb[2], v31);
6832 __ andr(vb[3], __ T16B, vb[3], v31);
6833 __ ushr(vb[4], __ T8H, vb[4], 4);
6834 __ ushr(vb[5], __ T8H, vb[5], 4);
6835
6836 // sum hi 4 bits and lo 8 bits of the 1st 12-bit int in each pair and
6837 // hi 8 bits plus lo 4 bits of the 2nd 12-bit int in each pair
6838 // n.b. the ordering ensures: i) inputs are consumed before they
6839 // are overwritten ii) the order of 16-bit results across successive
6840 // pairs of vectors in va and then vb reflects the order of the
6841 // corresponding 12-bit inputs
6842 __ addv(va[0], __ T8H, va[0], va[2]);
6843 __ addv(va[2], __ T8H, va[1], va[3]);
6844 __ addv(va[1], __ T8H, va[4], va[6]);
6845 __ addv(va[3], __ T8H, va[5], va[7]);
6846 __ addv(vb[0], __ T8H, vb[0], vb[2]);
6847 __ addv(vb[2], __ T8H, vb[1], vb[3]);
6848 __ addv(vb[1], __ T8H, vb[4], vb[6]);
6849 __ addv(vb[3], __ T8H, vb[5], vb[7]);
6850
6851 // store 64 results interleaved as shorts
6852 vs_st2_post(vs_front(va), __ T8H, parsed);
6853 vs_st2_post(vs_front(vb), __ T8H, parsed);
6854
6855 __ sub(parsedLength, parsedLength, 64);
6856 __ cmp(parsedLength, (u1)0);
6857 __ br(Assembler::GT, L_loop);
6858
6859 __ leave(); // required for proper stackwalking of RuntimeStub frame
6860 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6861 __ ret(lr);
6862
6863 // bind label and generate constant data used by this stub
6864 __ BIND(L_F00);
6865 __ emit_int64(0x0f000f000f000f00);
6866 __ emit_int64(0x0f000f000f000f00);
6867
6868 // record the stub entry and end
6869 store_archive_data(stub_id, start, __ pc());
6870
6871 return start;
6872 }
6873
6874 // Kyber Barrett reduce function.
6875 // Implements
6876 // static int implKyberBarrettReduce(short[] coeffs) {}
6877 //
6878 // coeffs (short[256]) = c_rarg0
6879 address generate_kyberBarrettReduce() {
6880 StubId stub_id = StubId::stubgen_kyberBarrettReduce_id;
6881 int entry_count = StubInfo::entry_count(stub_id);
6882 assert(entry_count == 1, "sanity check");
6883 address start = load_archive_data(stub_id);
6884 if (start != nullptr) {
6885 return start;
6886 }
6887 __ align(CodeEntryAlignment);
6888 StubCodeMark mark(this, stub_id);
6889 start = __ pc();
6890 __ enter();
6891
6892 const Register coeffs = c_rarg0;
6893
6894 const Register kyberConsts = r10;
6895 const Register result = r11;
6896
6897 // As above we process 256 sets of values in total i.e. 32 x
6898 // 8H quadwords. So, we can load, add and store the data in 3
6899 // groups of 11, 11 and 10 at a time i.e. we need to map sets
6900 // of 10 or 11 registers. A further constraint is that the
6901 // mapping needs to skip callee saves. So, we allocate the
6902 // register sequences using two 8 sequences, two 2 sequences
6903 // and two single registers.
6904 VSeq<8> vs1_1(0);
6905 VSeq<2> vs1_2(16);
6906 FloatRegister vs1_3 = v28;
6907 VSeq<8> vs2_1(18);
6908 VSeq<2> vs2_2(26);
6909 FloatRegister vs2_3 = v29;
6910
6911 // we also need a pair of corresponding constant sequences
6912
6913 VSeq<8> vc1_1(30, 0);
6914 VSeq<2> vc1_2(30, 0);
6915 FloatRegister vc1_3 = v30; // for kyber_q
6916
6917 VSeq<8> vc2_1(31, 0);
6918 VSeq<2> vc2_2(31, 0);
6919 FloatRegister vc2_3 = v31; // for kyberBarrettMultiplier
6920
6921 __ add(result, coeffs, 0);
6922 __ lea(kyberConsts,
6923 ExternalAddress((address) StubRoutines::aarch64::_kyberConsts));
6924
6925 // load q and the multiplier for the Barrett reduction
6926 __ add(kyberConsts, kyberConsts, 16);
6927 __ ldpq(vc1_3, vc2_3, kyberConsts);
6928
6929 for (int i = 0; i < 3; i++) {
6930 // load 80 or 88 coefficients
6931 vs_ldpq_post(vs1_1, coeffs);
6932 vs_ldpq_post(vs1_2, coeffs);
6933 if (i < 2) {
6934 __ ldr(vs1_3, __ Q, __ post(coeffs, 16));
6935 }
6936
6937 // vs2 <- (2 * vs1 * kyberBarrettMultiplier) >> 16
6938 vs_sqdmulh(vs2_1, __ T8H, vs1_1, vc2_1);
6939 vs_sqdmulh(vs2_2, __ T8H, vs1_2, vc2_2);
6940 if (i < 2) {
6941 __ sqdmulh(vs2_3, __ T8H, vs1_3, vc2_3);
6942 }
6943
6944 // vs2 <- (vs1 * kyberBarrettMultiplier) >> 26
6945 vs_sshr(vs2_1, __ T8H, vs2_1, 11);
6946 vs_sshr(vs2_2, __ T8H, vs2_2, 11);
6947 if (i < 2) {
6948 __ sshr(vs2_3, __ T8H, vs2_3, 11);
6949 }
6950
6951 // vs1 <- vs1 - vs2 * kyber_q
6952 vs_mlsv(vs1_1, __ T8H, vs2_1, vc1_1);
6953 vs_mlsv(vs1_2, __ T8H, vs2_2, vc1_2);
6954 if (i < 2) {
6955 __ mlsv(vs1_3, __ T8H, vs2_3, vc1_3);
6956 }
6957
6958 vs_stpq_post(vs1_1, result);
6959 vs_stpq_post(vs1_2, result);
6960 if (i < 2) {
6961 __ str(vs1_3, __ Q, __ post(result, 16));
6962 }
6963 }
6964
6965 __ leave(); // required for proper stackwalking of RuntimeStub frame
6966 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
6967 __ ret(lr);
6968
6969 // record the stub entry and end
6970 store_archive_data(stub_id, start, __ pc());
6971
6972 return start;
6973 }
6974
6975
6976 // Dilithium-specific montmul helper routines that generate parallel
6977 // code for, respectively, a single 4x4s vector sequence montmul or
6978 // two such multiplies in a row.
6979
6980 // Perform 16 32-bit Montgomery multiplications in parallel
6981 void dilithium_montmul16(const VSeq<4>& va, const VSeq<4>& vb, const VSeq<4>& vc,
6982 const VSeq<4>& vtmp, const VSeq<2>& vq) {
6983 // Use the helper routine to schedule a 4x4S Montgomery multiply.
6984 // It will assert that the register use is valid
6985 vs_montmul4(va, vb, vc, __ T4S, vtmp, vq);
6986 }
6987
6988 // Perform 2x16 32-bit Montgomery multiplications in parallel
6989 void dilithium_montmul32(const VSeq<8>& va, const VSeq<8>& vb, const VSeq<8>& vc,
6990 const VSeq<4>& vtmp, const VSeq<2>& vq) {
6991 // Schedule two successive 4x4S multiplies via the montmul helper
6992 // on the front and back halves of va, vb and vc. The helper will
6993 // assert that the register use has no overlap conflicts on each
6994 // individual call but we also need to ensure that the necessary
6995 // disjoint/equality constraints are met across both calls.
6996
6997 // vb, vc, vtmp and vq must be disjoint. va must either be
6998 // disjoint from all other registers or equal vc
6999
7000 assert(vs_disjoint(vb, vc), "vb and vc overlap");
7001 assert(vs_disjoint(vb, vq), "vb and vq overlap");
7002 assert(vs_disjoint(vb, vtmp), "vb and vtmp overlap");
7003
7004 assert(vs_disjoint(vc, vq), "vc and vq overlap");
7005 assert(vs_disjoint(vc, vtmp), "vc and vtmp overlap");
7006
7007 assert(vs_disjoint(vq, vtmp), "vq and vtmp overlap");
7008
7009 assert(vs_disjoint(va, vc) || vs_same(va, vc), "va and vc neither disjoint nor equal");
7010 assert(vs_disjoint(va, vb), "va and vb overlap");
7011 assert(vs_disjoint(va, vq), "va and vq overlap");
7012 assert(vs_disjoint(va, vtmp), "va and vtmp overlap");
7013
7014 // We multiply the front and back halves of each sequence 4 at a
7015 // time because
7016 //
7017 // 1) we are currently only able to get 4-way instruction
7018 // parallelism at best
7019 //
7020 // 2) we need registers for the constants in vq and temporary
7021 // scratch registers to hold intermediate results so vtmp can only
7022 // be a VSeq<4> which means we only have 4 scratch slots.
7023
7024 vs_montmul4(vs_front(va), vs_front(vb), vs_front(vc), __ T4S, vtmp, vq);
7025 vs_montmul4(vs_back(va), vs_back(vb), vs_back(vc), __ T4S, vtmp, vq);
7026 }
7027
7028 // Perform combined montmul then add/sub on 4x4S vectors.
7029 void dilithium_montmul16_sub_add(
7030 const VSeq<4>& va0, const VSeq<4>& va1, const VSeq<4>& vc,
7031 const VSeq<4>& vtmp, const VSeq<2>& vq) {
7032 // compute a = montmul(a1, c)
7033 dilithium_montmul16(vc, va1, vc, vtmp, vq);
7034 // ouptut a1 = a0 - a
7035 vs_subv(va1, __ T4S, va0, vc);
7036 // and a0 = a0 + a
7037 vs_addv(va0, __ T4S, va0, vc);
7038 }
7039
7040 // Perform combined add/sub then montmul on 4x4S vectors.
7041 void dilithium_sub_add_montmul16(
7042 const VSeq<4>& va0, const VSeq<4>& va1, const VSeq<4>& vb,
7043 const VSeq<4>& vtmp1, const VSeq<4>& vtmp2, const VSeq<2>& vq) {
7044 // compute c = a0 - a1
7045 vs_subv(vtmp1, __ T4S, va0, va1);
7046 // output a0 = a0 + a1
7047 vs_addv(va0, __ T4S, va0, va1);
7048 // output a1 = b montmul c
7049 dilithium_montmul16(va1, vtmp1, vb, vtmp2, vq);
7050 }
7051
7052 // At these levels, the indices that correspond to the 'j's (and 'j+l's)
7053 // in the Java implementation come in sequences of at least 8, so we
7054 // can use ldpq to collect the corresponding data into pairs of vector
7055 // registers.
7056 // We collect the coefficients corresponding to the 'j+l' indexes into
7057 // the vector registers v0-v7, the zetas into the vector registers v16-v23
7058 // then we do the (Montgomery) multiplications by the zetas in parallel
7059 // into v16-v23, load the coeffs corresponding to the 'j' indexes into
7060 // v0-v7, then do the additions into v24-v31 and the subtractions into
7061 // v0-v7 and finally save the results back to the coeffs array.
7062 void dilithiumNttLevel0_4(const Register dilithiumConsts,
7063 const Register coeffs, const Register zetas) {
7064 int c1 = 0;
7065 int c2 = 512;
7066 int startIncr;
7067 // don't use callee save registers v8 - v15
7068 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7069 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7070 VSeq<2> vq(30); // n.b. constants overlap vs3
7071 int offsets[4] = { 0, 32, 64, 96 };
7072
7073 for (int level = 0; level < 5; level++) {
7074 int c1Start = c1;
7075 int c2Start = c2;
7076 if (level == 3) {
7077 offsets[1] = 32;
7078 offsets[2] = 128;
7079 offsets[3] = 160;
7080 } else if (level == 4) {
7081 offsets[1] = 64;
7082 offsets[2] = 128;
7083 offsets[3] = 192;
7084 }
7085
7086 // For levels 1 - 4 we simply load 2 x 4 adjacent values at a
7087 // time at 4 different offsets and multiply them in order by the
7088 // next set of input values. So we employ indexed load and store
7089 // pair instructions with arrangement 4S.
7090 for (int i = 0; i < 4; i++) {
7091 // reload q and qinv
7092 vs_ldpq(vq, dilithiumConsts); // qInv, q
7093 // load 8x4S coefficients via second start pos == c2
7094 vs_ldpq_indexed(vs1, coeffs, c2Start, offsets);
7095 // load next 8x4S inputs == b
7096 vs_ldpq_post(vs2, zetas);
7097 // compute a == c2 * b mod MONT_Q
7098 dilithium_montmul32(vs2, vs1, vs2, vtmp, vq);
7099 // load 8x4s coefficients via first start pos == c1
7100 vs_ldpq_indexed(vs1, coeffs, c1Start, offsets);
7101 // compute a1 = c1 + a
7102 vs_addv(vs3, __ T4S, vs1, vs2);
7103 // compute a2 = c1 - a
7104 vs_subv(vs1, __ T4S, vs1, vs2);
7105 // output a1 and a2
7106 vs_stpq_indexed(vs3, coeffs, c1Start, offsets);
7107 vs_stpq_indexed(vs1, coeffs, c2Start, offsets);
7108
7109 int k = 4 * level + i;
7110
7111 if (k > 7) {
7112 startIncr = 256;
7113 } else if (k == 5) {
7114 startIncr = 384;
7115 } else {
7116 startIncr = 128;
7117 }
7118
7119 c1Start += startIncr;
7120 c2Start += startIncr;
7121 }
7122
7123 c2 /= 2;
7124 }
7125 }
7126
7127 // Dilithium NTT function except for the final "normalization" to |coeff| < Q.
7128 // Implements the method
7129 // static int implDilithiumAlmostNtt(int[] coeffs, int zetas[]) {}
7130 // of the Java class sun.security.provider
7131 //
7132 // coeffs (int[256]) = c_rarg0
7133 // zetas (int[256]) = c_rarg1
7134 address generate_dilithiumAlmostNtt() {
7135 StubId stub_id = StubId::stubgen_dilithiumAlmostNtt_id;
7136 int entry_count = StubInfo::entry_count(stub_id);
7137 assert(entry_count == 1, "sanity check");
7138 address start = load_archive_data(stub_id);
7139 if (start != nullptr) {
7140 return start;
7141 }
7142 __ align(CodeEntryAlignment);
7143 StubCodeMark mark(this, stub_id);
7144 start = __ pc();
7145 __ enter();
7146
7147 const Register coeffs = c_rarg0;
7148 const Register zetas = c_rarg1;
7149
7150 const Register tmpAddr = r9;
7151 const Register dilithiumConsts = r10;
7152 const Register result = r11;
7153 // don't use callee save registers v8 - v15
7154 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7155 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7156 VSeq<2> vq(30); // n.b. constants overlap vs3
7157 int offsets[4] = { 0, 32, 64, 96};
7158 int offsets1[8] = { 16, 48, 80, 112, 144, 176, 208, 240 };
7159 int offsets2[8] = { 0, 32, 64, 96, 128, 160, 192, 224 };
7160 __ add(result, coeffs, 0);
7161 __ lea(dilithiumConsts,
7162 ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts));
7163
7164 // Each level represents one iteration of the outer for loop of the Java version.
7165
7166 // level 0-4
7167 dilithiumNttLevel0_4(dilithiumConsts, coeffs, zetas);
7168
7169 // level 5
7170
7171 // At level 5 the coefficients we need to combine with the zetas
7172 // are grouped in memory in blocks of size 4. So, for both sets of
7173 // coefficients we load 4 adjacent values at 8 different offsets
7174 // using an indexed ldr with register variant Q and multiply them
7175 // in sequence order by the next set of inputs. Likewise we store
7176 // the results using an indexed str with register variant Q.
7177 for (int i = 0; i < 1024; i += 256) {
7178 // reload constants q, qinv each iteration as they get clobbered later
7179 vs_ldpq(vq, dilithiumConsts); // qInv, q
7180 // load 32 (8x4S) coefficients via first offsets = c1
7181 vs_ldr_indexed(vs1, __ Q, coeffs, i, offsets1);
7182 // load next 32 (8x4S) inputs = b
7183 vs_ldpq_post(vs2, zetas);
7184 // a = b montul c1
7185 dilithium_montmul32(vs2, vs1, vs2, vtmp, vq);
7186 // load 32 (8x4S) coefficients via second offsets = c2
7187 vs_ldr_indexed(vs1, __ Q, coeffs, i, offsets2);
7188 // add/sub with result of multiply
7189 vs_addv(vs3, __ T4S, vs1, vs2); // a1 = a - c2
7190 vs_subv(vs1, __ T4S, vs1, vs2); // a0 = a + c1
7191 // write back new coefficients using same offsets
7192 vs_str_indexed(vs3, __ Q, coeffs, i, offsets2);
7193 vs_str_indexed(vs1, __ Q, coeffs, i, offsets1);
7194 }
7195
7196 // level 6
7197 // At level 6 the coefficients we need to combine with the zetas
7198 // are grouped in memory in pairs, the first two being montmul
7199 // inputs and the second add/sub inputs. We can still implement
7200 // the montmul+sub+add using 4-way parallelism but only if we
7201 // combine the coefficients with the zetas 16 at a time. We load 8
7202 // adjacent values at 4 different offsets using an ld2 load with
7203 // arrangement 2D. That interleaves the lower and upper halves of
7204 // each pair of quadwords into successive vector registers. We
7205 // then need to montmul the 4 even elements of the coefficients
7206 // register sequence by the zetas in order and then add/sub the 4
7207 // odd elements of the coefficients register sequence. We use an
7208 // equivalent st2 operation to store the results back into memory
7209 // de-interleaved.
7210 for (int i = 0; i < 1024; i += 128) {
7211 // reload constants q, qinv each iteration as they get clobbered later
7212 vs_ldpq(vq, dilithiumConsts); // qInv, q
7213 // load interleaved 16 (4x2D) coefficients via offsets
7214 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, i, offsets);
7215 // load next 16 (4x4S) inputs
7216 vs_ldpq_post(vs_front(vs2), zetas);
7217 // mont multiply odd elements of vs1 by vs2 and add/sub into odds/evens
7218 dilithium_montmul16_sub_add(vs_even(vs1), vs_odd(vs1),
7219 vs_front(vs2), vtmp, vq);
7220 // store interleaved 16 (4x2D) coefficients via offsets
7221 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, i, offsets);
7222 }
7223
7224 // level 7
7225 // At level 7 the coefficients we need to combine with the zetas
7226 // occur singly with montmul inputs alternating with add/sub
7227 // inputs. Once again we can use 4-way parallelism to combine 16
7228 // zetas at a time. However, we have to load 8 adjacent values at
7229 // 4 different offsets using an ld2 load with arrangement 4S. That
7230 // interleaves the odd words of each pair into one
7231 // coefficients vector register and the even words of the pair
7232 // into the next register. We then need to montmul the 4 even
7233 // elements of the coefficients register sequence by the zetas in
7234 // order and then add/sub the 4 odd elements of the coefficients
7235 // register sequence. We use an equivalent st2 operation to store
7236 // the results back into memory de-interleaved.
7237
7238 for (int i = 0; i < 1024; i += 128) {
7239 // reload constants q, qinv each iteration as they get clobbered later
7240 vs_ldpq(vq, dilithiumConsts); // qInv, q
7241 // load interleaved 16 (4x4S) coefficients via offsets
7242 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, i, offsets);
7243 // load next 16 (4x4S) inputs
7244 vs_ldpq_post(vs_front(vs2), zetas);
7245 // mont multiply odd elements of vs1 by vs2 and add/sub into odds/evens
7246 dilithium_montmul16_sub_add(vs_even(vs1), vs_odd(vs1),
7247 vs_front(vs2), vtmp, vq);
7248 // store interleaved 16 (4x4S) coefficients via offsets
7249 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, i, offsets);
7250 }
7251 __ leave(); // required for proper stackwalking of RuntimeStub frame
7252 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
7253 __ ret(lr);
7254
7255 // record the stub entry and end
7256 store_archive_data(stub_id, start, __ pc());
7257
7258 return start;
7259 }
7260
7261 // At these levels, the indices that correspond to the 'j's (and 'j+l's)
7262 // in the Java implementation come in sequences of at least 8, so we
7263 // can use ldpq to collect the corresponding data into pairs of vector
7264 // registers
7265 // We collect the coefficients that correspond to the 'j's into vs1
7266 // the coefficiets that correspond to the 'j+l's into vs2 then
7267 // do the additions into vs3 and the subtractions into vs1 then
7268 // save the result of the additions, load the zetas into vs2
7269 // do the (Montgomery) multiplications by zeta in parallel into vs2
7270 // finally save the results back to the coeffs array
7271 void dilithiumInverseNttLevel3_7(const Register dilithiumConsts,
7272 const Register coeffs, const Register zetas) {
7273 int c1 = 0;
7274 int c2 = 32;
7275 int startIncr;
7276 int offsets[4];
7277 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7278 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7279 VSeq<2> vq(30); // n.b. constants overlap vs3
7280
7281 offsets[0] = 0;
7282
7283 for (int level = 3; level < 8; level++) {
7284 int c1Start = c1;
7285 int c2Start = c2;
7286 if (level == 3) {
7287 offsets[1] = 64;
7288 offsets[2] = 128;
7289 offsets[3] = 192;
7290 } else if (level == 4) {
7291 offsets[1] = 32;
7292 offsets[2] = 128;
7293 offsets[3] = 160;
7294 } else {
7295 offsets[1] = 32;
7296 offsets[2] = 64;
7297 offsets[3] = 96;
7298 }
7299
7300 // For levels 3 - 7 we simply load 2 x 4 adjacent values at a
7301 // time at 4 different offsets and multiply them in order by the
7302 // next set of input values. So we employ indexed load and store
7303 // pair instructions with arrangement 4S.
7304 for (int i = 0; i < 4; i++) {
7305 // load v1 32 (8x4S) coefficients relative to first start index
7306 vs_ldpq_indexed(vs1, coeffs, c1Start, offsets);
7307 // load v2 32 (8x4S) coefficients relative to second start index
7308 vs_ldpq_indexed(vs2, coeffs, c2Start, offsets);
7309 // a0 = v1 + v2 -- n.b. clobbers vqs
7310 vs_addv(vs3, __ T4S, vs1, vs2);
7311 // a1 = v1 - v2
7312 vs_subv(vs1, __ T4S, vs1, vs2);
7313 // save a1 relative to first start index
7314 vs_stpq_indexed(vs3, coeffs, c1Start, offsets);
7315 // load constants q, qinv each iteration as they get clobbered above
7316 vs_ldpq(vq, dilithiumConsts); // qInv, q
7317 // load b next 32 (8x4S) inputs
7318 vs_ldpq_post(vs2, zetas);
7319 // a = a1 montmul b
7320 dilithium_montmul32(vs2, vs1, vs2, vtmp, vq);
7321 // save a relative to second start index
7322 vs_stpq_indexed(vs2, coeffs, c2Start, offsets);
7323
7324 int k = 4 * level + i;
7325
7326 if (k < 24) {
7327 startIncr = 256;
7328 } else if (k == 25) {
7329 startIncr = 384;
7330 } else {
7331 startIncr = 128;
7332 }
7333
7334 c1Start += startIncr;
7335 c2Start += startIncr;
7336 }
7337
7338 c2 *= 2;
7339 }
7340 }
7341
7342 // Dilithium Inverse NTT function except the final mod Q division by 2^256.
7343 // Implements the method
7344 // static int implDilithiumAlmostInverseNtt(int[] coeffs, int[] zetas) {} of
7345 // the sun.security.provider.ML_DSA class.
7346 //
7347 // coeffs (int[256]) = c_rarg0
7348 // zetas (int[256]) = c_rarg1
7349 address generate_dilithiumAlmostInverseNtt() {
7350 StubId stub_id = StubId::stubgen_dilithiumAlmostInverseNtt_id;
7351 int entry_count = StubInfo::entry_count(stub_id);
7352 assert(entry_count == 1, "sanity check");
7353 address start = load_archive_data(stub_id);
7354 if (start != nullptr) {
7355 return start;
7356 }
7357 __ align(CodeEntryAlignment);
7358 StubCodeMark mark(this, stub_id);
7359 start = __ pc();
7360 __ enter();
7361
7362 const Register coeffs = c_rarg0;
7363 const Register zetas = c_rarg1;
7364
7365 const Register tmpAddr = r9;
7366 const Register dilithiumConsts = r10;
7367 const Register result = r11;
7368 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7369 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7370 VSeq<2> vq(30); // n.b. constants overlap vs3
7371 int offsets[4] = { 0, 32, 64, 96 };
7372 int offsets1[8] = { 0, 32, 64, 96, 128, 160, 192, 224 };
7373 int offsets2[8] = { 16, 48, 80, 112, 144, 176, 208, 240 };
7374
7375 __ add(result, coeffs, 0);
7376 __ lea(dilithiumConsts,
7377 ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts));
7378
7379 // Each level represents one iteration of the outer for loop of the Java version
7380
7381 // level 0
7382 // At level 0 we need to interleave adjacent quartets of
7383 // coefficients before we multiply and add/sub by the next 16
7384 // zetas just as we did for level 7 in the multiply code. So we
7385 // load and store the values using an ld2/st2 with arrangement 4S.
7386 for (int i = 0; i < 1024; i += 128) {
7387 // load constants q, qinv
7388 // n.b. this can be moved out of the loop as they do not get
7389 // clobbered by first two loops
7390 vs_ldpq(vq, dilithiumConsts); // qInv, q
7391 // a0/a1 load interleaved 32 (8x4S) coefficients
7392 vs_ld2_indexed(vs1, __ T4S, coeffs, tmpAddr, i, offsets);
7393 // b load next 32 (8x4S) inputs
7394 vs_ldpq_post(vs_front(vs2), zetas);
7395 // compute in parallel (a0, a1) = (a0 + a1, (a0 - a1) montmul b)
7396 // n.b. second half of vs2 provides temporary register storage
7397 dilithium_sub_add_montmul16(vs_even(vs1), vs_odd(vs1),
7398 vs_front(vs2), vs_back(vs2), vtmp, vq);
7399 // a0/a1 store interleaved 32 (8x4S) coefficients
7400 vs_st2_indexed(vs1, __ T4S, coeffs, tmpAddr, i, offsets);
7401 }
7402
7403 // level 1
7404 // At level 1 we need to interleave pairs of adjacent pairs of
7405 // coefficients before we multiply by the next 16 zetas just as we
7406 // did for level 6 in the multiply code. So we load and store the
7407 // values an ld2/st2 with arrangement 2D.
7408 for (int i = 0; i < 1024; i += 128) {
7409 // a0/a1 load interleaved 32 (8x2D) coefficients
7410 vs_ld2_indexed(vs1, __ T2D, coeffs, tmpAddr, i, offsets);
7411 // b load next 16 (4x4S) inputs
7412 vs_ldpq_post(vs_front(vs2), zetas);
7413 // compute in parallel (a0, a1) = (a0 + a1, (a0 - a1) montmul b)
7414 // n.b. second half of vs2 provides temporary register storage
7415 dilithium_sub_add_montmul16(vs_even(vs1), vs_odd(vs1),
7416 vs_front(vs2), vs_back(vs2), vtmp, vq);
7417 // a0/a1 store interleaved 32 (8x2D) coefficients
7418 vs_st2_indexed(vs1, __ T2D, coeffs, tmpAddr, i, offsets);
7419 }
7420
7421 // level 2
7422 // At level 2 coefficients come in blocks of 4. So, we load 4
7423 // adjacent coefficients at 8 distinct offsets for both the first
7424 // and second coefficient sequences, using an ldr with register
7425 // variant Q then combine them with next set of 32 zetas. Likewise
7426 // we store the results using an str with register variant Q.
7427 for (int i = 0; i < 1024; i += 256) {
7428 // c0 load 32 (8x4S) coefficients via first offsets
7429 vs_ldr_indexed(vs1, __ Q, coeffs, i, offsets1);
7430 // c1 load 32 (8x4S) coefficients via second offsets
7431 vs_ldr_indexed(vs2, __ Q, coeffs, i, offsets2);
7432 // a0 = c0 + c1 n.b. clobbers vq which overlaps vs3
7433 vs_addv(vs3, __ T4S, vs1, vs2);
7434 // c = c0 - c1
7435 vs_subv(vs1, __ T4S, vs1, vs2);
7436 // store a0 32 (8x4S) coefficients via first offsets
7437 vs_str_indexed(vs3, __ Q, coeffs, i, offsets1);
7438 // b load 32 (8x4S) next inputs
7439 vs_ldpq_post(vs2, zetas);
7440 // reload constants q, qinv -- they were clobbered earlier
7441 vs_ldpq(vq, dilithiumConsts); // qInv, q
7442 // compute a1 = b montmul c
7443 dilithium_montmul32(vs2, vs1, vs2, vtmp, vq);
7444 // store a1 32 (8x4S) coefficients via second offsets
7445 vs_str_indexed(vs2, __ Q, coeffs, i, offsets2);
7446 }
7447
7448 // level 3-7
7449 dilithiumInverseNttLevel3_7(dilithiumConsts, coeffs, zetas);
7450
7451 __ leave(); // required for proper stackwalking of RuntimeStub frame
7452 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
7453 __ ret(lr);
7454
7455 // record the stub entry and end
7456 store_archive_data(stub_id, start, __ pc());
7457
7458 return start;
7459 }
7460
7461 // Dilithium multiply polynomials in the NTT domain.
7462 // Straightforward implementation of the method
7463 // static int implDilithiumNttMult(
7464 // int[] product, int[] coeffs1, int[] coeffs2) {}
7465 // of the sun.security.provider.ML_DSA class.
7466 //
7467 // result (int[256]) = c_rarg0
7468 // poly1 (int[256]) = c_rarg1
7469 // poly2 (int[256]) = c_rarg2
7470 address generate_dilithiumNttMult() {
7471 StubId stub_id = StubId::stubgen_dilithiumNttMult_id;
7472 int entry_count = StubInfo::entry_count(stub_id);
7473 assert(entry_count == 1, "sanity check");
7474 address start = load_archive_data(stub_id);
7475 if (start != nullptr) {
7476 return start;
7477 }
7478 __ align(CodeEntryAlignment);
7479 StubCodeMark mark(this, stub_id);
7480 start = __ pc();
7481 __ enter();
7482
7483 Label L_loop;
7484
7485 const Register result = c_rarg0;
7486 const Register poly1 = c_rarg1;
7487 const Register poly2 = c_rarg2;
7488
7489 const Register dilithiumConsts = r10;
7490 const Register len = r11;
7491
7492 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7493 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7494 VSeq<2> vq(30); // n.b. constants overlap vs3
7495 VSeq<8> vrsquare(29, 0); // for montmul by constant RSQUARE
7496
7497 __ lea(dilithiumConsts,
7498 ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts));
7499
7500 // load constants q, qinv
7501 vs_ldpq(vq, dilithiumConsts); // qInv, q
7502 // load constant rSquare into v29
7503 __ ldr(v29, __ Q, Address(dilithiumConsts, 48)); // rSquare
7504
7505 __ mov(len, zr);
7506 __ add(len, len, 1024);
7507
7508 __ BIND(L_loop);
7509
7510 // b load 32 (8x4S) next inputs from poly1
7511 vs_ldpq_post(vs1, poly1);
7512 // c load 32 (8x4S) next inputs from poly2
7513 vs_ldpq_post(vs2, poly2);
7514 // compute a = b montmul c
7515 dilithium_montmul32(vs2, vs1, vs2, vtmp, vq);
7516 // compute a = rsquare montmul a
7517 dilithium_montmul32(vs2, vrsquare, vs2, vtmp, vq);
7518 // save a 32 (8x4S) results
7519 vs_stpq_post(vs2, result);
7520
7521 __ sub(len, len, 128);
7522 __ cmp(len, (u1)128);
7523 __ br(Assembler::GE, L_loop);
7524
7525 __ leave(); // required for proper stackwalking of RuntimeStub frame
7526 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
7527 __ ret(lr);
7528
7529 // record the stub entry and end
7530 store_archive_data(stub_id, start, __ pc());
7531
7532 return start;
7533 }
7534
7535 // Dilithium Montgomery multiply an array by a constant.
7536 // A straightforward implementation of the method
7537 // static int implDilithiumMontMulByConstant(int[] coeffs, int constant) {}
7538 // of the sun.security.provider.ML_DSA class
7539 //
7540 // coeffs (int[256]) = c_rarg0
7541 // constant (int) = c_rarg1
7542 address generate_dilithiumMontMulByConstant() {
7543 StubId stub_id = StubId::stubgen_dilithiumMontMulByConstant_id;
7544 int entry_count = StubInfo::entry_count(stub_id);
7545 assert(entry_count == 1, "sanity check");
7546 address start = load_archive_data(stub_id);
7547 if (start != nullptr) {
7548 return start;
7549 }
7550 __ align(CodeEntryAlignment);
7551 StubCodeMark mark(this, stub_id);
7552 start = __ pc();
7553 __ enter();
7554
7555 Label L_loop;
7556
7557 const Register coeffs = c_rarg0;
7558 const Register constant = c_rarg1;
7559
7560 const Register dilithiumConsts = r10;
7561 const Register result = r11;
7562 const Register len = r12;
7563
7564 VSeq<8> vs1(0), vs2(16), vs3(24); // 3 sets of 8x4s inputs/outputs
7565 VSeq<4> vtmp = vs_front(vs3); // n.b. tmp registers overlap vs3
7566 VSeq<2> vq(30); // n.b. constants overlap vs3
7567 VSeq<8> vconst(29, 0); // for montmul by constant
7568
7569 // results track inputs
7570 __ add(result, coeffs, 0);
7571 __ lea(dilithiumConsts,
7572 ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts));
7573
7574 // load constants q, qinv -- they do not get clobbered by first two loops
7575 vs_ldpq(vq, dilithiumConsts); // qInv, q
7576 // copy caller supplied constant across vconst
7577 __ dup(vconst[0], __ T4S, constant);
7578 __ mov(len, zr);
7579 __ add(len, len, 1024);
7580
7581 __ BIND(L_loop);
7582
7583 // load next 32 inputs
7584 vs_ldpq_post(vs2, coeffs);
7585 // mont mul by constant
7586 dilithium_montmul32(vs2, vconst, vs2, vtmp, vq);
7587 // write next 32 results
7588 vs_stpq_post(vs2, result);
7589
7590 __ sub(len, len, 128);
7591 __ cmp(len, (u1)128);
7592 __ br(Assembler::GE, L_loop);
7593
7594 __ leave(); // required for proper stackwalking of RuntimeStub frame
7595 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
7596 __ ret(lr);
7597
7598 // record the stub entry and end
7599 store_archive_data(stub_id, start, __ pc());
7600
7601 return start;
7602 }
7603
7604 // Dilithium decompose poly.
7605 // Implements the method
7606 // static int implDilithiumDecomposePoly(int[] input, int[] lowPart, int[] highPart,
7607 // int twoGamma2, int multiplier) {
7608 // of the sun.security.provider.ML_DSA class
7609 //
7610 // input (int[256]) = c_rarg0
7611 // lowPart (int[256]) = c_rarg1
7612 // highPart (int[256]) = c_rarg2
7613 // twoGamma2 (int) = c_rarg3
7614 // multiplier (int) = c_rarg4
7615 address generate_dilithiumDecomposePoly() {
7616 StubId stub_id = StubId::stubgen_dilithiumDecomposePoly_id;
7617 int entry_count = StubInfo::entry_count(stub_id);
7618 assert(entry_count == 1, "sanity check");
7619 address start = load_archive_data(stub_id);
7620 if (start != nullptr) {
7621 return start;
7622 }
7623 __ align(CodeEntryAlignment);
7624 StubCodeMark mark(this, stub_id);
7625 start = __ pc();
7626 Label L_loop;
7627
7628 const Register input = c_rarg0;
7629 const Register lowPart = c_rarg1;
7630 const Register highPart = c_rarg2;
7631 const Register twoGamma2 = c_rarg3;
7632 const Register multiplier = c_rarg4;
7633
7634 const Register len = r9;
7635 const Register dilithiumConsts = r10;
7636 const Register tmp = r11;
7637
7638 // 6 independent sets of 4x4s values
7639 VSeq<4> vs1(0), vs2(4), vs3(8);
7640 VSeq<4> vs4(12), vs5(16), vtmp(20);
7641
7642 // 7 constants for cross-multiplying
7643 VSeq<4> one(25, 0);
7644 VSeq<4> qminus1(26, 0);
7645 VSeq<4> g2(27, 0);
7646 VSeq<4> twog2(28, 0);
7647 VSeq<4> mult(29, 0);
7648 VSeq<4> q(30, 0);
7649 VSeq<4> qadd(31, 0);
7650
7651 __ enter();
7652
7653 __ lea(dilithiumConsts,
7654 ExternalAddress((address) StubRoutines::aarch64::_dilithiumConsts));
7655
7656 // save callee-saved registers
7657 __ stpd(v8, v9, __ pre(sp, -64));
7658 __ stpd(v10, v11, Address(sp, 16));
7659 __ stpd(v12, v13, Address(sp, 32));
7660 __ stpd(v14, v15, Address(sp, 48));
7661
7662 // populate constant registers
7663 __ mov(tmp, zr);
7664 __ add(tmp, tmp, 1);
7665 __ dup(one[0], __ T4S, tmp); // 1
7666 __ ldr(q[0], __ Q, Address(dilithiumConsts, 16)); // q
7667 __ ldr(qadd[0], __ Q, Address(dilithiumConsts, 64)); // addend for mod q reduce
7668 __ dup(twog2[0], __ T4S, twoGamma2); // 2 * gamma2
7669 __ dup(mult[0], __ T4S, multiplier); // multiplier for mod 2 * gamma reduce
7670 __ subv(qminus1[0], __ T4S, v30, v25); // q - 1
7671 __ sshr(g2[0], __ T4S, v28, 1); // gamma2
7672
7673 __ mov(len, zr);
7674 __ add(len, len, 1024);
7675
7676 __ BIND(L_loop);
7677
7678 // load next 4x4S inputs interleaved: rplus --> vs1
7679 __ ld4(vs1[0], vs1[1], vs1[2], vs1[3], __ T4S, __ post(input, 64));
7680
7681 // rplus = rplus - ((rplus + qadd) >> 23) * q
7682 vs_addv(vtmp, __ T4S, vs1, qadd);
7683 vs_sshr(vtmp, __ T4S, vtmp, 23);
7684 vs_mulv(vtmp, __ T4S, vtmp, q);
7685 vs_subv(vs1, __ T4S, vs1, vtmp);
7686
7687 // rplus = rplus + ((rplus >> 31) & dilithium_q);
7688 vs_sshr(vtmp, __ T4S, vs1, 31);
7689 vs_andr(vtmp, vtmp, q);
7690 vs_addv(vs1, __ T4S, vs1, vtmp);
7691
7692 // quotient --> vs2
7693 // int quotient = (rplus * multiplier) >> 22;
7694 vs_mulv(vtmp, __ T4S, vs1, mult);
7695 vs_sshr(vs2, __ T4S, vtmp, 22);
7696
7697 // r0 --> vs3
7698 // int r0 = rplus - quotient * twoGamma2;
7699 vs_mulv(vtmp, __ T4S, vs2, twog2);
7700 vs_subv(vs3, __ T4S, vs1, vtmp);
7701
7702 // mask --> vs4
7703 // int mask = (twoGamma2 - r0) >> 22;
7704 vs_subv(vtmp, __ T4S, twog2, vs3);
7705 vs_sshr(vs4, __ T4S, vtmp, 22);
7706
7707 // r0 -= (mask & twoGamma2);
7708 vs_andr(vtmp, vs4, twog2);
7709 vs_subv(vs3, __ T4S, vs3, vtmp);
7710
7711 // quotient += (mask & 1);
7712 vs_andr(vtmp, vs4, one);
7713 vs_addv(vs2, __ T4S, vs2, vtmp);
7714
7715 // mask = (twoGamma2 / 2 - r0) >> 31;
7716 vs_subv(vtmp, __ T4S, g2, vs3);
7717 vs_sshr(vs4, __ T4S, vtmp, 31);
7718
7719 // r0 -= (mask & twoGamma2);
7720 vs_andr(vtmp, vs4, twog2);
7721 vs_subv(vs3, __ T4S, vs3, vtmp);
7722
7723 // quotient += (mask & 1);
7724 vs_andr(vtmp, vs4, one);
7725 vs_addv(vs2, __ T4S, vs2, vtmp);
7726
7727 // r1 --> vs5
7728 // int r1 = rplus - r0 - (dilithium_q - 1);
7729 vs_subv(vtmp, __ T4S, vs1, vs3);
7730 vs_subv(vs5, __ T4S, vtmp, qminus1);
7731
7732 // r1 --> vs1 (overwriting rplus)
7733 // r1 = (r1 | (-r1)) >> 31; // 0 if rplus - r0 == (dilithium_q - 1), -1 otherwise
7734 vs_negr(vtmp, __ T4S, vs5);
7735 vs_orr(vtmp, vs5, vtmp);
7736 vs_sshr(vs1, __ T4S, vtmp, 31);
7737
7738 // r0 += ~r1;
7739 vs_notr(vtmp, vs1);
7740 vs_addv(vs3, __ T4S, vs3, vtmp);
7741
7742 // r1 = r1 & quotient;
7743 vs_andr(vs1, vs2, vs1);
7744
7745 // store results interleaved
7746 // lowPart[m] = r0;
7747 // highPart[m] = r1;
7748 __ st4(vs3[0], vs3[1], vs3[2], vs3[3], __ T4S, __ post(lowPart, 64));
7749 __ st4(vs1[0], vs1[1], vs1[2], vs1[3], __ T4S, __ post(highPart, 64));
7750
7751 __ sub(len, len, 64);
7752 __ cmp(len, (u1)64);
7753 __ br(Assembler::GE, L_loop);
7754
7755 // restore callee-saved vector registers
7756 __ ldpd(v14, v15, Address(sp, 48));
7757 __ ldpd(v12, v13, Address(sp, 32));
7758 __ ldpd(v10, v11, Address(sp, 16));
7759 __ ldpd(v8, v9, __ post(sp, 64));
7760
7761 __ leave(); // required for proper stackwalking of RuntimeStub frame
7762 __ mov(r0, zr); // return 0 (Java callees return 1. Caller ignores the return value)
7763 __ ret(lr);
7764
7765 // record the stub entry and end
7766 store_archive_data(stub_id, start, __ pc());
7767
7768 return start;
7769 }
7770
7771 static constexpr int montMulP256Shift1 = 12; // 64 - bits per limb
7772 static constexpr int montMulP256Shift2 = 52; // bits per limb
7773 // stack space needed for carry computation
7774 static constexpr int cDataSize = 6 * BytesPerLong;
7775 // stack space needed for data computed by the neon side
7776 static constexpr int mulDataSize = 16 * BytesPerLong;
7777
7778
7779 // Subroutine used by the 52 x 52 bit multiplication algorithm in
7780 // generate_intpoly_montgomeryMult_P256().
7781 // This function computes partial results of eight 52 x 52 bit multiplications,
7782 // where the multiplicands are stored as 64-bit values, specifically
7783 // (b_0, b_1, b_2, b_3) * (a_3, a_4). (The 4 calls to this function
7784 // together provide the results of these limb-multiplications.)
7785 // Calls to this function accept either the low 32 bits or high 20 bits
7786 // of each b_i packed into bs in ascending order. a_3 and a_4 are packed
7787 // into successive 64 bit elements of as. lane selects the low 32 or high
7788 // 20 bits of each a_j value. So four calls with the appropriate parameters
7789 // will produce the 64-bit low32 * low32, low32 * high20, high20 * low32,
7790 // high20 * high20 values in the output register sequences vs. The
7791 // 64-bit partial products are returned in vs in ascending order:
7792 // vs[0] = (b_0*a_3, b_1*a_3) . . . vs[3] = (b_2*a_4, b_3*a_4)
7793
7794 void neon_partial_mult_64(const VSeq<4>& vs, FloatRegister bs, FloatRegister as, int lane_lo) {
7795 __ umullv(vs[0], __ T2D, bs, __ T2S, as, __ S, lane_lo);
7796 __ umull2v(vs[1], __ T2D, bs, __ T4S, as, __ S, lane_lo);
7797 __ umullv(vs[2], __ T2D, bs, __ T2S, as, __ S, lane_lo + 2);
7798 __ umull2v(vs[3], __ T2D, bs, __ T4S, as, __ S, lane_lo + 2);
7799 }
7800
7801 // Subroutine used by the generate_intpoly_montgomeryMult_P256() function
7802 // to compute the result of a 52 x 52 bit multiplications where the
7803 // multiplicands, a and b are available as 64-bit values.
7804 // The result is going to two 64-bit registers lo (least significant 52 bits)
7805 // and hi (most significant 52 bits).
7806 void gpr_partial_mult_52(Register a, Register b, Register hi, Register lo,
7807 Register mask) {
7808 // compute 104-bit (40 + 64) full product
7809 __ umulh(hi, a, b);
7810 __ mul(lo, a, b);
7811 // combine 40 + 12 bits into hi result
7812 // on certain implementations of aarch64 (e.g. apple M1) replacing extr()
7813 // with the following equivalent instruction sequence the performance
7814 // improves slightly (despite it is two instructions longer and needs
7815 // an additional register)
7816 // __ lsl(hi, hi, montMulP256Shift1);
7817 // __ lsr(tmp, lo, montMulP256Shift2);
7818 // __ orr(hi, hi, tmp);
7819 __ extr(hi, hi, lo, montMulP256Shift2);
7820 // mask off 52 bits of lo result
7821 __ andr(lo, lo, mask);
7822 }
7823
7824 // This assembly follows the Java code in MontgomeryIntegerPolynomial256.mult()
7825 // quite closely. The main difference is that the computations done with the
7826 // last two limbs of `a` are done using Neon registers. This allows us to take
7827 // advantage of both the Neon registers and GPRs simultaneously.
7828 // It is also worth noting that since Neon does not support 64 bit
7829 // multiplication, we split each 64 bit value into lower and upper halves
7830 // and use the "schoolbook" multiplication algorithm.
7831 address generate_intpoly_montgomeryMult_P256() {
7832 assert(UseIntPolyIntrinsics, "what are we doing here?");
7833 StubId stub_id = StubId::stubgen_intpoly_montgomeryMult_P256_id;
7834 int entry_count = StubInfo::entry_count(stub_id);
7835 assert(entry_count == 1, "sanity check");
7836 address start = load_archive_data(stub_id);
7837 if (start != nullptr) {
7838 return start;
7839 }
7840 __ align(CodeEntryAlignment);
7841 StubCodeMark mark(this, stub_id);
7842 start = __ pc();
7843 __ enter();
7844
7845 // Registers that are used throughout entire routine
7846 const Register a = c_rarg0;
7847 const Register b = c_rarg1;
7848 const Register result = c_rarg2;
7849
7850 RegSet regs = RegSet::range(r0, r28) - rscratch1 - rscratch2
7851 - r16 - r17 - r18_tls - a - b - result;
7852
7853 auto common_regs = regs.begin();
7854 Register limb_mask = *common_regs++,
7855 c_ptr = *common_regs++,
7856 mod_0 = *common_regs++,
7857 mod_1 = *common_regs++,
7858 mod_3 = *common_regs++,
7859 mod_4 = *common_regs++,
7860 b_0 = *common_regs++,
7861 b_1 = *common_regs++,
7862 b_2 = *common_regs++,
7863 b_3 = *common_regs++,
7864 b_4 = *common_regs++;
7865
7866 FloatRegSet floatRegs = FloatRegSet::range(v0, v31)
7867 - FloatRegSet::range(v8, v15) // Caller saved vectors
7868 - FloatRegSet::range(v16, v31); // Manually-allocated vectors
7869
7870 auto common_vectors = floatRegs.begin();
7871 FloatRegister limb_mask_vec = *common_vectors++,
7872 b_lows = *common_vectors++,
7873 b_highs = *common_vectors++,
7874 a_vals = *common_vectors++;
7875
7876 // Push callee saved registers on to the stack
7877 RegSet callee_saved = RegSet::range(r19, r28);
7878 __ push(callee_saved, sp);
7879
7880 // Allocate space on the stack for carry values
7881 __ sub(sp, sp, cDataSize);
7882 __ mov(c_ptr, sp);
7883
7884 // Calculate (52-bit) limb masks for both gpr and vector registers
7885 __ mov(limb_mask, -UCONST64(1) >> montMulP256Shift1);
7886 __ dup(limb_mask_vec, __ T2D, limb_mask);
7887
7888 //Load input arrays and modulus
7889 Register a_ptr = *common_regs++, mod_ptr = *common_regs++;
7890 // skip 3 limbs so a_ptr addresses trailing pair {a3, a4}
7891 __ add(a_ptr, a, 3 * BytesPerLong);
7892 __ lea(mod_ptr, ExternalAddress((address)_modulus_P256));
7893 __ ldr(b_0, Address(b));
7894 __ ldr(b_1, Address(b, BytesPerLong));
7895 __ ldr(b_2, Address(b, 2 * BytesPerLong));
7896 __ ldr(b_3, Address(b, 3 * BytesPerLong));
7897 __ ldr(b_4, Address(b, 4 * BytesPerLong));
7898 __ ldr(mod_0, __ post(mod_ptr, BytesPerLong));
7899 __ ldr(mod_1, __ post(mod_ptr, BytesPerLong));
7900 __ ldr(mod_3, __ post(mod_ptr, BytesPerLong));
7901 __ ldr(mod_4, mod_ptr);
7902 __ ld1(a_vals, __ T2D, a_ptr);
7903 // use an interleaved load to group low 32 bits and high 20 bits
7904 // of 4 successive b values into two vector registers
7905 // n.b. these are the same inputs as the ones in b_0 ... b4
7906 __ ld2(b_lows, b_highs, __ T4S, b);
7907 common_regs = common_regs.remaining()
7908 + a_ptr + mod_ptr;
7909 a_ptr = mod_ptr = noreg;
7910
7911 //Regs used throughout the main "loop", which is partially unrolled here
7912 Register high = *common_regs++,
7913 low = *common_regs++,
7914 mul_ptr = *common_regs++,
7915 mod_high = *common_regs++,
7916 mod_low = *common_regs++,
7917 a_i = *common_regs++,
7918 c_i = *common_regs++,
7919 tmp = *common_regs++,
7920 n = *common_regs++;
7921
7922 // vector sequences used to compute and combine partial products of
7923 // b_i * a_j for i = {0,1,2,3} j = {3,4}
7924 VSeq<4> A(16);
7925 VSeq<4> B(20);
7926 VSeq<4> C(24);
7927 VSeq<4> D(28);
7928
7929
7930 // neon and gpr computations are interleaved to maximize parallelism
7931
7932 // allocate stack space for the neon results
7933 __ sub(sp, sp, mulDataSize);
7934 __ mov(mul_ptr, sp);
7935
7936 // cross-multiply low * low for limbs b0-b3 and a3-a4 in parallel
7937 neon_partial_mult_64(A, b_lows, a_vals, 0);
7938
7939 // Limb 0
7940 __ ldr(a_i, __ post(a, BytesPerLong));
7941 gpr_partial_mult_52(a_i, b_0, high, low, limb_mask);
7942 __ mov(n, low);
7943 // __ andr(n, low, limb_mask);
7944
7945 // cross-multiply high * low for limbs b0-b3 and a3-a4 in parallel
7946 neon_partial_mult_64(B, b_highs, a_vals, 0);
7947
7948 // Limb 0 modulus computation
7949 // n.b. modulus computation requires multiplying successive
7950 // limbs of the product by corresponding limbs of the p256
7951 // prime adding the result to the limb and folding this
7952 // partial result into a running 256-bit sum in c_i. Limbs
7953 // of c_i are stored via c_ptr once carries are included.
7954 // n.b. the mul + add is omitted for limb 2 since the
7955 // corresponding prime bits are zero.
7956 gpr_partial_mult_52(n, mod_0, mod_high, mod_low, limb_mask);
7957 __ add(low, low, mod_low);
7958 __ add(high, high, mod_high);
7959 __ lsr(c_i, low, montMulP256Shift2);
7960 __ add(c_i, c_i, high);
7961
7962 // cross-multiply low * high for limbs b0-b3 and a3-a4 in parallel
7963 neon_partial_mult_64(C, b_lows, a_vals, 1);
7964
7965 // Limb 1
7966 gpr_partial_mult_52(a_i, b_1, high, low, limb_mask);
7967
7968 // cross-multiply high * high for limbs b0-b3 and a3-a4 in parallel
7969 neon_partial_mult_64(D, b_highs, a_vals, 1);
7970
7971 gpr_partial_mult_52(n, mod_1, mod_high, mod_low, limb_mask);
7972 __ add(low, low, mod_low);
7973 __ add(high, high, mod_high);
7974 __ add(c_i, c_i, low);
7975 __ str(c_i, c_ptr);
7976 __ mov(c_i, high);
7977
7978 // combine neon 32-bit partial products, regrouping to produce
7979 // 8*52-bit low products in A and 8*52-bit high products in D
7980
7981 // add low*high/high*low intermediate products before regrouping
7982 vs_addv(B, __ T2D, B, C); // Store (B+C) in B
7983
7984 // Limb 2
7985 gpr_partial_mult_52(a_i, b_2, high, low, limb_mask);
7986 __ add(c_i, c_i, low);
7987 __ str(c_i, Address(c_ptr, 8));
7988 __ mov(c_i, high);
7989
7990 // shift high*high (40-bit) product up into 52-bits of output
7991 vs_shl(D, __ T2D, D, montMulP256Shift1);
7992
7993 // Limb 3
7994 gpr_partial_mult_52(a_i, b_3, high, low, limb_mask);
7995
7996 // shift high 32 (or 33) bits of intermediate products for addition to D
7997 vs_ushr(C, __ T2D, B, 32 - montMulP256Shift1); // Use C for ((B+C) >>> 20)
7998
7999 gpr_partial_mult_52(n, mod_3, mod_high, mod_low, limb_mask);
8000 __ add(low, low, mod_low);
8001 __ add(high, high, mod_high);
8002 __ add(c_i, c_i, low);
8003 __ str(c_i, Address(c_ptr, 2 * BytesPerLong));
8004 __ mov(c_i, high);
8005
8006 // shift low 32 bits of intermediate product up for masking and addition to A
8007 vs_shl(B, __ T2D, B, 32);
8008
8009 // Limb 4
8010 gpr_partial_mult_52(a_i, b_4, high, low, limb_mask);
8011
8012 // add high bits of intermediate product into D
8013 vs_addv(D, __ T2D, D, C);
8014
8015 gpr_partial_mult_52(n, mod_4, mod_high, mod_low, limb_mask);
8016 __ add(low, low, mod_low);
8017 __ add(high, high, mod_high);
8018 __ add(c_i, c_i, low);
8019 __ str(c_i, Address(c_ptr, 3 * BytesPerLong));
8020 __ str(high, Address(c_ptr, 4 * BytesPerLong));
8021
8022 // top 12 bits of 32*32 bit product in A need adding into high 52-bit output
8023 vs_ushr(C, __ T2D, A, 52); // C now holds (A >>> 52)
8024 // Only 20 of the 32 bits now in the top of B should be added into A
8025 vs_andr(B, B, limb_mask_vec);
8026 // reduce original 64-bit product to 52-bits
8027 vs_andr(A, A, limb_mask_vec);
8028 // add intermediate products to high 52-bit result in D
8029 vs_addv(D, __ T2D, D, C);
8030 // add 20/21 bits of intermediate product in top of B into low 52-bit result
8031 vs_addv(A, __ T2D, A, B);
8032 // save and then mask off any overflow bit from computing low 52-bit result
8033 vs_ushr(B, __ T2D, A, montMulP256Shift2);
8034 vs_andr(A, A, limb_mask_vec);
8035 // add any remaining carry into the high 52-bit result
8036 vs_addv(D, __ T2D, D, B);
8037
8038 // the write interleaves the 4 successive pairs of low and
8039 // high results: (l0, l1), (h0, h1), ... (l6, l7), (h6, h7)
8040 vs_st1_interleaved(A, D, mul_ptr);
8041
8042 // Free mul_ptr
8043 common_regs = common_regs.remaining() + mul_ptr;
8044 mul_ptr = noreg;
8045
8046 /////////////////////////
8047 // Loop 2 & 3
8048 /////////////////////////
8049
8050 for (int i = 0; i < 2; i++) {
8051 // Load a_i and increment by 8 bytes
8052 __ ldr(a_i, __ post(a, BytesPerLong));
8053 __ ldr(c_i, c_ptr); //Load prior c_i
8054
8055 // Limb 0
8056 gpr_partial_mult_52(a_i, b_0, high, low, limb_mask);
8057 __ add(low, low, c_i);
8058 __ ldr(c_i, Address(c_ptr, BytesPerLong));
8059 __ andr(n, low, limb_mask);
8060 gpr_partial_mult_52(n, mod_0, mod_high, mod_low, limb_mask);
8061 __ add(low, low, mod_low);
8062 __ add(high, high, mod_high);
8063 __ lsr(tmp, low, montMulP256Shift2);
8064 __ add(c_i, c_i, tmp);
8065 __ add(c_i, c_i, high);
8066
8067 // Limb 1
8068 gpr_partial_mult_52(a_i, b_1, high, low, limb_mask);
8069 gpr_partial_mult_52(n, mod_1, mod_high, mod_low, limb_mask);
8070 __ ldr(tmp, Address(c_ptr, 2 * BytesPerLong));
8071 __ add(low, low, mod_low);
8072 __ add(high, high, mod_high);
8073 __ add(c_i, c_i, low);
8074 __ str(c_i, c_ptr);
8075 __ add(c_i, tmp, high);
8076
8077 // Limb 2
8078 gpr_partial_mult_52(a_i, b_2, high, low, limb_mask);
8079 __ ldr(tmp, Address(c_ptr, 3 * BytesPerLong));
8080 __ add(c_i, c_i, low);
8081 __ str(c_i, Address(c_ptr, BytesPerLong));
8082 __ add(c_i, tmp, high);
8083
8084 // Limb 3
8085 gpr_partial_mult_52(a_i, b_3, high, low, limb_mask);
8086 gpr_partial_mult_52(n, mod_3, mod_high, mod_low, limb_mask);
8087 __ ldr(tmp, Address(c_ptr, 4 * BytesPerLong));
8088 __ add(low, low, mod_low);
8089 __ add(high, high, mod_high);
8090 __ add(c_i, c_i, low);
8091 __ str(c_i, Address(c_ptr, 2 * BytesPerLong));
8092 __ add(c_i, tmp, high);
8093
8094 // Limb 4
8095 gpr_partial_mult_52(a_i, b_4, high, low, limb_mask);
8096 gpr_partial_mult_52(n, mod_4, mod_high, mod_low, limb_mask);
8097 __ add(low, low, mod_low);
8098 __ add(high, high, mod_high);
8099 __ add(c_i, c_i, low);
8100 __ str(c_i, Address(c_ptr, 3 * BytesPerLong));
8101 __ str(high, Address(c_ptr, 4 * BytesPerLong));
8102 }
8103 // Reallocate regs b_0, b_1, b_2 and b_3
8104 common_regs = common_regs.remaining()
8105 + b_0 + b_1 + b_2 + b_3;
8106 b_0 = b_1 = b_2 = b_3 = noreg;
8107
8108 Register low_1 = *common_regs++;
8109 Register high_1 = *common_regs++;
8110
8111 //////////////////////////////
8112 // a[3]
8113 //////////////////////////////
8114
8115 // For a_3 and a_4 we have already computed the cross-products
8116 // with b_0 ... b_3 and stored them on the stack relative to
8117 // `mul_ptr` i.e. the current `sp`in the order
8118 // l(a_3 * b_0), l(a_3 * b_1), h(a_3 * b_0), h(a_3 * b_1),
8119 // l(a_3 * b_2), l(a_3 * b_3), h(a_3 * b_2), h(a_3 * b_3),
8120 // l(a_4 * b_0), l(a_4 * b_1), h(a_4 * b_0), h(a_4 * b_1),
8121 // l(a_4 * b_2), l(a_4 * b_3), h(a_4 * b_2), h(a_4 * b_3),
8122 // where l(x) is the low 52 bits of x and h(x) is the high 52 bits
8123
8124 __ ldr(low_1, Address(sp));
8125 __ ldr(high_1, Address(sp, 2 * BytesPerLong));
8126
8127 __ ldr(low, Address(sp, BytesPerLong));
8128 __ ldr(high, Address(sp, 3 * BytesPerLong));
8129 __ ldr(a_i, __ post(a, BytesPerLong));
8130 __ ldr(c_i, c_ptr);
8131
8132 // Limb 0
8133 __ add(low_1, low_1, c_i);
8134 __ ldr(c_i, Address(c_ptr, BytesPerLong));
8135 __ andr(n, low_1, limb_mask);
8136 gpr_partial_mult_52(n, mod_0, mod_high, mod_low, limb_mask);
8137 __ add(low_1, low_1, mod_low);
8138 __ add(high_1, high_1, mod_high);
8139 __ lsr(tmp, low_1, montMulP256Shift2);
8140 __ add(c_i, c_i, tmp);
8141 __ add(c_i, c_i, high_1);
8142
8143 // Limb 1
8144 __ ldr(low_1, Address(sp, 4 * BytesPerLong));
8145 __ ldr(high_1, Address(sp, 6 * BytesPerLong));
8146 gpr_partial_mult_52(n, mod_1, mod_high, mod_low, limb_mask);
8147 __ ldr(tmp, Address(c_ptr, 2 * BytesPerLong));
8148 __ andr(mod_low, mod_low, limb_mask);
8149 __ add(low, low, mod_low);
8150 __ add(high, high, mod_high);
8151 __ add(c_i, c_i, low);
8152 __ str(c_i, c_ptr);
8153 __ add(c_i, tmp, high);
8154
8155 // Limb 2
8156 __ ldr(low, Address(sp, 5 * BytesPerLong));
8157 __ ldr(high, Address(sp, 7 * BytesPerLong));
8158 __ ldr(tmp, Address(c_ptr, 3 * BytesPerLong));
8159 __ add(c_i, c_i, low_1);
8160 __ str(c_i, Address(c_ptr, BytesPerLong));
8161 __ add(c_i, tmp, high_1);
8162
8163 // Limb 3
8164 gpr_partial_mult_52(n, mod_3, mod_high, mod_low, limb_mask);
8165 __ ldr(tmp, Address(c_ptr, 4 * BytesPerLong));
8166 __ add(low, low, mod_low);
8167 __ add(high, high, mod_high);
8168 __ add(c_i, c_i, low);
8169 __ str(c_i, Address(c_ptr, 2 * BytesPerLong));
8170 __ add(c_i, tmp, high);
8171
8172 // Limb 4
8173 __ ldr(low, Address(sp, 8 * BytesPerLong));
8174 __ ldr(high, Address(sp, 10 * BytesPerLong));
8175 gpr_partial_mult_52(a_i, b_4, high_1, low_1, limb_mask);
8176 gpr_partial_mult_52(n, mod_4, mod_high, mod_low, limb_mask);
8177 __ add(low_1, low_1, mod_low);
8178 __ add(high_1, high_1, mod_high);
8179 __ add(c_i, c_i, low_1);
8180 __ str(c_i, Address(c_ptr, 3 * BytesPerLong));
8181 __ str(high_1, Address(c_ptr, 4 * BytesPerLong));
8182
8183 //////////////////////////////
8184 // a[4]
8185 //////////////////////////////
8186
8187 Register c5 = *common_regs++,
8188 c6 = *common_regs++,
8189 c7 = *common_regs++;
8190
8191 __ ldr(a_i, a);
8192 __ ldr(c_i, c_ptr);
8193
8194 // Limb 0
8195 __ ldr(low_1, Address(sp, 9 * BytesPerLong));
8196 __ ldr(high_1, Address(sp, 11 * BytesPerLong));
8197
8198 __ add(low, low, c_i);
8199 __ ldr(c_i, Address(c_ptr, BytesPerLong));
8200 __ andr(n, low, limb_mask);
8201 gpr_partial_mult_52(n, mod_0, mod_high, mod_low, limb_mask);
8202 __ add(low, low, mod_low);
8203 __ add(high, high, mod_high);
8204 __ lsr(tmp, low, montMulP256Shift2);
8205 __ add(c_i, c_i, tmp);
8206 __ add(c_i, c_i, high);
8207
8208 __ ldr(low, Address(sp, 12 * BytesPerLong));
8209 __ ldr(high, Address(sp, 14 * BytesPerLong));
8210 gpr_partial_mult_52(n, mod_1, mod_high, mod_low, limb_mask);
8211 __ add(low_1, low_1, mod_low);
8212 __ add(high_1, high_1, mod_high);
8213 __ add(c5, c_i, low_1);
8214 __ ldr(c_i, Address(c_ptr, 2 * BytesPerLong));
8215 __ lsr(tmp, c5, montMulP256Shift2);
8216 __ add(c_i, c_i, tmp);
8217 __ add(c_i, c_i, high_1);
8218
8219 // Limb 2
8220 __ ldr(low_1, Address(sp, 13 * BytesPerLong));
8221 __ ldr(high_1, Address(sp, 15 * BytesPerLong));
8222 __ add(c6, c_i, low);
8223 __ ldr(c_i, Address(c_ptr, 3 * BytesPerLong));
8224 __ lsr(tmp, c6, montMulP256Shift2);
8225 __ add(c_i, c_i, tmp);
8226 __ add(c_i, c_i, high);
8227
8228 // Limb 3
8229 gpr_partial_mult_52(n, mod_3, mod_high, mod_low, limb_mask);
8230 __ add(low_1, low_1, mod_low);
8231 __ add(high_1, high_1, mod_high);
8232 __ add(c7, c_i, low_1);
8233 __ ldr(c_i, Address(c_ptr, 4 * BytesPerLong));
8234 __ lsr(tmp, c7, montMulP256Shift2);
8235 __ add(c_i, c_i, tmp);
8236 __ add(c_i, c_i, high_1);
8237
8238 // Limb 4
8239 gpr_partial_mult_52(a_i, b_4, high, low, limb_mask);
8240 gpr_partial_mult_52(n, mod_4, mod_high, mod_low, limb_mask);
8241 __ add(low, low, mod_low);
8242 __ add(high, high, mod_high);
8243
8244 // Reallocate b_4
8245 common_regs = common_regs.remaining() + b_4;
8246 b_4 = noreg;
8247
8248 Register c8 = *common_regs++,
8249 c9 = *common_regs++;
8250
8251 __ add(c8, c_i, low);
8252 __ lsr(c9, c8, montMulP256Shift2);
8253 __ add(c9, c9, high);
8254
8255 __ andr(c5, c5, limb_mask);
8256 __ andr(c6, c6, limb_mask);
8257 __ andr(c7, c7, limb_mask);
8258 __ andr(c8, c8, limb_mask);
8259
8260 /////////////////////////////
8261 // Final carry propagate
8262 /////////////////////////////
8263
8264 // c0 = c5 - modulus[0];
8265 // c1 = c6 - modulus[1] + (c0 >> BITS_PER_LIMB);
8266 // c0 &= LIMB_MASK;
8267 // c2 = c7 + (c1 >> BITS_PER_LIMB);
8268 // c1 &= LIMB_MASK;
8269 // c3 = c8 - modulus[3] + (c2 >> BITS_PER_LIMB);
8270 // c2 &= LIMB_MASK;
8271 // c4 = c9 - modulus4] + (c3 >> BITS_PER_LIMB);
8272 // c3 &= LIMB_MASK;
8273
8274 // Free up all unused regs
8275 common_regs = common_regs.remaining()
8276 + c_ptr + low + high + mod_high
8277 + mod_low + a_i + c_i + n + low_1 + high_1;
8278 c_ptr = low = high = mod_high
8279 = mod_low = a_i = c_i = n = low_1 = high_1 = noreg;
8280
8281 Register c0 = *common_regs++,
8282 c1 = *common_regs++,
8283 c2 = *common_regs++,
8284 c3 = *common_regs++,
8285 c4 = *common_regs++;
8286
8287 __ sub(c0, c5, mod_0);
8288 __ sub(c1, c6, mod_1);
8289 __ sub(c3, c8, mod_3);
8290 __ sub(c4, c9, mod_4);
8291 __ add(c1, c1, c0, Assembler::ASR, montMulP256Shift2);
8292 __ andr(c0, c0, limb_mask);
8293 __ add(c2, c7, c1, Assembler::ASR, montMulP256Shift2);
8294 __ andr(c1, c1, limb_mask);
8295 __ add(c3, c3, c2, Assembler::ASR, montMulP256Shift2);
8296 __ andr(c2, c2, limb_mask);
8297 __ add(c4, c4, c3, Assembler::ASR, montMulP256Shift2);
8298 __ andr(c3, c3, limb_mask);
8299
8300 // Final write back
8301 // mask = c4 >> 63
8302 // r[0] = ((c5 & mask) | (c0 & ~mask));
8303 // r[1] = ((c6 & mask) | (c1 & ~mask));
8304 // r[2] = ((c7 & mask) | (c2 & ~mask));
8305 // r[3] = ((c8 & mask) | (c3 & ~mask));
8306 // r[4] = ((c9 & mask) | (c4 & ~mask));
8307
8308 common_regs = common_regs.remaining()
8309 + mod_0 + mod_1 + mod_3 + mod_4;
8310 mod_0 = mod_1 = mod_3 = mod_4 = noreg;
8311
8312 Register mask = *common_regs++;
8313 Register nmask = *common_regs++;
8314
8315 __ asr(mask, c4, 63);
8316 __ mvn(nmask, mask);
8317 __ andr(c5, c5, mask);
8318 __ andr(tmp, c0, nmask);
8319 __ orr(c5, c5, tmp);
8320 __ andr(c6, c6, mask);
8321 __ andr(tmp, c1, nmask);
8322 __ orr(c6, c6, tmp);
8323 __ andr(c7, c7, mask);
8324 __ andr(tmp, c2, nmask);
8325 __ orr(c7, c7, tmp);
8326 __ andr(c8, c8, mask);
8327 __ andr(tmp, c3, nmask);
8328 __ orr(c8, c8, tmp);
8329 __ andr(c9, c9, mask);
8330 __ andr(tmp, c4, nmask);
8331 __ orr(c9, c9, tmp);
8332
8333 __ str(c5, result);
8334 __ str(c6, Address(result, BytesPerLong));
8335 __ str(c7, Address(result, 2 * BytesPerLong));
8336 __ str(c8, Address(result, 3 * BytesPerLong));
8337 __ str(c9, Address(result, 4 * BytesPerLong));
8338
8339 // End intrinsic call
8340 __ add(sp, sp, cDataSize + mulDataSize);
8341 __ pop(callee_saved, sp);
8342 __ leave();
8343 __ mov(r0, zr); // return 0
8344 __ ret(lr);
8345
8346 // record the stub entry and end
8347 store_archive_data(stub_id, start, __ pc());
8348
8349 return start;
8350 }
8351
8352 address generate_intpoly_assign() {
8353 // KNOWN Lengths:
8354 // MontgomeryIntPolynP256: 5 = 4 + 1
8355 // IntegerPolynomial1305: 5 = 4 + 1
8356 // IntegerPolynomial25519: 10 = 8 + 2
8357 // IntegerPolynomialP256: 10 = 8 + 2
8358 // Curve25519OrderField: 10 = 8 + 2
8359 // Curve25519OrderField: 10 = 8 + 2
8360 // P256OrderField: 10 = 8 + 2
8361 // IntegerPolynomialP384: 14 = 8 + 4 + 2
8362 // P384OrderField: 14 = 8 + 4 + 2
8363 // IntegerPolynomial448: 16 = 8 + 8
8364 // Curve448OrderField: 16 = 8 + 8
8365 // Curve448OrderField: 16 = 8 + 8
8366 // IntegerPolynomialP521: 19 = 8 + 8 + 2 + 1
8367 // P521OrderField: 19 = 8 + 8 + 2 + 1
8368 // Special Cases 5, 10, 14, 16, 19
8369 assert(UseIntPolyIntrinsics, "what are we doing here?");
8370 StubId stub_id = StubId::stubgen_intpoly_assign_id;
8371 int entry_count = StubInfo::entry_count(stub_id);
8372 assert(entry_count == 1, "sanity check");
8373 address start = load_archive_data(stub_id);
8374 if (start != nullptr) {
8375 return start;
8376 }
8377
8378 __ align(CodeEntryAlignment);
8379 StubCodeMark mark(this, stub_id);
8380 start = __ pc();
8381 __ enter();
8382
8383 // Inputs
8384 const Register set = c_rarg0;
8385 const Register aLimbs = c_rarg1;
8386 const Register bLimbs = c_rarg2;
8387 const Register length = c_rarg3;
8388
8389 Label L_Length5, L_Length10, L_Length14, L_Length16, L_Length19, L_Default, L_Done;
8390
8391 /*
8392 int maskValue = -set;
8393 for (int i = 0; i < a.length; i++) {
8394 long dummyLimbs = maskValue & (a[i] ^ b[i]);
8395 a[i] = dummyLimbs ^ a[i];
8396 }
8397 */
8398 Register mask_scalar = r4;
8399 FloatRegister mask_vec = v0;
8400
8401 __ neg(mask_scalar, set);
8402 __ dup(mask_vec, __ T2D, mask_scalar);
8403
8404 __ cmp(length, (u1)5);
8405 __ br(Assembler::EQ, L_Length5);
8406 __ cmp(length, (u1)10);
8407 __ br(Assembler::EQ, L_Length10);
8408 __ cmp(length, (u1)14);
8409 __ br(Assembler::EQ, L_Length14);
8410 __ cmp(length, (u1)16);
8411 __ br(Assembler::EQ, L_Length16);
8412 __ cmp(length, (u1)19);
8413 __ br(Assembler::EQ, L_Length19);
8414 __ b(L_Default);
8415
8416
8417 // Length = 5
8418 // Use 5 GPRs (neon not faster with this few limbs)
8419 __ BIND(L_Length5);
8420 {
8421 Register a0 = r5;
8422 Register a1 = r6;
8423 Register a2 = r7;
8424 Register a3 = r10;
8425 Register a4 = r11;
8426 Register b0 = r12;
8427 Register b1 = r13;
8428 Register b2 = r14;
8429 Register b3 = r15;
8430 Register b4 = r19;
8431
8432 __ push(r19, sp);
8433
8434 __ ldr(a0, aLimbs);
8435 __ ldr(a1, Address(aLimbs, 1 * BytesPerLong));
8436 __ ldr(a2, Address(aLimbs, 2 * BytesPerLong));
8437 __ ldr(a3, Address(aLimbs, 3 * BytesPerLong));
8438 __ ldr(a4, Address(aLimbs, 4 * BytesPerLong));
8439
8440 __ ldr(b0, bLimbs);
8441 __ ldr(b1, Address(bLimbs, 1 * BytesPerLong));
8442 __ ldr(b2, Address(bLimbs, 2 * BytesPerLong));
8443 __ ldr(b3, Address(bLimbs, 3 * BytesPerLong));
8444 __ ldr(b4, Address(bLimbs, 4 * BytesPerLong));
8445
8446 __ eor(b0, b0, a0);
8447 __ eor(b1, b1, a1);
8448 __ eor(b2, b2, a2);
8449 __ eor(b3, b3, a3);
8450 __ eor(b4, b4, a4);
8451
8452 __ andr(b0, b0, mask_scalar);
8453 __ andr(b1, b1, mask_scalar);
8454 __ andr(b2, b2, mask_scalar);
8455 __ andr(b3, b3, mask_scalar);
8456 __ andr(b4, b4, mask_scalar);
8457
8458 __ eor(a0, a0, b0);
8459 __ eor(a1, a1, b1);
8460 __ eor(a2, a2, b2);
8461 __ eor(a3, a3, b3);
8462 __ eor(a4, a4, b4);
8463
8464 __ str(a0, aLimbs);
8465 __ str(a1, Address(aLimbs, 1 * BytesPerLong));
8466 __ str(a2, Address(aLimbs, 2 * BytesPerLong));
8467 __ str(a3, Address(aLimbs, 3 * BytesPerLong));
8468 __ str(a4, Address(aLimbs, 4 * BytesPerLong));
8469
8470 __ pop(r19, sp);
8471 __ b(L_Done);
8472 }
8473
8474 // Length = 10
8475 // Split into 4 neon regs and 2 GPRs
8476 __ BIND(L_Length10);
8477 {
8478 Register a9 = r10;
8479 Register a10 = r11;
8480 Register b9 = r12;
8481 Register b10 = r13;
8482
8483 VSeq<4> a_vec(16);
8484 VSeq<4> b_vec(20);
8485
8486 __ ldr(a9, Address(aLimbs, 8 * BytesPerLong));
8487 __ ldr(a10, Address(aLimbs, 9 * BytesPerLong));
8488 __ ldr(b9, Address(bLimbs, 8 * BytesPerLong));
8489 __ ldr(b10, Address(bLimbs, 9 * BytesPerLong));
8490
8491 vs_ldpq(a_vec, aLimbs);
8492
8493 __ eor(b9, b9, a9);
8494 __ eor(b10, b10, a10);
8495
8496 vs_ldpq(b_vec, bLimbs);
8497
8498 __ andr(b9, b9, mask_scalar);
8499 __ andr(b10, b10, mask_scalar);
8500
8501 vs_eor(b_vec, b_vec, a_vec);
8502
8503 __ eor(a9, a9, b9);
8504 __ eor(a10, a10, b10);
8505
8506 vs_andr(b_vec, b_vec, mask_vec);
8507
8508 __ str(a9, Address(aLimbs, 8 * BytesPerLong));
8509 __ str(a10, Address(aLimbs, 9 * BytesPerLong));
8510
8511 vs_eor(a_vec, a_vec, b_vec);
8512 vs_stpq_post(a_vec, aLimbs);
8513
8514 __ b(L_Done);
8515 }
8516
8517 // Length = 14
8518 // Split into 5 neon regs and 4 GPRs
8519 __ BIND(L_Length14);
8520 {
8521 Register a10 = r5;
8522 Register a11 = r6;
8523 Register a12 = r7;
8524 Register a13 = r8;
8525 Register b10 = r9;
8526 Register b11 = r10;
8527 Register b12 = r11;
8528 Register b13 = r12;
8529
8530 VSeq<5> a_vec(16);
8531 VSeq<5> b_vec(22);
8532
8533 int offsets[2] = { 0, 32 };
8534
8535 __ ldr(a10, Address(aLimbs, 10 * BytesPerLong));
8536 __ ldr(a11, Address(aLimbs, 11 * BytesPerLong));
8537 __ ldr(a12, Address(aLimbs, 12 * BytesPerLong));
8538 __ ldr(a13, Address(aLimbs, 13 * BytesPerLong));
8539
8540 __ ldr(b10, Address(bLimbs, 10 * BytesPerLong));
8541 __ ldr(b11, Address(bLimbs, 11 * BytesPerLong));
8542 __ ldr(b12, Address(bLimbs, 12 * BytesPerLong));
8543 __ ldr(b13, Address(bLimbs, 13 * BytesPerLong));
8544
8545 __ ld1(a_vec[0], __ T2D, aLimbs);
8546 vs_ldpq_indexed(vs_tail(a_vec), aLimbs, 16, offsets);
8547
8548 __ eor(b10, b10, a10);
8549 __ eor(b11, b11, a11);
8550 __ eor(b12, b12, a12);
8551 __ eor(b13, b13, a13);
8552
8553 __ ld1(b_vec[0], __ T2D, bLimbs);
8554 vs_ldpq_indexed(vs_tail(b_vec), bLimbs, 16, offsets);
8555
8556 __ andr(b10, b10, mask_scalar);
8557 __ andr(b11, b11, mask_scalar);
8558 __ andr(b12, b12, mask_scalar);
8559 __ andr(b13, b13, mask_scalar);
8560
8561 vs_eor(b_vec, b_vec, a_vec);
8562
8563 __ eor(a10, a10, b10);
8564 __ eor(a11, a11, b11);
8565 __ eor(a12, a12, b12);
8566 __ eor(a13, a13, b13);
8567
8568 vs_andr(b_vec, b_vec, mask_vec);
8569
8570 __ str(a10, Address(aLimbs, 10 * BytesPerLong));
8571 __ str(a11, Address(aLimbs, 11 * BytesPerLong));
8572 __ str(a12, Address(aLimbs, 12 * BytesPerLong));
8573 __ str(a13, Address(aLimbs, 13 * BytesPerLong));
8574
8575 vs_eor(a_vec, a_vec, b_vec);
8576
8577 __ st1(a_vec[0], __ T2D, aLimbs);
8578 vs_stpq_indexed(vs_tail(a_vec), aLimbs, 16, offsets);
8579
8580 __ b(L_Done);
8581 }
8582
8583 // Length = 16
8584 // Use 8 neon regs
8585 __ BIND(L_Length16);
8586 {
8587 VSeq<8> a_vec(16);
8588 VSeq<8> b_vec(24);
8589
8590 vs_ldpq(a_vec, aLimbs);
8591 vs_ldpq(b_vec, bLimbs);
8592 vs_eor(b_vec, b_vec, a_vec);
8593 vs_andr(b_vec, b_vec, mask_vec);
8594 vs_eor(a_vec, a_vec, b_vec);
8595 vs_stpq_post(a_vec, aLimbs);
8596
8597 __ b(L_Done);
8598 }
8599
8600 // Length = 19
8601 // Split into 8 neon regs and 3 GPRs
8602 __ BIND(L_Length19);
8603 {
8604 Register a17 = r10;
8605 Register a18 = r11;
8606 Register a19 = r12;
8607 Register b17 = r13;
8608 Register b18 = r14;
8609 Register b19 = r15;
8610
8611 VSeq<8> a_vec(16);
8612 VSeq<8> b_vec(24);
8613
8614 __ ldr(a17, Address(aLimbs, 16 * BytesPerLong));
8615 __ ldr(a18, Address(aLimbs, 17 * BytesPerLong));
8616 __ ldr(a19, Address(aLimbs, 18 * BytesPerLong));
8617 __ ldr(b17, Address(bLimbs, 16 * BytesPerLong));
8618 __ ldr(b18, Address(bLimbs, 17 * BytesPerLong));
8619 __ ldr(b19, Address(bLimbs, 18 * BytesPerLong));
8620
8621 vs_ldpq(a_vec, aLimbs);
8622
8623 __ eor(b17, b17, a17);
8624 __ eor(b18, b18, a18);
8625 __ eor(b19, b19, a19);
8626
8627 vs_ldpq(b_vec, bLimbs);
8628
8629 __ andr(b17, b17, mask_scalar);
8630 __ andr(b18, b18, mask_scalar);
8631 __ andr(b19, b19, mask_scalar);
8632
8633 vs_eor(b_vec, b_vec, a_vec);
8634
8635 __ eor(a17, a17, b17);
8636 __ eor(a18, a18, b18);
8637 __ eor(a19, a19, b19);
8638
8639 vs_andr(b_vec, b_vec, mask_vec);
8640
8641 __ str(a17, Address(aLimbs, 16 * BytesPerLong));
8642 __ str(a18, Address(aLimbs, 17 * BytesPerLong));
8643 __ str(a19, Address(aLimbs, 18 * BytesPerLong));
8644
8645 vs_eor(a_vec, a_vec, b_vec);
8646 vs_stpq_post(a_vec, aLimbs);
8647
8648 __ b(L_Done);
8649 }
8650
8651 __ BIND(L_Default);
8652 {
8653 Register ctr = r5;
8654 Register a_val = r6;
8655 Register b_val = r7;
8656
8657 __ mov(ctr, length); // length (the number of limbs) is never 0
8658
8659 Label default_loop;
8660 __ BIND(default_loop);
8661
8662 __ ldr(a_val, aLimbs);
8663 __ ldr(b_val, __ post(bLimbs, 8));
8664 __ eor(b_val, b_val, a_val);
8665 __ andr(b_val, b_val, mask_scalar);
8666 __ eor(a_val, a_val, b_val);
8667 __ str(a_val, __ post(aLimbs, 8));
8668 __ sub(ctr, ctr, 1);
8669 __ cmp(ctr, (u1)0);
8670 __ br(Assembler::NE, default_loop);
8671 }
8672
8673 __ BIND(L_Done);
8674 __ leave(); // required for proper stackwalking of RuntimeStub frame
8675 __ mov(r0, zr); // return 0
8676 __ ret(lr);
8677
8678 // record the stub entry and end
8679 store_archive_data(stub_id, start, __ pc());
8680
8681 return start;
8682 }
8683
8684 /**
8685 * Arithmetic polynomial multiplication in Curve25519. The algorithm mimics
8686 * the version in the IntegerPolynomial25519 class, including the use of all
8687 * columns (no folding method).
8688 *
8689 * Arguments:
8690 *
8691 * Inputs:
8692 * c_rarg0 - long[] aLimbs
8693 * c_rarg1 - long[] bLimbs
8694 *
8695 * Output:
8696 * c_rarg2 - long[] rLimbs result
8697 */
8698 address generate_intpoly_mult_25519() {
8699 StubId stub_id = StubId::stubgen_intpoly_mult_25519_id;
8700 int entry_count = StubInfo::entry_count(stub_id);
8701 assert(entry_count == 1, "sanity check");
8702 address start = load_archive_data(stub_id);
8703 if (start != nullptr) {
8704 return start;
8705 }
8706 __ align(CodeEntryAlignment);
8707 StubCodeMark mark(this, stub_id);
8708 start = __ pc();
8709 __ enter();
8710
8711 // Register Map
8712 const Register aLimbs = c_rarg0; // r0
8713 const Register bLimbs = c_rarg1; // r1
8714 const Register rLimbs = c_rarg2; // r2
8715
8716 Register c[] = {r3, r4, r5, r6, r7, r8, r9, r10, r11, r12};
8717 Register a = r13;
8718 Register b = r14;
8719 Register term = r15;
8720 Register low = r16;
8721 Register high = r17;
8722
8723 const int32_t limbs = 5;
8724 const int32_t bpl = 51;
8725 const int32_t rem = 64 - bpl;
8726 const int32_t TERM = 19;
8727 const int32_t columns = limbs * 2;
8728 const uint64_t mask = (uint64_t) -1 >> rem;
8729 const uint64_t CARRY_ADD = (uint64_t) 1 << (bpl - 1);
8730
8731 __ mov(term, TERM);
8732 for (int i = 0; i < columns; i++) {
8733 __ mov(c[i], zr);
8734 }
8735
8736 // Perform high/low multiplication with signed 5x51 bit limbs
8737 for (int i = 0; i < limbs; i++) {
8738 __ ldr(b, Address(bLimbs, i * 8));
8739 for (int j = 0; j < limbs; j++) {
8740 __ ldr(a, Address(aLimbs, j * 8));
8741 __ smulh(high, a, b);
8742 __ mul(low, a, b);
8743 __ extr(high, high, low, bpl);
8744 __ andr(low, low, mask);
8745 __ add(c[i + j], c[i + j], low);
8746 __ add(c[i + j + 1], c[i + j + 1], high);
8747 }
8748 }
8749
8750 for (int i = 0; i < limbs; i++) {
8751 __ mul(c[i + 5], c[i + 5], term);
8752 __ add(c[i], c[i], c[i + 5]);
8753 }
8754
8755 // Carry-add with reduction from high limb
8756 Register tmp = low;
8757 Register carry_add = high;
8758 __ mov(carry_add, CARRY_ADD);
8759
8760 // Limb 3
8761 __ add(tmp, c[3], carry_add);
8762 __ asr(tmp, tmp, bpl);
8763 __ add(c[4], c[4], tmp);
8764 __ lsl(tmp, tmp, bpl);
8765 __ sub(c[3], c[3], tmp);
8766
8767 // Limb 4
8768 __ add(tmp, c[4], carry_add);
8769 __ asr(tmp, tmp, bpl);
8770
8771 // Reduce high order limb and fold back into low order limb
8772 __ mul(term, tmp, term);
8773 __ add(c[0], c[0], term);
8774
8775 __ lsl(tmp, tmp, bpl);
8776 __ sub(c[4], c[4], tmp);
8777
8778 // Limbs 0 - 3
8779 for (int i = 0; i < (limbs - 1); i++) {
8780 __ add(tmp, c[i], carry_add);
8781 __ asr(tmp, tmp, bpl);
8782 __ add(c[i + 1], c[i + 1], tmp);
8783 __ lsl(tmp, tmp, bpl);
8784 __ sub(c[i], c[i], tmp);
8785 }
8786
8787 for (int i = 0; i < limbs; i++) {
8788 __ str(c[i], Address(rLimbs, i * 8));
8789 }
8790
8791 __ mov(r0, 0);
8792 __ leave(); // required for proper stackwalking of RuntimeStub frame
8793 __ ret(lr);
8794
8795 // record the stub entry and end
8796 store_archive_data(stub_id, start, __ pc());
8797
8798 return start;
8799 }
8800
8801 void bcax5(Register a0, Register a1, Register a2, Register a3, Register a4,
8802 Register tmp0, Register tmp1, Register tmp2) {
8803 __ bic(tmp0, a2, a1); // for a0
8804 __ bic(tmp1, a3, a2); // for a1
8805 __ bic(tmp2, a4, a3); // for a2
8806 __ eor(a2, a2, tmp2);
8807 __ bic(tmp2, a0, a4); // for a3
8808 __ eor(a3, a3, tmp2);
8809 __ bic(tmp2, a1, a0); // for a4
8810 __ eor(a0, a0, tmp0);
8811 __ eor(a1, a1, tmp1);
8812 __ eor(a4, a4, tmp2);
8813 }
8814
8815 void keccak_round_gpr(bool can_use_fp, bool can_use_r18, Register rc,
8816 Register a0, Register a1, Register a2, Register a3, Register a4,
8817 Register a5, Register a6, Register a7, Register a8, Register a9,
8818 Register a10, Register a11, Register a12, Register a13, Register a14,
8819 Register a15, Register a16, Register a17, Register a18, Register a19,
8820 Register a20, Register a21, Register a22, Register a23, Register a24,
8821 Register tmp0, Register tmp1, Register tmp2) {
8822 __ eor3(tmp1, a4, a9, a14);
8823 __ eor3(tmp0, tmp1, a19, a24); // tmp0 = a4^a9^a14^a19^a24 = c4
8824 __ eor3(tmp2, a1, a6, a11);
8825 __ eor3(tmp1, tmp2, a16, a21); // tmp1 = a1^a6^a11^a16^a21 = c1
8826 __ rax1(tmp2, tmp0, tmp1); // d0
8827 {
8828
8829 Register tmp3, tmp4;
8830 if (can_use_fp && can_use_r18) {
8831 tmp3 = rfp;
8832 tmp4 = r18_tls;
8833 } else {
8834 tmp3 = a4;
8835 tmp4 = a9;
8836 __ stp(tmp3, tmp4, __ pre(sp, -16));
8837 }
8838
8839 __ eor3(tmp3, a0, a5, a10);
8840 __ eor3(tmp4, tmp3, a15, a20); // tmp4 = a0^a5^a10^a15^a20 = c0
8841 __ eor(a0, a0, tmp2);
8842 __ eor(a5, a5, tmp2);
8843 __ eor(a10, a10, tmp2);
8844 __ eor(a15, a15, tmp2);
8845 __ eor(a20, a20, tmp2); // d0(tmp2)
8846 __ eor3(tmp3, a2, a7, a12);
8847 __ eor3(tmp2, tmp3, a17, a22); // tmp2 = a2^a7^a12^a17^a22 = c2
8848 __ rax1(tmp3, tmp4, tmp2); // d1
8849 __ eor(a1, a1, tmp3);
8850 __ eor(a6, a6, tmp3);
8851 __ eor(a11, a11, tmp3);
8852 __ eor(a16, a16, tmp3);
8853 __ eor(a21, a21, tmp3); // d1(tmp3)
8854 __ rax1(tmp3, tmp2, tmp0); // d3
8855 __ eor3(tmp2, a3, a8, a13);
8856 __ eor3(tmp0, tmp2, a18, a23); // tmp0 = a3^a8^a13^a18^a23 = c3
8857 __ eor(a3, a3, tmp3);
8858 __ eor(a8, a8, tmp3);
8859 __ eor(a13, a13, tmp3);
8860 __ eor(a18, a18, tmp3);
8861 __ eor(a23, a23, tmp3);
8862 __ rax1(tmp2, tmp1, tmp0); // d2
8863 __ eor(a2, a2, tmp2);
8864 __ eor(a7, a7, tmp2);
8865 __ eor(a12, a12, tmp2);
8866 __ rax1(tmp0, tmp0, tmp4); // d4
8867 if (!can_use_fp || !can_use_r18) {
8868 __ ldp(tmp3, tmp4, __ post(sp, 16));
8869 }
8870 __ eor(a17, a17, tmp2);
8871 __ eor(a22, a22, tmp2);
8872 __ eor(a4, a4, tmp0);
8873 __ eor(a9, a9, tmp0);
8874 __ eor(a14, a14, tmp0);
8875 __ eor(a19, a19, tmp0);
8876 __ eor(a24, a24, tmp0);
8877 }
8878
8879 __ rol(tmp0, a10, 3);
8880 __ rol(a10, a1, 1);
8881 __ rol(a1, a6, 44);
8882 __ rol(a6, a9, 20);
8883 __ rol(a9, a22, 61);
8884 __ rol(a22, a14, 39);
8885 __ rol(a14, a20, 18);
8886 __ rol(a20, a2, 62);
8887 __ rol(a2, a12, 43);
8888 __ rol(a12, a13, 25);
8889 __ rol(a13, a19, 8) ;
8890 __ rol(a19, a23, 56);
8891 __ rol(a23, a15, 41);
8892 __ rol(a15, a4, 27);
8893 __ rol(a4, a24, 14);
8894 __ rol(a24, a21, 2);
8895 __ rol(a21, a8, 55);
8896 __ rol(a8, a16, 45);
8897 __ rol(a16, a5, 36);
8898 __ rol(a5, a3, 28);
8899 __ rol(a3, a18, 21);
8900 __ rol(a18, a17, 15);
8901 __ rol(a17, a11, 10);
8902 __ rol(a11, a7, 6);
8903 __ mov(a7, tmp0);
8904
8905 bcax5(a0, a1, a2, a3, a4, tmp0, tmp1, tmp2);
8906 bcax5(a5, a6, a7, a8, a9, tmp0, tmp1, tmp2);
8907 bcax5(a10, a11, a12, a13, a14, tmp0, tmp1, tmp2);
8908 bcax5(a15, a16, a17, a18, a19, tmp0, tmp1, tmp2);
8909 bcax5(a20, a21, a22, a23, a24, tmp0, tmp1, tmp2);
8910
8911 __ ldr(tmp1, __ post(rc, 8));
8912 __ eor(a0, a0, tmp1);
8913
8914 }
8915
8916 // Arguments:
8917 //
8918 // Inputs:
8919 // c_rarg0 - byte[] source+offset
8920 // c_rarg1 - byte[] SHA.state
8921 // c_rarg2 - int block_size
8922 // c_rarg3 - int offset
8923 // c_rarg4 - int limit
8924 //
8925 address generate_sha3_implCompress_gpr(StubId stub_id) {
8926 bool multi_block;
8927 switch (stub_id) {
8928 case StubId::stubgen_sha3_implCompress_id:
8929 multi_block = false;
8930 break;
8931 case StubId::stubgen_sha3_implCompressMB_id:
8932 multi_block = true;
8933 break;
8934 default:
8935 ShouldNotReachHere();
8936 }
8937 int entry_count = StubInfo::entry_count(stub_id);
8938 assert(entry_count == 1, "sanity check");
8939 address start = load_archive_data(stub_id);
8940 if (start != nullptr) {
8941 return start;
8942 }
8943 __ align(CodeEntryAlignment);
8944 StubCodeMark mark(this, stub_id);
8945 start = __ pc();
8946
8947 Register buf = c_rarg0;
8948 Register state = c_rarg1;
8949 Register block_size = c_rarg2;
8950 Register ofs = c_rarg3;
8951 Register limit = c_rarg4;
8952
8953 // use r3.r17,r19..r28 to keep a0..a24.
8954 // a0..a24 are respective locals from SHA3.java
8955 Register a0 = r25,
8956 a1 = r26,
8957 a2 = r27,
8958 a3 = r3,
8959 a4 = r4,
8960 a5 = r5,
8961 a6 = r6,
8962 a7 = r7,
8963 a8 = rscratch1, // r8
8964 a9 = rscratch2, // r9
8965 a10 = r10,
8966 a11 = r11,
8967 a12 = r12,
8968 a13 = r13,
8969 a14 = r14,
8970 a15 = r15,
8971 a16 = r16,
8972 a17 = r17,
8973 a18 = r28,
8974 a19 = r19,
8975 a20 = r20,
8976 a21 = r21,
8977 a22 = r22,
8978 a23 = r23,
8979 a24 = r24;
8980
8981 Register tmp0 = block_size, tmp1 = buf, tmp2 = state, tmp3 = r30;
8982
8983 Label sha3_loop, rounds24_preloop, loop_body;
8984 Label sha3_512_or_sha3_384, shake128;
8985
8986 bool can_use_r18 = false;
8987 #ifndef R18_RESERVED
8988 can_use_r18 = true;
8989 #endif
8990 bool can_use_fp = !PreserveFramePointer;
8991
8992 __ enter();
8993
8994 // save almost all yet unsaved gpr registers on stack
8995 __ str(block_size, __ pre(sp, -128));
8996 if (multi_block) {
8997 __ stpw(ofs, limit, Address(sp, 8));
8998 }
8999 // 8 bytes at sp+16 will be used to keep buf
9000 __ stp(r19, r20, Address(sp, 32));
9001 __ stp(r21, r22, Address(sp, 48));
9002 __ stp(r23, r24, Address(sp, 64));
9003 __ stp(r25, r26, Address(sp, 80));
9004 __ stp(r27, r28, Address(sp, 96));
9005 if (can_use_r18 && can_use_fp) {
9006 __ stp(r18_tls, state, Address(sp, 112));
9007 } else {
9008 __ str(state, Address(sp, 112));
9009 }
9010
9011 // begin sha3 calculations: loading a0..a24 from state arrary
9012 __ ldp(a0, a1, state);
9013 __ ldp(a2, a3, Address(state, 16));
9014 __ ldp(a4, a5, Address(state, 32));
9015 __ ldp(a6, a7, Address(state, 48));
9016 __ ldp(a8, a9, Address(state, 64));
9017 __ ldp(a10, a11, Address(state, 80));
9018 __ ldp(a12, a13, Address(state, 96));
9019 __ ldp(a14, a15, Address(state, 112));
9020 __ ldp(a16, a17, Address(state, 128));
9021 __ ldp(a18, a19, Address(state, 144));
9022 __ ldp(a20, a21, Address(state, 160));
9023 __ ldp(a22, a23, Address(state, 176));
9024 __ ldr(a24, Address(state, 192));
9025
9026 __ BIND(sha3_loop);
9027
9028 // load input
9029 __ ldp(tmp3, tmp2, __ post(buf, 16));
9030 __ eor(a0, a0, tmp3);
9031 __ eor(a1, a1, tmp2);
9032 __ ldp(tmp3, tmp2, __ post(buf, 16));
9033 __ eor(a2, a2, tmp3);
9034 __ eor(a3, a3, tmp2);
9035 __ ldp(tmp3, tmp2, __ post(buf, 16));
9036 __ eor(a4, a4, tmp3);
9037 __ eor(a5, a5, tmp2);
9038 __ ldr(tmp3, __ post(buf, 8));
9039 __ eor(a6, a6, tmp3);
9040
9041 // block_size == 72, SHA3-512; block_size == 104, SHA3-384
9042 __ tbz(block_size, 7, sha3_512_or_sha3_384);
9043
9044 __ ldp(tmp3, tmp2, __ post(buf, 16));
9045 __ eor(a7, a7, tmp3);
9046 __ eor(a8, a8, tmp2);
9047 __ ldp(tmp3, tmp2, __ post(buf, 16));
9048 __ eor(a9, a9, tmp3);
9049 __ eor(a10, a10, tmp2);
9050 __ ldp(tmp3, tmp2, __ post(buf, 16));
9051 __ eor(a11, a11, tmp3);
9052 __ eor(a12, a12, tmp2);
9053 __ ldp(tmp3, tmp2, __ post(buf, 16));
9054 __ eor(a13, a13, tmp3);
9055 __ eor(a14, a14, tmp2);
9056 __ ldp(tmp3, tmp2, __ post(buf, 16));
9057 __ eor(a15, a15, tmp3);
9058 __ eor(a16, a16, tmp2);
9059
9060 // block_size == 136, bit4 == 0 and bit5 == 0, SHA3-256 or SHAKE256
9061 __ andw(tmp2, block_size, 48);
9062 __ cbzw(tmp2, rounds24_preloop);
9063 __ tbnz(block_size, 5, shake128);
9064 // block_size == 144, bit5 == 0, SHA3-244
9065 __ ldr(tmp3, __ post(buf, 8));
9066 __ eor(a17, a17, tmp3);
9067 __ b(rounds24_preloop);
9068
9069 __ BIND(shake128);
9070 __ ldp(tmp3, tmp2, __ post(buf, 16));
9071 __ eor(a17, a17, tmp3);
9072 __ eor(a18, a18, tmp2);
9073 __ ldp(tmp3, tmp2, __ post(buf, 16));
9074 __ eor(a19, a19, tmp3);
9075 __ eor(a20, a20, tmp2);
9076 __ b(rounds24_preloop); // block_size == 168, SHAKE128
9077
9078 __ BIND(sha3_512_or_sha3_384);
9079 __ ldp(tmp3, tmp2, __ post(buf, 16));
9080 __ eor(a7, a7, tmp3);
9081 __ eor(a8, a8, tmp2);
9082 __ tbz(block_size, 5, rounds24_preloop); // SHA3-512
9083
9084 // SHA3-384
9085 __ ldp(tmp3, tmp2, __ post(buf, 16));
9086 __ eor(a9, a9, tmp3);
9087 __ eor(a10, a10, tmp2);
9088 __ ldp(tmp3, tmp2, __ post(buf, 16));
9089 __ eor(a11, a11, tmp3);
9090 __ eor(a12, a12, tmp2);
9091
9092 __ BIND(rounds24_preloop);
9093 __ fmovs(v0, 24.0); // float loop counter,
9094 __ fmovs(v1, 1.0); // exact representation
9095
9096 __ str(buf, Address(sp, 16));
9097 __ lea(tmp3, ExternalAddress((address) _sha3_round_consts));
9098
9099 __ BIND(loop_body);
9100 keccak_round_gpr(can_use_fp, can_use_r18, tmp3,
9101 a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12,
9102 a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24,
9103 tmp0, tmp1, tmp2);
9104 __ fsubs(v0, v0, v1);
9105 __ fcmps(v0, 0.0);
9106 __ br(__ NE, loop_body);
9107
9108 if (multi_block) {
9109 __ ldrw(block_size, sp); // block_size
9110 __ ldpw(tmp2, tmp1, Address(sp, 8)); // offset, limit
9111 __ addw(tmp2, tmp2, block_size);
9112 __ cmpw(tmp2, tmp1);
9113 __ strw(tmp2, Address(sp, 8)); // store offset in case we're jumping
9114 __ ldr(buf, Address(sp, 16)); // restore buf in case we're jumping
9115 __ br(Assembler::LE, sha3_loop);
9116 __ movw(c_rarg0, tmp2); // return offset
9117 }
9118 if (can_use_fp && can_use_r18) {
9119 __ ldp(r18_tls, state, Address(sp, 112));
9120 } else {
9121 __ ldr(state, Address(sp, 112));
9122 }
9123 // save calculated sha3 state
9124 __ stp(a0, a1, Address(state));
9125 __ stp(a2, a3, Address(state, 16));
9126 __ stp(a4, a5, Address(state, 32));
9127 __ stp(a6, a7, Address(state, 48));
9128 __ stp(a8, a9, Address(state, 64));
9129 __ stp(a10, a11, Address(state, 80));
9130 __ stp(a12, a13, Address(state, 96));
9131 __ stp(a14, a15, Address(state, 112));
9132 __ stp(a16, a17, Address(state, 128));
9133 __ stp(a18, a19, Address(state, 144));
9134 __ stp(a20, a21, Address(state, 160));
9135 __ stp(a22, a23, Address(state, 176));
9136 __ str(a24, Address(state, 192));
9137
9138 // restore required registers from stack
9139 __ ldp(r19, r20, Address(sp, 32));
9140 __ ldp(r21, r22, Address(sp, 48));
9141 __ ldp(r23, r24, Address(sp, 64));
9142 __ ldp(r25, r26, Address(sp, 80));
9143 __ ldp(r27, r28, Address(sp, 96));
9144 if (can_use_fp && can_use_r18) {
9145 __ add(rfp, sp, 128); // leave() will copy rfp to sp below
9146 } // else no need to recalculate rfp, since it wasn't changed
9147
9148 __ leave();
9149
9150 __ ret(lr);
9151
9152 // record the stub entry and end
9153 store_archive_data(stub_id, start, __ pc());
9154
9155 return start;
9156 }
9157
9158 /**
9159 * Arguments:
9160 *
9161 * Inputs:
9162 * c_rarg0 - int crc
9163 * c_rarg1 - byte* buf
9164 * c_rarg2 - int length
9165 *
9166 * Output:
9167 * rax - int crc result
9168 */
9169 address generate_updateBytesCRC32() {
9170 assert(UseCRC32Intrinsics, "what are we doing here?");
9171 StubId stub_id = StubId::stubgen_updateBytesCRC32_id;
9172 int entry_count = StubInfo::entry_count(stub_id);
9173 assert(entry_count == 1, "sanity check");
9174 address start = load_archive_data(stub_id);
9175 if (start != nullptr) {
9176 return start;
9177 }
9178 __ align(CodeEntryAlignment);
9179 StubCodeMark mark(this, stub_id);
9180
9181 start = __ pc();
9182
9183 const Register crc = c_rarg0; // crc
9184 const Register buf = c_rarg1; // source java byte array address
9185 const Register len = c_rarg2; // length
9186 const Register table0 = c_rarg3; // crc_table address
9187 const Register table1 = c_rarg4;
9188 const Register table2 = c_rarg5;
9189 const Register table3 = c_rarg6;
9190 const Register tmp3 = c_rarg7;
9191
9192 BLOCK_COMMENT("Entry:");
9193 __ enter(); // required for proper stackwalking of RuntimeStub frame
9194
9195 __ kernel_crc32(crc, buf, len,
9196 table0, table1, table2, table3, rscratch1, rscratch2, tmp3);
9197
9198 __ leave(); // required for proper stackwalking of RuntimeStub frame
9199 __ ret(lr);
9200
9201 // record the stub entry and end
9202 store_archive_data(stub_id, start, __ pc());
9203
9204 return start;
9205 }
9206
9207 /**
9208 * Arguments:
9209 *
9210 * Inputs:
9211 * c_rarg0 - int crc
9212 * c_rarg1 - byte* buf
9213 * c_rarg2 - int length
9214 * c_rarg3 - int* table
9215 *
9216 * Output:
9217 * r0 - int crc result
9218 */
9219 address generate_updateBytesCRC32C() {
9220 assert(UseCRC32CIntrinsics, "what are we doing here?");
9221 StubId stub_id = StubId::stubgen_updateBytesCRC32C_id;
9222 int entry_count = StubInfo::entry_count(stub_id);
9223 assert(entry_count == 1, "sanity check");
9224 address start = load_archive_data(stub_id);
9225 if (start != nullptr) {
9226 return start;
9227 }
9228 __ align(CodeEntryAlignment);
9229 StubCodeMark mark(this, stub_id);
9230
9231 start = __ pc();
9232
9233 const Register crc = c_rarg0; // crc
9234 const Register buf = c_rarg1; // source java byte array address
9235 const Register len = c_rarg2; // length
9236 const Register table0 = c_rarg3; // crc_table address
9237 const Register table1 = c_rarg4;
9238 const Register table2 = c_rarg5;
9239 const Register table3 = c_rarg6;
9240 const Register tmp3 = c_rarg7;
9241
9242 BLOCK_COMMENT("Entry:");
9243 __ enter(); // required for proper stackwalking of RuntimeStub frame
9244
9245 __ kernel_crc32c(crc, buf, len,
9246 table0, table1, table2, table3, rscratch1, rscratch2, tmp3);
9247
9248 __ leave(); // required for proper stackwalking of RuntimeStub frame
9249 __ ret(lr);
9250
9251 // record the stub entry and end
9252 store_archive_data(stub_id, start, __ pc());
9253
9254 return start;
9255 }
9256
9257 /***
9258 * Arguments:
9259 *
9260 * Inputs:
9261 * c_rarg0 - int adler
9262 * c_rarg1 - byte* buff
9263 * c_rarg2 - int len
9264 *
9265 * Output:
9266 * c_rarg0 - int adler result
9267 */
9268 address generate_updateBytesAdler32() {
9269 StubId stub_id = StubId::stubgen_updateBytesAdler32_id;
9270 int entry_count = StubInfo::entry_count(stub_id);
9271 assert(entry_count == 1, "sanity check");
9272 address start = load_archive_data(stub_id);
9273 if (start != nullptr) {
9274 return start;
9275 }
9276 __ align(CodeEntryAlignment);
9277 StubCodeMark mark(this, stub_id);
9278 start = __ pc();
9279
9280 Label L_simple_by1_loop, L_nmax, L_nmax_loop, L_by16, L_by16_loop, L_by1_loop, L_do_mod, L_combine, L_by1;
9281
9282 // Aliases
9283 Register adler = c_rarg0;
9284 Register s1 = c_rarg0;
9285 Register s2 = c_rarg3;
9286 Register buff = c_rarg1;
9287 Register len = c_rarg2;
9288 Register nmax = r4;
9289 Register base = r5;
9290 Register count = r6;
9291 Register temp0 = rscratch1;
9292 Register temp1 = rscratch2;
9293 FloatRegister vbytes = v0;
9294 FloatRegister vs1acc = v1;
9295 FloatRegister vs2acc = v2;
9296 FloatRegister vtable = v3;
9297
9298 // Max number of bytes we can process before having to take the mod
9299 // 0x15B0 is 5552 in decimal, the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
9300 uint64_t BASE = 0xfff1;
9301 uint64_t NMAX = 0x15B0;
9302
9303 __ mov(base, BASE);
9304 __ mov(nmax, NMAX);
9305
9306 // Load accumulation coefficients for the upper 16 bits
9307 __ lea(temp0, ExternalAddress((address) StubRoutines::aarch64::_adler_table));
9308 __ ld1(vtable, __ T16B, Address(temp0));
9309
9310 // s1 is initialized to the lower 16 bits of adler
9311 // s2 is initialized to the upper 16 bits of adler
9312 __ ubfx(s2, adler, 16, 16); // s2 = ((adler >> 16) & 0xffff)
9313 __ uxth(s1, adler); // s1 = (adler & 0xffff)
9314
9315 // The pipelined loop needs at least 16 elements for 1 iteration
9316 // It does check this, but it is more effective to skip to the cleanup loop
9317 __ cmp(len, (u1)16);
9318 __ br(Assembler::HS, L_nmax);
9319 __ cbz(len, L_combine);
9320
9321 __ bind(L_simple_by1_loop);
9322 __ ldrb(temp0, Address(__ post(buff, 1)));
9323 __ add(s1, s1, temp0);
9324 __ add(s2, s2, s1);
9325 __ subs(len, len, 1);
9326 __ br(Assembler::HI, L_simple_by1_loop);
9327
9328 // s1 = s1 % BASE
9329 __ subs(temp0, s1, base);
9330 __ csel(s1, temp0, s1, Assembler::HS);
9331
9332 // s2 = s2 % BASE
9333 __ lsr(temp0, s2, 16);
9334 __ lsl(temp1, temp0, 4);
9335 __ sub(temp1, temp1, temp0);
9336 __ add(s2, temp1, s2, ext::uxth);
9337
9338 __ subs(temp0, s2, base);
9339 __ csel(s2, temp0, s2, Assembler::HS);
9340
9341 __ b(L_combine);
9342
9343 __ bind(L_nmax);
9344 __ subs(len, len, nmax);
9345 __ sub(count, nmax, 16);
9346 __ br(Assembler::LO, L_by16);
9347
9348 __ bind(L_nmax_loop);
9349
9350 generate_updateBytesAdler32_accum(s1, s2, buff, temp0, temp1,
9351 vbytes, vs1acc, vs2acc, vtable);
9352
9353 __ subs(count, count, 16);
9354 __ br(Assembler::HS, L_nmax_loop);
9355
9356 // s1 = s1 % BASE
9357 __ lsr(temp0, s1, 16);
9358 __ lsl(temp1, temp0, 4);
9359 __ sub(temp1, temp1, temp0);
9360 __ add(temp1, temp1, s1, ext::uxth);
9361
9362 __ lsr(temp0, temp1, 16);
9363 __ lsl(s1, temp0, 4);
9364 __ sub(s1, s1, temp0);
9365 __ add(s1, s1, temp1, ext:: uxth);
9366
9367 __ subs(temp0, s1, base);
9368 __ csel(s1, temp0, s1, Assembler::HS);
9369
9370 // s2 = s2 % BASE
9371 __ lsr(temp0, s2, 16);
9372 __ lsl(temp1, temp0, 4);
9373 __ sub(temp1, temp1, temp0);
9374 __ add(temp1, temp1, s2, ext::uxth);
9375
9376 __ lsr(temp0, temp1, 16);
9377 __ lsl(s2, temp0, 4);
9378 __ sub(s2, s2, temp0);
9379 __ add(s2, s2, temp1, ext:: uxth);
9380
9381 __ subs(temp0, s2, base);
9382 __ csel(s2, temp0, s2, Assembler::HS);
9383
9384 __ subs(len, len, nmax);
9385 __ sub(count, nmax, 16);
9386 __ br(Assembler::HS, L_nmax_loop);
9387
9388 __ bind(L_by16);
9389 __ adds(len, len, count);
9390 __ br(Assembler::LO, L_by1);
9391
9392 __ bind(L_by16_loop);
9393
9394 generate_updateBytesAdler32_accum(s1, s2, buff, temp0, temp1,
9395 vbytes, vs1acc, vs2acc, vtable);
9396
9397 __ subs(len, len, 16);
9398 __ br(Assembler::HS, L_by16_loop);
9399
9400 __ bind(L_by1);
9401 __ adds(len, len, 15);
9402 __ br(Assembler::LO, L_do_mod);
9403
9404 __ bind(L_by1_loop);
9405 __ ldrb(temp0, Address(__ post(buff, 1)));
9406 __ add(s1, temp0, s1);
9407 __ add(s2, s2, s1);
9408 __ subs(len, len, 1);
9409 __ br(Assembler::HS, L_by1_loop);
9410
9411 __ bind(L_do_mod);
9412 // s1 = s1 % BASE
9413 __ lsr(temp0, s1, 16);
9414 __ lsl(temp1, temp0, 4);
9415 __ sub(temp1, temp1, temp0);
9416 __ add(temp1, temp1, s1, ext::uxth);
9417
9418 __ lsr(temp0, temp1, 16);
9419 __ lsl(s1, temp0, 4);
9420 __ sub(s1, s1, temp0);
9421 __ add(s1, s1, temp1, ext:: uxth);
9422
9423 __ subs(temp0, s1, base);
9424 __ csel(s1, temp0, s1, Assembler::HS);
9425
9426 // s2 = s2 % BASE
9427 __ lsr(temp0, s2, 16);
9428 __ lsl(temp1, temp0, 4);
9429 __ sub(temp1, temp1, temp0);
9430 __ add(temp1, temp1, s2, ext::uxth);
9431
9432 __ lsr(temp0, temp1, 16);
9433 __ lsl(s2, temp0, 4);
9434 __ sub(s2, s2, temp0);
9435 __ add(s2, s2, temp1, ext:: uxth);
9436
9437 __ subs(temp0, s2, base);
9438 __ csel(s2, temp0, s2, Assembler::HS);
9439
9440 // Combine lower bits and higher bits
9441 __ bind(L_combine);
9442 __ orr(s1, s1, s2, Assembler::LSL, 16); // adler = s1 | (s2 << 16)
9443
9444 __ ret(lr);
9445
9446 // record the stub entry and end
9447 store_archive_data(stub_id, start, __ pc());
9448
9449 return start;
9450 }
9451
9452 void generate_updateBytesAdler32_accum(Register s1, Register s2, Register buff,
9453 Register temp0, Register temp1, FloatRegister vbytes,
9454 FloatRegister vs1acc, FloatRegister vs2acc, FloatRegister vtable) {
9455 // Below is a vectorized implementation of updating s1 and s2 for 16 bytes.
9456 // We use b1, b2, ..., b16 to denote the 16 bytes loaded in each iteration.
9457 // In non-vectorized code, we update s1 and s2 as:
9458 // s1 <- s1 + b1
9459 // s2 <- s2 + s1
9460 // s1 <- s1 + b2
9461 // s2 <- s2 + b1
9462 // ...
9463 // s1 <- s1 + b16
9464 // s2 <- s2 + s1
9465 // Putting above assignments together, we have:
9466 // s1_new = s1 + b1 + b2 + ... + b16
9467 // s2_new = s2 + (s1 + b1) + (s1 + b1 + b2) + ... + (s1 + b1 + b2 + ... + b16)
9468 // = s2 + s1 * 16 + (b1 * 16 + b2 * 15 + ... + b16 * 1)
9469 // = s2 + s1 * 16 + (b1, b2, ... b16) dot (16, 15, ... 1)
9470 __ ld1(vbytes, __ T16B, Address(__ post(buff, 16)));
9471
9472 // s2 = s2 + s1 * 16
9473 __ add(s2, s2, s1, Assembler::LSL, 4);
9474
9475 // vs1acc = b1 + b2 + b3 + ... + b16
9476 // vs2acc = (b1 * 16) + (b2 * 15) + (b3 * 14) + ... + (b16 * 1)
9477 __ umullv(vs2acc, __ T8B, vtable, vbytes);
9478 __ umlalv(vs2acc, __ T16B, vtable, vbytes);
9479 __ uaddlv(vs1acc, __ T16B, vbytes);
9480 __ uaddlv(vs2acc, __ T8H, vs2acc);
9481
9482 // s1 = s1 + vs1acc, s2 = s2 + vs2acc
9483 __ fmovd(temp0, vs1acc);
9484 __ fmovd(temp1, vs2acc);
9485 __ add(s1, s1, temp0);
9486 __ add(s2, s2, temp1);
9487 }
9488
9489 /**
9490 * Arguments:
9491 *
9492 * Input:
9493 * c_rarg0 - x address
9494 * c_rarg1 - x length
9495 * c_rarg2 - y address
9496 * c_rarg3 - y length
9497 * c_rarg4 - z address
9498 */
9499 address generate_multiplyToLen() {
9500 StubId stub_id = StubId::stubgen_multiplyToLen_id;
9501 int entry_count = StubInfo::entry_count(stub_id);
9502 assert(entry_count == 1, "sanity check");
9503 address start = load_archive_data(stub_id);
9504 if (start != nullptr) {
9505 return start;
9506 }
9507 __ align(CodeEntryAlignment);
9508 StubCodeMark mark(this, stub_id);
9509
9510 start = __ pc();
9511 const Register x = r0;
9512 const Register xlen = r1;
9513 const Register y = r2;
9514 const Register ylen = r3;
9515 const Register z = r4;
9516
9517 const Register tmp0 = r5;
9518 const Register tmp1 = r10;
9519 const Register tmp2 = r11;
9520 const Register tmp3 = r12;
9521 const Register tmp4 = r13;
9522 const Register tmp5 = r14;
9523 const Register tmp6 = r15;
9524 const Register tmp7 = r16;
9525
9526 BLOCK_COMMENT("Entry:");
9527 __ enter(); // required for proper stackwalking of RuntimeStub frame
9528 __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
9529 __ leave(); // required for proper stackwalking of RuntimeStub frame
9530 __ ret(lr);
9531
9532 // record the stub entry and end
9533 store_archive_data(stub_id, start, __ pc());
9534
9535 return start;
9536 }
9537
9538 address generate_squareToLen() {
9539 // squareToLen algorithm for sizes 1..127 described in java code works
9540 // faster than multiply_to_len on some CPUs and slower on others, but
9541 // multiply_to_len shows a bit better overall results
9542 StubId stub_id = StubId::stubgen_squareToLen_id;
9543 int entry_count = StubInfo::entry_count(stub_id);
9544 assert(entry_count == 1, "sanity check");
9545 address start = load_archive_data(stub_id);
9546 if (start != nullptr) {
9547 return start;
9548 }
9549 __ align(CodeEntryAlignment);
9550 StubCodeMark mark(this, stub_id);
9551 start = __ pc();
9552
9553 const Register x = r0;
9554 const Register xlen = r1;
9555 const Register z = r2;
9556 const Register y = r4; // == x
9557 const Register ylen = r5; // == xlen
9558
9559 const Register tmp0 = r3;
9560 const Register tmp1 = r10;
9561 const Register tmp2 = r11;
9562 const Register tmp3 = r12;
9563 const Register tmp4 = r13;
9564 const Register tmp5 = r14;
9565 const Register tmp6 = r15;
9566 const Register tmp7 = r16;
9567
9568 RegSet spilled_regs = RegSet::of(y, ylen);
9569 BLOCK_COMMENT("Entry:");
9570 __ enter();
9571 __ push(spilled_regs, sp);
9572 __ mov(y, x);
9573 __ mov(ylen, xlen);
9574 __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
9575 __ pop(spilled_regs, sp);
9576 __ leave();
9577 __ ret(lr);
9578
9579 // record the stub entry and end
9580 store_archive_data(stub_id, start, __ pc());
9581
9582 return start;
9583 }
9584
9585 address generate_mulAdd() {
9586 StubId stub_id = StubId::stubgen_mulAdd_id;
9587 int entry_count = StubInfo::entry_count(stub_id);
9588 assert(entry_count == 1, "sanity check");
9589 address start = load_archive_data(stub_id);
9590 if (start != nullptr) {
9591 return start;
9592 }
9593 __ align(CodeEntryAlignment);
9594 StubCodeMark mark(this, stub_id);
9595
9596 start = __ pc();
9597
9598 const Register out = r0;
9599 const Register in = r1;
9600 const Register offset = r2;
9601 const Register len = r3;
9602 const Register k = r4;
9603
9604 BLOCK_COMMENT("Entry:");
9605 __ enter();
9606 __ mul_add(out, in, offset, len, k);
9607 __ leave();
9608 __ ret(lr);
9609
9610 // record the stub entry and end
9611 store_archive_data(stub_id, start, __ pc());
9612
9613 return start;
9614 }
9615
9616 // Arguments:
9617 //
9618 // Input:
9619 // c_rarg0 - newArr address
9620 // c_rarg1 - oldArr address
9621 // c_rarg2 - newIdx
9622 // c_rarg3 - shiftCount
9623 // c_rarg4 - numIter
9624 //
9625 address generate_bigIntegerRightShift() {
9626 StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id;
9627 int entry_count = StubInfo::entry_count(stub_id);
9628 assert(entry_count == 1, "sanity check");
9629 address start = load_archive_data(stub_id);
9630 if (start != nullptr) {
9631 return start;
9632 }
9633 __ align(CodeEntryAlignment);
9634 StubCodeMark mark(this, stub_id);
9635 start = __ pc();
9636
9637 Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit;
9638
9639 Register newArr = c_rarg0;
9640 Register oldArr = c_rarg1;
9641 Register newIdx = c_rarg2;
9642 Register shiftCount = c_rarg3;
9643 Register numIter = c_rarg4;
9644 Register idx = numIter;
9645
9646 Register newArrCur = rscratch1;
9647 Register shiftRevCount = rscratch2;
9648 Register oldArrCur = r13;
9649 Register oldArrNext = r14;
9650
9651 FloatRegister oldElem0 = v0;
9652 FloatRegister oldElem1 = v1;
9653 FloatRegister newElem = v2;
9654 FloatRegister shiftVCount = v3;
9655 FloatRegister shiftVRevCount = v4;
9656
9657 __ cbz(idx, Exit);
9658
9659 __ add(newArr, newArr, newIdx, Assembler::LSL, 2);
9660
9661 // left shift count
9662 __ movw(shiftRevCount, 32);
9663 __ subw(shiftRevCount, shiftRevCount, shiftCount);
9664
9665 // numIter too small to allow a 4-words SIMD loop, rolling back
9666 __ cmp(numIter, (u1)4);
9667 __ br(Assembler::LT, ShiftThree);
9668
9669 __ dup(shiftVCount, __ T4S, shiftCount);
9670 __ dup(shiftVRevCount, __ T4S, shiftRevCount);
9671 __ negr(shiftVCount, __ T4S, shiftVCount);
9672
9673 __ BIND(ShiftSIMDLoop);
9674
9675 // Calculate the load addresses
9676 __ sub(idx, idx, 4);
9677 __ add(oldArrNext, oldArr, idx, Assembler::LSL, 2);
9678 __ add(newArrCur, newArr, idx, Assembler::LSL, 2);
9679 __ add(oldArrCur, oldArrNext, 4);
9680
9681 // Load 4 words and process
9682 __ ld1(oldElem0, __ T4S, Address(oldArrCur));
9683 __ ld1(oldElem1, __ T4S, Address(oldArrNext));
9684 __ ushl(oldElem0, __ T4S, oldElem0, shiftVCount);
9685 __ ushl(oldElem1, __ T4S, oldElem1, shiftVRevCount);
9686 __ orr(newElem, __ T16B, oldElem0, oldElem1);
9687 __ st1(newElem, __ T4S, Address(newArrCur));
9688
9689 __ cmp(idx, (u1)4);
9690 __ br(Assembler::LT, ShiftTwoLoop);
9691 __ b(ShiftSIMDLoop);
9692
9693 __ BIND(ShiftTwoLoop);
9694 __ cbz(idx, Exit);
9695 __ cmp(idx, (u1)1);
9696 __ br(Assembler::EQ, ShiftOne);
9697
9698 // Calculate the load addresses
9699 __ sub(idx, idx, 2);
9700 __ add(oldArrNext, oldArr, idx, Assembler::LSL, 2);
9701 __ add(newArrCur, newArr, idx, Assembler::LSL, 2);
9702 __ add(oldArrCur, oldArrNext, 4);
9703
9704 // Load 2 words and process
9705 __ ld1(oldElem0, __ T2S, Address(oldArrCur));
9706 __ ld1(oldElem1, __ T2S, Address(oldArrNext));
9707 __ ushl(oldElem0, __ T2S, oldElem0, shiftVCount);
9708 __ ushl(oldElem1, __ T2S, oldElem1, shiftVRevCount);
9709 __ orr(newElem, __ T8B, oldElem0, oldElem1);
9710 __ st1(newElem, __ T2S, Address(newArrCur));
9711 __ b(ShiftTwoLoop);
9712
9713 __ BIND(ShiftThree);
9714 __ tbz(idx, 1, ShiftOne);
9715 __ tbz(idx, 0, ShiftTwo);
9716 __ ldrw(r10, Address(oldArr, 12));
9717 __ ldrw(r11, Address(oldArr, 8));
9718 __ lsrvw(r10, r10, shiftCount);
9719 __ lslvw(r11, r11, shiftRevCount);
9720 __ orrw(r12, r10, r11);
9721 __ strw(r12, Address(newArr, 8));
9722
9723 __ BIND(ShiftTwo);
9724 __ ldrw(r10, Address(oldArr, 8));
9725 __ ldrw(r11, Address(oldArr, 4));
9726 __ lsrvw(r10, r10, shiftCount);
9727 __ lslvw(r11, r11, shiftRevCount);
9728 __ orrw(r12, r10, r11);
9729 __ strw(r12, Address(newArr, 4));
9730
9731 __ BIND(ShiftOne);
9732 __ ldrw(r10, Address(oldArr, 4));
9733 __ ldrw(r11, Address(oldArr));
9734 __ lsrvw(r10, r10, shiftCount);
9735 __ lslvw(r11, r11, shiftRevCount);
9736 __ orrw(r12, r10, r11);
9737 __ strw(r12, Address(newArr));
9738
9739 __ BIND(Exit);
9740 __ ret(lr);
9741
9742 // record the stub entry and end
9743 store_archive_data(stub_id, start, __ pc());
9744
9745 return start;
9746 }
9747
9748 // Arguments:
9749 //
9750 // Input:
9751 // c_rarg0 - newArr address
9752 // c_rarg1 - oldArr address
9753 // c_rarg2 - newIdx
9754 // c_rarg3 - shiftCount
9755 // c_rarg4 - numIter
9756 //
9757 address generate_bigIntegerLeftShift() {
9758 StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id;
9759 int entry_count = StubInfo::entry_count(stub_id);
9760 assert(entry_count == 1, "sanity check");
9761 address start = load_archive_data(stub_id);
9762 if (start != nullptr) {
9763 return start;
9764 }
9765 __ align(CodeEntryAlignment);
9766 StubCodeMark mark(this, stub_id);
9767 start = __ pc();
9768
9769 Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit;
9770
9771 Register newArr = c_rarg0;
9772 Register oldArr = c_rarg1;
9773 Register newIdx = c_rarg2;
9774 Register shiftCount = c_rarg3;
9775 Register numIter = c_rarg4;
9776
9777 Register shiftRevCount = rscratch1;
9778 Register oldArrNext = rscratch2;
9779
9780 FloatRegister oldElem0 = v0;
9781 FloatRegister oldElem1 = v1;
9782 FloatRegister newElem = v2;
9783 FloatRegister shiftVCount = v3;
9784 FloatRegister shiftVRevCount = v4;
9785
9786 __ cbz(numIter, Exit);
9787
9788 __ add(oldArrNext, oldArr, 4);
9789 __ add(newArr, newArr, newIdx, Assembler::LSL, 2);
9790
9791 // right shift count
9792 __ movw(shiftRevCount, 32);
9793 __ subw(shiftRevCount, shiftRevCount, shiftCount);
9794
9795 // numIter too small to allow a 4-words SIMD loop, rolling back
9796 __ cmp(numIter, (u1)4);
9797 __ br(Assembler::LT, ShiftThree);
9798
9799 __ dup(shiftVCount, __ T4S, shiftCount);
9800 __ dup(shiftVRevCount, __ T4S, shiftRevCount);
9801 __ negr(shiftVRevCount, __ T4S, shiftVRevCount);
9802
9803 __ BIND(ShiftSIMDLoop);
9804
9805 // load 4 words and process
9806 __ ld1(oldElem0, __ T4S, __ post(oldArr, 16));
9807 __ ld1(oldElem1, __ T4S, __ post(oldArrNext, 16));
9808 __ ushl(oldElem0, __ T4S, oldElem0, shiftVCount);
9809 __ ushl(oldElem1, __ T4S, oldElem1, shiftVRevCount);
9810 __ orr(newElem, __ T16B, oldElem0, oldElem1);
9811 __ st1(newElem, __ T4S, __ post(newArr, 16));
9812 __ sub(numIter, numIter, 4);
9813
9814 __ cmp(numIter, (u1)4);
9815 __ br(Assembler::LT, ShiftTwoLoop);
9816 __ b(ShiftSIMDLoop);
9817
9818 __ BIND(ShiftTwoLoop);
9819 __ cbz(numIter, Exit);
9820 __ cmp(numIter, (u1)1);
9821 __ br(Assembler::EQ, ShiftOne);
9822
9823 // load 2 words and process
9824 __ ld1(oldElem0, __ T2S, __ post(oldArr, 8));
9825 __ ld1(oldElem1, __ T2S, __ post(oldArrNext, 8));
9826 __ ushl(oldElem0, __ T2S, oldElem0, shiftVCount);
9827 __ ushl(oldElem1, __ T2S, oldElem1, shiftVRevCount);
9828 __ orr(newElem, __ T8B, oldElem0, oldElem1);
9829 __ st1(newElem, __ T2S, __ post(newArr, 8));
9830 __ sub(numIter, numIter, 2);
9831 __ b(ShiftTwoLoop);
9832
9833 __ BIND(ShiftThree);
9834 __ ldrw(r10, __ post(oldArr, 4));
9835 __ ldrw(r11, __ post(oldArrNext, 4));
9836 __ lslvw(r10, r10, shiftCount);
9837 __ lsrvw(r11, r11, shiftRevCount);
9838 __ orrw(r12, r10, r11);
9839 __ strw(r12, __ post(newArr, 4));
9840 __ tbz(numIter, 1, Exit);
9841 __ tbz(numIter, 0, ShiftOne);
9842
9843 __ BIND(ShiftTwo);
9844 __ ldrw(r10, __ post(oldArr, 4));
9845 __ ldrw(r11, __ post(oldArrNext, 4));
9846 __ lslvw(r10, r10, shiftCount);
9847 __ lsrvw(r11, r11, shiftRevCount);
9848 __ orrw(r12, r10, r11);
9849 __ strw(r12, __ post(newArr, 4));
9850
9851 __ BIND(ShiftOne);
9852 __ ldrw(r10, Address(oldArr));
9853 __ ldrw(r11, Address(oldArrNext));
9854 __ lslvw(r10, r10, shiftCount);
9855 __ lsrvw(r11, r11, shiftRevCount);
9856 __ orrw(r12, r10, r11);
9857 __ strw(r12, Address(newArr));
9858
9859 __ BIND(Exit);
9860 __ ret(lr);
9861
9862 // record the stub entry and end
9863 store_archive_data(stub_id, start, __ pc());
9864
9865 return start;
9866 }
9867
9868 address generate_count_positives(address &count_positives_long) {
9869 StubId stub_id = StubId::stubgen_count_positives_id;
9870 GrowableArray<address> entries;
9871 int entry_count = StubInfo::entry_count(stub_id);
9872 // We have an extra entry for count_positives_long.
9873 assert(entry_count == 2, "sanity check");
9874 address start = load_archive_data(stub_id, &entries);
9875 if (start != nullptr) {
9876 assert(entries.length() == 1,
9877 "unexpected extra entry count %d", entries.length());
9878 count_positives_long = entries.at(0);
9879 return start;
9880 }
9881 const u1 large_loop_size = 64;
9882 const uint64_t UPPER_BIT_MASK=0x8080808080808080;
9883 int dcache_line = VM_Version::dcache_line_size();
9884
9885 Register ary1 = r1, len = r2, result = r0;
9886
9887 __ align(CodeEntryAlignment);
9888 StubCodeMark mark(this, stub_id);
9889
9890 address entry = __ pc();
9891
9892 __ enter();
9893 // precondition: a copy of len is already in result
9894 // __ mov(result, len);
9895
9896 Label RET_ADJUST, RET_ADJUST_16, RET_ADJUST_LONG, RET_NO_POP, RET_LEN, ALIGNED, LOOP16, CHECK_16,
9897 LARGE_LOOP, POST_LOOP16, LEN_OVER_15, LEN_OVER_8, POST_LOOP16_LOAD_TAIL;
9898
9899 __ cmp(len, (u1)15);
9900 __ br(Assembler::GT, LEN_OVER_15);
9901 // The only case when execution falls into this code is when pointer is near
9902 // the end of memory page and we have to avoid reading next page
9903 __ add(ary1, ary1, len);
9904 __ subs(len, len, 8);
9905 __ br(Assembler::GT, LEN_OVER_8);
9906 __ ldr(rscratch2, Address(ary1, -8));
9907 __ sub(rscratch1, zr, len, __ LSL, 3); // LSL 3 is to get bits from bytes.
9908 __ lsrv(rscratch2, rscratch2, rscratch1);
9909 __ tst(rscratch2, UPPER_BIT_MASK);
9910 __ csel(result, zr, result, Assembler::NE);
9911 __ leave();
9912 __ ret(lr);
9913 __ bind(LEN_OVER_8);
9914 __ ldp(rscratch1, rscratch2, Address(ary1, -16));
9915 __ sub(len, len, 8); // no data dep., then sub can be executed while loading
9916 __ tst(rscratch2, UPPER_BIT_MASK);
9917 __ br(Assembler::NE, RET_NO_POP);
9918 __ sub(rscratch2, zr, len, __ LSL, 3); // LSL 3 is to get bits from bytes
9919 __ lsrv(rscratch1, rscratch1, rscratch2);
9920 __ tst(rscratch1, UPPER_BIT_MASK);
9921 __ bind(RET_NO_POP);
9922 __ csel(result, zr, result, Assembler::NE);
9923 __ leave();
9924 __ ret(lr);
9925
9926 Register tmp1 = r3, tmp2 = r4, tmp3 = r5, tmp4 = r6, tmp5 = r7, tmp6 = r10;
9927 const RegSet spilled_regs = RegSet::range(tmp1, tmp5) + tmp6;
9928
9929 count_positives_long = __ pc(); // 2nd entry point
9930 entries.append(count_positives_long);
9931
9932 __ enter();
9933
9934 __ bind(LEN_OVER_15);
9935 __ push(spilled_regs, sp);
9936 __ andr(rscratch2, ary1, 15); // check pointer for 16-byte alignment
9937 __ cbz(rscratch2, ALIGNED);
9938 __ ldp(tmp6, tmp1, Address(ary1));
9939 __ mov(tmp5, 16);
9940 __ sub(rscratch1, tmp5, rscratch2); // amount of bytes until aligned address
9941 __ add(ary1, ary1, rscratch1);
9942 __ orr(tmp6, tmp6, tmp1);
9943 __ tst(tmp6, UPPER_BIT_MASK);
9944 __ br(Assembler::NE, RET_ADJUST);
9945 __ sub(len, len, rscratch1);
9946
9947 __ bind(ALIGNED);
9948 __ cmp(len, large_loop_size);
9949 __ br(Assembler::LT, CHECK_16);
9950 // Perform 16-byte load as early return in pre-loop to handle situation
9951 // when initially aligned large array has negative values at starting bytes,
9952 // so LARGE_LOOP would do 4 reads instead of 1 (in worst case), which is
9953 // slower. Cases with negative bytes further ahead won't be affected that
9954 // much. In fact, it'll be faster due to early loads, less instructions and
9955 // less branches in LARGE_LOOP.
9956 __ ldp(tmp6, tmp1, Address(__ post(ary1, 16)));
9957 __ sub(len, len, 16);
9958 __ orr(tmp6, tmp6, tmp1);
9959 __ tst(tmp6, UPPER_BIT_MASK);
9960 __ br(Assembler::NE, RET_ADJUST_16);
9961 __ cmp(len, large_loop_size);
9962 __ br(Assembler::LT, CHECK_16);
9963
9964 if (SoftwarePrefetchHintDistance >= 0
9965 && SoftwarePrefetchHintDistance >= dcache_line) {
9966 // initial prefetch
9967 __ prfm(Address(ary1, SoftwarePrefetchHintDistance - dcache_line));
9968 }
9969 __ bind(LARGE_LOOP);
9970 if (SoftwarePrefetchHintDistance >= 0) {
9971 __ prfm(Address(ary1, SoftwarePrefetchHintDistance));
9972 }
9973 // Issue load instructions first, since it can save few CPU/MEM cycles, also
9974 // instead of 4 triples of "orr(...), addr(...);cbnz(...);" (for each ldp)
9975 // better generate 7 * orr(...) + 1 andr(...) + 1 cbnz(...) which saves 3
9976 // instructions per cycle and have less branches, but this approach disables
9977 // early return, thus, all 64 bytes are loaded and checked every time.
9978 __ ldp(tmp2, tmp3, Address(ary1));
9979 __ ldp(tmp4, tmp5, Address(ary1, 16));
9980 __ ldp(rscratch1, rscratch2, Address(ary1, 32));
9981 __ ldp(tmp6, tmp1, Address(ary1, 48));
9982 __ add(ary1, ary1, large_loop_size);
9983 __ sub(len, len, large_loop_size);
9984 __ orr(tmp2, tmp2, tmp3);
9985 __ orr(tmp4, tmp4, tmp5);
9986 __ orr(rscratch1, rscratch1, rscratch2);
9987 __ orr(tmp6, tmp6, tmp1);
9988 __ orr(tmp2, tmp2, tmp4);
9989 __ orr(rscratch1, rscratch1, tmp6);
9990 __ orr(tmp2, tmp2, rscratch1);
9991 __ tst(tmp2, UPPER_BIT_MASK);
9992 __ br(Assembler::NE, RET_ADJUST_LONG);
9993 __ cmp(len, large_loop_size);
9994 __ br(Assembler::GE, LARGE_LOOP);
9995
9996 __ bind(CHECK_16); // small 16-byte load pre-loop
9997 __ cmp(len, (u1)16);
9998 __ br(Assembler::LT, POST_LOOP16);
9999
10000 __ bind(LOOP16); // small 16-byte load loop
10001 __ ldp(tmp2, tmp3, Address(__ post(ary1, 16)));
10002 __ sub(len, len, 16);
10003 __ orr(tmp2, tmp2, tmp3);
10004 __ tst(tmp2, UPPER_BIT_MASK);
10005 __ br(Assembler::NE, RET_ADJUST_16);
10006 __ cmp(len, (u1)16);
10007 __ br(Assembler::GE, LOOP16); // 16-byte load loop end
10008
10009 __ bind(POST_LOOP16); // 16-byte aligned, so we can read unconditionally
10010 __ cmp(len, (u1)8);
10011 __ br(Assembler::LE, POST_LOOP16_LOAD_TAIL);
10012 __ ldr(tmp3, Address(__ post(ary1, 8)));
10013 __ tst(tmp3, UPPER_BIT_MASK);
10014 __ br(Assembler::NE, RET_ADJUST);
10015 __ sub(len, len, 8);
10016
10017 __ bind(POST_LOOP16_LOAD_TAIL);
10018 __ cbz(len, RET_LEN); // Can't shift left by 64 when len==0
10019 __ ldr(tmp1, Address(ary1));
10020 __ mov(tmp2, 64);
10021 __ sub(tmp4, tmp2, len, __ LSL, 3);
10022 __ lslv(tmp1, tmp1, tmp4);
10023 __ tst(tmp1, UPPER_BIT_MASK);
10024 __ br(Assembler::NE, RET_ADJUST);
10025 // Fallthrough
10026
10027 __ bind(RET_LEN);
10028 __ pop(spilled_regs, sp);
10029 __ leave();
10030 __ ret(lr);
10031
10032 // difference result - len is the count of guaranteed to be
10033 // positive bytes
10034
10035 __ bind(RET_ADJUST_LONG);
10036 __ add(len, len, (u1)(large_loop_size - 16));
10037 __ bind(RET_ADJUST_16);
10038 __ add(len, len, 16);
10039 __ bind(RET_ADJUST);
10040 __ pop(spilled_regs, sp);
10041 __ leave();
10042 __ sub(result, result, len);
10043 __ ret(lr);
10044
10045 // record the stub entry and end plus the extra entry
10046 store_archive_data(stub_id, entry, __ pc(), &entries);
10047
10048 return entry;
10049 }
10050
10051 void generate_large_array_equals_loop_nonsimd(int loopThreshold,
10052 bool usePrefetch, Label &NOT_EQUAL) {
10053 Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
10054 tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11,
10055 tmp7 = r12, tmp8 = r13;
10056 Label LOOP;
10057
10058 __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize)));
10059 __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize)));
10060 __ bind(LOOP);
10061 if (usePrefetch) {
10062 __ prfm(Address(a1, SoftwarePrefetchHintDistance));
10063 __ prfm(Address(a2, SoftwarePrefetchHintDistance));
10064 }
10065 __ ldp(tmp5, tmp7, Address(__ post(a1, 2 * wordSize)));
10066 __ eor(tmp1, tmp1, tmp2);
10067 __ eor(tmp3, tmp3, tmp4);
10068 __ ldp(tmp6, tmp8, Address(__ post(a2, 2 * wordSize)));
10069 __ orr(tmp1, tmp1, tmp3);
10070 __ cbnz(tmp1, NOT_EQUAL);
10071 __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize)));
10072 __ eor(tmp5, tmp5, tmp6);
10073 __ eor(tmp7, tmp7, tmp8);
10074 __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize)));
10075 __ orr(tmp5, tmp5, tmp7);
10076 __ cbnz(tmp5, NOT_EQUAL);
10077 __ ldp(tmp5, tmp7, Address(__ post(a1, 2 * wordSize)));
10078 __ eor(tmp1, tmp1, tmp2);
10079 __ eor(tmp3, tmp3, tmp4);
10080 __ ldp(tmp6, tmp8, Address(__ post(a2, 2 * wordSize)));
10081 __ orr(tmp1, tmp1, tmp3);
10082 __ cbnz(tmp1, NOT_EQUAL);
10083 __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize)));
10084 __ eor(tmp5, tmp5, tmp6);
10085 __ sub(cnt1, cnt1, 8 * wordSize);
10086 __ eor(tmp7, tmp7, tmp8);
10087 __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize)));
10088 // tmp6 is not used. MacroAssembler::subs is used here (rather than
10089 // cmp) because subs allows an unlimited range of immediate operand.
10090 __ subs(tmp6, cnt1, loopThreshold);
10091 __ orr(tmp5, tmp5, tmp7);
10092 __ cbnz(tmp5, NOT_EQUAL);
10093 __ br(__ GE, LOOP);
10094 // post-loop
10095 __ eor(tmp1, tmp1, tmp2);
10096 __ eor(tmp3, tmp3, tmp4);
10097 __ orr(tmp1, tmp1, tmp3);
10098 __ sub(cnt1, cnt1, 2 * wordSize);
10099 __ cbnz(tmp1, NOT_EQUAL);
10100 }
10101
10102 void generate_large_array_equals_loop_simd(int loopThreshold,
10103 bool usePrefetch, Label &NOT_EQUAL) {
10104 Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
10105 tmp2 = rscratch2;
10106 Label LOOP;
10107
10108 __ bind(LOOP);
10109 if (usePrefetch) {
10110 __ prfm(Address(a1, SoftwarePrefetchHintDistance));
10111 __ prfm(Address(a2, SoftwarePrefetchHintDistance));
10112 }
10113 __ ld1(v0, v1, v2, v3, __ T2D, Address(__ post(a1, 4 * 2 * wordSize)));
10114 __ sub(cnt1, cnt1, 8 * wordSize);
10115 __ ld1(v4, v5, v6, v7, __ T2D, Address(__ post(a2, 4 * 2 * wordSize)));
10116 __ subs(tmp1, cnt1, loopThreshold);
10117 __ eor(v0, __ T16B, v0, v4);
10118 __ eor(v1, __ T16B, v1, v5);
10119 __ eor(v2, __ T16B, v2, v6);
10120 __ eor(v3, __ T16B, v3, v7);
10121 __ orr(v0, __ T16B, v0, v1);
10122 __ orr(v1, __ T16B, v2, v3);
10123 __ orr(v0, __ T16B, v0, v1);
10124 __ umov(tmp1, v0, __ D, 0);
10125 __ umov(tmp2, v0, __ D, 1);
10126 __ orr(tmp1, tmp1, tmp2);
10127 __ cbnz(tmp1, NOT_EQUAL);
10128 __ br(__ GE, LOOP);
10129 }
10130
10131 // a1 = r1 - array1 address
10132 // a2 = r2 - array2 address
10133 // result = r0 - return value. Already contains "false"
10134 // cnt1 = r10 - amount of elements left to check, reduced by wordSize
10135 // r3-r5 are reserved temporary registers
10136 // Clobbers: v0-v7 when UseSIMDForArrayEquals, rscratch1, rscratch2
10137 address generate_large_array_equals() {
10138 StubId stub_id = StubId::stubgen_large_array_equals_id;
10139 int entry_count = StubInfo::entry_count(stub_id);
10140 assert(entry_count == 1, "sanity check");
10141 address start = load_archive_data(stub_id);
10142 if (start != nullptr) {
10143 return start;
10144 }
10145 Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
10146 tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11,
10147 tmp7 = r12, tmp8 = r13;
10148 Label TAIL, NOT_EQUAL, EQUAL, NOT_EQUAL_NO_POP, NO_PREFETCH_LARGE_LOOP,
10149 SMALL_LOOP, POST_LOOP;
10150 const int PRE_LOOP_SIZE = UseSIMDForArrayEquals ? 0 : 16;
10151 // calculate if at least 32 prefetched bytes are used
10152 int prefetchLoopThreshold = SoftwarePrefetchHintDistance + 32;
10153 int nonPrefetchLoopThreshold = (64 + PRE_LOOP_SIZE);
10154 RegSet spilled_regs = RegSet::range(tmp6, tmp8);
10155 assert_different_registers(a1, a2, result, cnt1, tmp1, tmp2, tmp3, tmp4,
10156 tmp5, tmp6, tmp7, tmp8);
10157
10158 __ align(CodeEntryAlignment);
10159
10160 StubCodeMark mark(this, stub_id);
10161
10162 address entry = __ pc();
10163 __ enter();
10164 __ sub(cnt1, cnt1, wordSize); // first 8 bytes were loaded outside of stub
10165 // also advance pointers to use post-increment instead of pre-increment
10166 __ add(a1, a1, wordSize);
10167 __ add(a2, a2, wordSize);
10168 if (AvoidUnalignedAccesses) {
10169 // both implementations (SIMD/nonSIMD) are using relatively large load
10170 // instructions (ld1/ldp), which has huge penalty (up to x2 exec time)
10171 // on some CPUs in case of address is not at least 16-byte aligned.
10172 // Arrays are 8-byte aligned currently, so, we can make additional 8-byte
10173 // load if needed at least for 1st address and make if 16-byte aligned.
10174 Label ALIGNED16;
10175 __ tbz(a1, 3, ALIGNED16);
10176 __ ldr(tmp1, Address(__ post(a1, wordSize)));
10177 __ ldr(tmp2, Address(__ post(a2, wordSize)));
10178 __ sub(cnt1, cnt1, wordSize);
10179 __ eor(tmp1, tmp1, tmp2);
10180 __ cbnz(tmp1, NOT_EQUAL_NO_POP);
10181 __ bind(ALIGNED16);
10182 }
10183 if (UseSIMDForArrayEquals) {
10184 if (SoftwarePrefetchHintDistance >= 0) {
10185 __ subs(tmp1, cnt1, prefetchLoopThreshold);
10186 __ br(__ LE, NO_PREFETCH_LARGE_LOOP);
10187 generate_large_array_equals_loop_simd(prefetchLoopThreshold,
10188 /* prfm = */ true, NOT_EQUAL);
10189 __ subs(zr, cnt1, nonPrefetchLoopThreshold);
10190 __ br(__ LT, TAIL);
10191 }
10192 __ bind(NO_PREFETCH_LARGE_LOOP);
10193 generate_large_array_equals_loop_simd(nonPrefetchLoopThreshold,
10194 /* prfm = */ false, NOT_EQUAL);
10195 } else {
10196 __ push(spilled_regs, sp);
10197 if (SoftwarePrefetchHintDistance >= 0) {
10198 __ subs(tmp1, cnt1, prefetchLoopThreshold);
10199 __ br(__ LE, NO_PREFETCH_LARGE_LOOP);
10200 generate_large_array_equals_loop_nonsimd(prefetchLoopThreshold,
10201 /* prfm = */ true, NOT_EQUAL);
10202 __ subs(zr, cnt1, nonPrefetchLoopThreshold);
10203 __ br(__ LT, TAIL);
10204 }
10205 __ bind(NO_PREFETCH_LARGE_LOOP);
10206 generate_large_array_equals_loop_nonsimd(nonPrefetchLoopThreshold,
10207 /* prfm = */ false, NOT_EQUAL);
10208 }
10209 __ bind(TAIL);
10210 __ cbz(cnt1, EQUAL);
10211 __ subs(cnt1, cnt1, wordSize);
10212 __ br(__ LE, POST_LOOP);
10213 __ bind(SMALL_LOOP);
10214 __ ldr(tmp1, Address(__ post(a1, wordSize)));
10215 __ ldr(tmp2, Address(__ post(a2, wordSize)));
10216 __ subs(cnt1, cnt1, wordSize);
10217 __ eor(tmp1, tmp1, tmp2);
10218 __ cbnz(tmp1, NOT_EQUAL);
10219 __ br(__ GT, SMALL_LOOP);
10220 __ bind(POST_LOOP);
10221 __ ldr(tmp1, Address(a1, cnt1));
10222 __ ldr(tmp2, Address(a2, cnt1));
10223 __ eor(tmp1, tmp1, tmp2);
10224 __ cbnz(tmp1, NOT_EQUAL);
10225 __ bind(EQUAL);
10226 __ mov(result, true);
10227 __ bind(NOT_EQUAL);
10228 if (!UseSIMDForArrayEquals) {
10229 __ pop(spilled_regs, sp);
10230 }
10231 __ bind(NOT_EQUAL_NO_POP);
10232 __ leave();
10233 __ ret(lr);
10234
10235 // record the stub entry and end
10236 store_archive_data(stub_id, entry, __ pc());
10237
10238 return entry;
10239 }
10240
10241 // result = r0 - return value. Contains initial hashcode value on entry.
10242 // ary = r1 - array address
10243 // cnt = r2 - elements count
10244 // Clobbers: v0-v13, rscratch1, rscratch2
10245 address generate_large_arrays_hashcode(BasicType eltype) {
10246 StubId stub_id;
10247 switch (eltype) {
10248 case T_BOOLEAN:
10249 stub_id = StubId::stubgen_large_arrays_hashcode_boolean_id;
10250 break;
10251 case T_BYTE:
10252 stub_id = StubId::stubgen_large_arrays_hashcode_byte_id;
10253 break;
10254 case T_CHAR:
10255 stub_id = StubId::stubgen_large_arrays_hashcode_char_id;
10256 break;
10257 case T_SHORT:
10258 stub_id = StubId::stubgen_large_arrays_hashcode_short_id;
10259 break;
10260 case T_INT:
10261 stub_id = StubId::stubgen_large_arrays_hashcode_int_id;
10262 break;
10263 default:
10264 stub_id = StubId::NO_STUBID;
10265 ShouldNotReachHere();
10266 };
10267 int entry_count = StubInfo::entry_count(stub_id);
10268 assert(entry_count == 1, "sanity check");
10269 address start = load_archive_data(stub_id);
10270 if (start != nullptr) {
10271 return start;
10272 }
10273 const Register result = r0, ary = r1, cnt = r2;
10274 const FloatRegister vdata0 = v3, vdata1 = v2, vdata2 = v1, vdata3 = v0;
10275 const FloatRegister vmul0 = v4, vmul1 = v5, vmul2 = v6, vmul3 = v7;
10276 const FloatRegister vpow = v12; // powers of 31: <31^3, ..., 31^0>
10277 const FloatRegister vpowm = v13;
10278
10279 ARRAYS_HASHCODE_REGISTERS;
10280
10281 Label SMALL_LOOP, LARGE_LOOP_PREHEADER, LARGE_LOOP, TAIL, TAIL_SHORTCUT, BR_BASE;
10282
10283 unsigned int vf; // vectorization factor
10284 bool multiply_by_halves;
10285 Assembler::SIMD_Arrangement load_arrangement;
10286 switch (eltype) {
10287 case T_BOOLEAN:
10288 case T_BYTE:
10289 load_arrangement = Assembler::T8B;
10290 multiply_by_halves = true;
10291 vf = 8;
10292 break;
10293 case T_CHAR:
10294 case T_SHORT:
10295 load_arrangement = Assembler::T8H;
10296 multiply_by_halves = true;
10297 vf = 8;
10298 break;
10299 case T_INT:
10300 load_arrangement = Assembler::T4S;
10301 multiply_by_halves = false;
10302 vf = 4;
10303 break;
10304 default:
10305 ShouldNotReachHere();
10306 }
10307
10308 // Unroll factor
10309 const unsigned uf = 4;
10310
10311 // Effective vectorization factor
10312 const unsigned evf = vf * uf;
10313
10314 __ align(CodeEntryAlignment);
10315
10316 StubCodeMark mark(this, stub_id);
10317
10318 address entry = __ pc();
10319 __ enter();
10320
10321 // Put 0-3'th powers of 31 into a single SIMD register together. The register will be used in
10322 // the SMALL and LARGE LOOPS' epilogues. The initialization is hoisted here and the register's
10323 // value shouldn't change throughout both loops.
10324 __ movw(rscratch1, intpow(31U, 3));
10325 __ mov(vpow, Assembler::S, 0, rscratch1);
10326 __ movw(rscratch1, intpow(31U, 2));
10327 __ mov(vpow, Assembler::S, 1, rscratch1);
10328 __ movw(rscratch1, intpow(31U, 1));
10329 __ mov(vpow, Assembler::S, 2, rscratch1);
10330 __ movw(rscratch1, intpow(31U, 0));
10331 __ mov(vpow, Assembler::S, 3, rscratch1);
10332
10333 __ mov(vmul0, Assembler::T16B, 0);
10334 __ mov(vmul0, Assembler::S, 3, result);
10335
10336 __ andr(rscratch2, cnt, (uf - 1) * vf);
10337 __ cbz(rscratch2, LARGE_LOOP_PREHEADER);
10338
10339 __ movw(rscratch1, intpow(31U, multiply_by_halves ? vf / 2 : vf));
10340 __ mov(vpowm, Assembler::S, 0, rscratch1);
10341
10342 // SMALL LOOP
10343 __ bind(SMALL_LOOP);
10344
10345 __ ld1(vdata0, load_arrangement, Address(__ post(ary, vf * type2aelembytes(eltype))));
10346 __ mulvs(vmul0, Assembler::T4S, vmul0, vpowm, 0);
10347 __ subsw(rscratch2, rscratch2, vf);
10348
10349 if (load_arrangement == Assembler::T8B) {
10350 // Extend 8B to 8H to be able to use vector multiply
10351 // instructions
10352 assert(load_arrangement == Assembler::T8B, "expected to extend 8B to 8H");
10353 if (is_signed_subword_type(eltype)) {
10354 __ sxtl(vdata0, Assembler::T8H, vdata0, load_arrangement);
10355 } else {
10356 __ uxtl(vdata0, Assembler::T8H, vdata0, load_arrangement);
10357 }
10358 }
10359
10360 switch (load_arrangement) {
10361 case Assembler::T4S:
10362 __ addv(vmul0, load_arrangement, vmul0, vdata0);
10363 break;
10364 case Assembler::T8B:
10365 case Assembler::T8H:
10366 assert(is_subword_type(eltype), "subword type expected");
10367 if (is_signed_subword_type(eltype)) {
10368 __ saddwv(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T4H);
10369 } else {
10370 __ uaddwv(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T4H);
10371 }
10372 break;
10373 default:
10374 __ should_not_reach_here();
10375 }
10376
10377 // Process the upper half of a vector
10378 if (load_arrangement == Assembler::T8B || load_arrangement == Assembler::T8H) {
10379 __ mulvs(vmul0, Assembler::T4S, vmul0, vpowm, 0);
10380 if (is_signed_subword_type(eltype)) {
10381 __ saddwv2(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T8H);
10382 } else {
10383 __ uaddwv2(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T8H);
10384 }
10385 }
10386
10387 __ br(Assembler::HI, SMALL_LOOP);
10388
10389 // SMALL LOOP'S EPILOQUE
10390 __ lsr(rscratch2, cnt, exact_log2(evf));
10391 __ cbnz(rscratch2, LARGE_LOOP_PREHEADER);
10392
10393 __ mulv(vmul0, Assembler::T4S, vmul0, vpow);
10394 __ addv(vmul0, Assembler::T4S, vmul0);
10395 __ umov(result, vmul0, Assembler::S, 0);
10396
10397 // TAIL
10398 __ bind(TAIL);
10399
10400 // The andr performs cnt % vf. The subtract shifted by 3 offsets past vf - 1 - (cnt % vf) pairs
10401 // of load + madd insns i.e. it only executes cnt % vf load + madd pairs.
10402 assert(is_power_of_2(vf), "can't use this value to calculate the jump target PC");
10403 __ andr(rscratch2, cnt, vf - 1);
10404 __ bind(TAIL_SHORTCUT);
10405 __ adr(rscratch1, BR_BASE);
10406 // For Cortex-A53 offset is 4 because 2 nops are generated.
10407 __ sub(rscratch1, rscratch1, rscratch2, ext::uxtw, VM_Version::supports_a53mac() ? 4 : 3);
10408 __ movw(rscratch2, 0x1f);
10409 __ br(rscratch1);
10410
10411 for (size_t i = 0; i < vf - 1; ++i) {
10412 __ load(rscratch1, Address(__ post(ary, type2aelembytes(eltype))),
10413 eltype);
10414 __ maddw(result, result, rscratch2, rscratch1);
10415 // maddw generates an extra nop for Cortex-A53 (see maddw definition in macroAssembler).
10416 // Generate 2nd nop to have 4 instructions per iteration.
10417 if (VM_Version::supports_a53mac()) {
10418 __ nop();
10419 }
10420 }
10421 __ bind(BR_BASE);
10422
10423 __ leave();
10424 __ ret(lr);
10425
10426 // LARGE LOOP
10427 __ bind(LARGE_LOOP_PREHEADER);
10428
10429 __ lsr(rscratch2, cnt, exact_log2(evf));
10430
10431 if (multiply_by_halves) {
10432 // 31^4 - multiplier between lower and upper parts of a register
10433 __ movw(rscratch1, intpow(31U, vf / 2));
10434 __ mov(vpowm, Assembler::S, 1, rscratch1);
10435 // 31^28 - remainder of the iteraion multiplier, 28 = 32 - 4
10436 __ movw(rscratch1, intpow(31U, evf - vf / 2));
10437 __ mov(vpowm, Assembler::S, 0, rscratch1);
10438 } else {
10439 // 31^16
10440 __ movw(rscratch1, intpow(31U, evf));
10441 __ mov(vpowm, Assembler::S, 0, rscratch1);
10442 }
10443
10444 __ mov(vmul3, Assembler::T16B, 0);
10445 __ mov(vmul2, Assembler::T16B, 0);
10446 __ mov(vmul1, Assembler::T16B, 0);
10447
10448 __ bind(LARGE_LOOP);
10449
10450 __ mulvs(vmul3, Assembler::T4S, vmul3, vpowm, 0);
10451 __ mulvs(vmul2, Assembler::T4S, vmul2, vpowm, 0);
10452 __ mulvs(vmul1, Assembler::T4S, vmul1, vpowm, 0);
10453 __ mulvs(vmul0, Assembler::T4S, vmul0, vpowm, 0);
10454
10455 __ ld1(vdata3, vdata2, vdata1, vdata0, load_arrangement,
10456 Address(__ post(ary, evf * type2aelembytes(eltype))));
10457
10458 if (load_arrangement == Assembler::T8B) {
10459 // Extend 8B to 8H to be able to use vector multiply
10460 // instructions
10461 assert(load_arrangement == Assembler::T8B, "expected to extend 8B to 8H");
10462 if (is_signed_subword_type(eltype)) {
10463 __ sxtl(vdata3, Assembler::T8H, vdata3, load_arrangement);
10464 __ sxtl(vdata2, Assembler::T8H, vdata2, load_arrangement);
10465 __ sxtl(vdata1, Assembler::T8H, vdata1, load_arrangement);
10466 __ sxtl(vdata0, Assembler::T8H, vdata0, load_arrangement);
10467 } else {
10468 __ uxtl(vdata3, Assembler::T8H, vdata3, load_arrangement);
10469 __ uxtl(vdata2, Assembler::T8H, vdata2, load_arrangement);
10470 __ uxtl(vdata1, Assembler::T8H, vdata1, load_arrangement);
10471 __ uxtl(vdata0, Assembler::T8H, vdata0, load_arrangement);
10472 }
10473 }
10474
10475 switch (load_arrangement) {
10476 case Assembler::T4S:
10477 __ addv(vmul3, load_arrangement, vmul3, vdata3);
10478 __ addv(vmul2, load_arrangement, vmul2, vdata2);
10479 __ addv(vmul1, load_arrangement, vmul1, vdata1);
10480 __ addv(vmul0, load_arrangement, vmul0, vdata0);
10481 break;
10482 case Assembler::T8B:
10483 case Assembler::T8H:
10484 assert(is_subword_type(eltype), "subword type expected");
10485 if (is_signed_subword_type(eltype)) {
10486 __ saddwv(vmul3, vmul3, Assembler::T4S, vdata3, Assembler::T4H);
10487 __ saddwv(vmul2, vmul2, Assembler::T4S, vdata2, Assembler::T4H);
10488 __ saddwv(vmul1, vmul1, Assembler::T4S, vdata1, Assembler::T4H);
10489 __ saddwv(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T4H);
10490 } else {
10491 __ uaddwv(vmul3, vmul3, Assembler::T4S, vdata3, Assembler::T4H);
10492 __ uaddwv(vmul2, vmul2, Assembler::T4S, vdata2, Assembler::T4H);
10493 __ uaddwv(vmul1, vmul1, Assembler::T4S, vdata1, Assembler::T4H);
10494 __ uaddwv(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T4H);
10495 }
10496 break;
10497 default:
10498 __ should_not_reach_here();
10499 }
10500
10501 // Process the upper half of a vector
10502 if (load_arrangement == Assembler::T8B || load_arrangement == Assembler::T8H) {
10503 __ mulvs(vmul3, Assembler::T4S, vmul3, vpowm, 1);
10504 __ mulvs(vmul2, Assembler::T4S, vmul2, vpowm, 1);
10505 __ mulvs(vmul1, Assembler::T4S, vmul1, vpowm, 1);
10506 __ mulvs(vmul0, Assembler::T4S, vmul0, vpowm, 1);
10507 if (is_signed_subword_type(eltype)) {
10508 __ saddwv2(vmul3, vmul3, Assembler::T4S, vdata3, Assembler::T8H);
10509 __ saddwv2(vmul2, vmul2, Assembler::T4S, vdata2, Assembler::T8H);
10510 __ saddwv2(vmul1, vmul1, Assembler::T4S, vdata1, Assembler::T8H);
10511 __ saddwv2(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T8H);
10512 } else {
10513 __ uaddwv2(vmul3, vmul3, Assembler::T4S, vdata3, Assembler::T8H);
10514 __ uaddwv2(vmul2, vmul2, Assembler::T4S, vdata2, Assembler::T8H);
10515 __ uaddwv2(vmul1, vmul1, Assembler::T4S, vdata1, Assembler::T8H);
10516 __ uaddwv2(vmul0, vmul0, Assembler::T4S, vdata0, Assembler::T8H);
10517 }
10518 }
10519
10520 __ subsw(rscratch2, rscratch2, 1);
10521 __ br(Assembler::HI, LARGE_LOOP);
10522
10523 __ mulv(vmul3, Assembler::T4S, vmul3, vpow);
10524 __ addv(vmul3, Assembler::T4S, vmul3);
10525 __ umov(result, vmul3, Assembler::S, 0);
10526
10527 __ mov(rscratch2, intpow(31U, vf));
10528
10529 __ mulv(vmul2, Assembler::T4S, vmul2, vpow);
10530 __ addv(vmul2, Assembler::T4S, vmul2);
10531 __ umov(rscratch1, vmul2, Assembler::S, 0);
10532 __ maddw(result, result, rscratch2, rscratch1);
10533
10534 __ mulv(vmul1, Assembler::T4S, vmul1, vpow);
10535 __ addv(vmul1, Assembler::T4S, vmul1);
10536 __ umov(rscratch1, vmul1, Assembler::S, 0);
10537 __ maddw(result, result, rscratch2, rscratch1);
10538
10539 __ mulv(vmul0, Assembler::T4S, vmul0, vpow);
10540 __ addv(vmul0, Assembler::T4S, vmul0);
10541 __ umov(rscratch1, vmul0, Assembler::S, 0);
10542 __ maddw(result, result, rscratch2, rscratch1);
10543
10544 __ andr(rscratch2, cnt, vf - 1);
10545 __ cbnz(rscratch2, TAIL_SHORTCUT);
10546
10547 __ leave();
10548 __ ret(lr);
10549
10550 // record the stub entry and end
10551 store_archive_data(stub_id, entry, __ pc());
10552
10553 return entry;
10554 }
10555
10556 address generate_dsin_dcos(bool isCos) {
10557 StubId stub_id = (isCos ? StubId::stubgen_dcos_id : StubId::stubgen_dsin_id);
10558 int entry_count = StubInfo::entry_count(stub_id);
10559 assert(entry_count == 1, "sanity check");
10560 address start = load_archive_data(stub_id);
10561 if (start != nullptr) {
10562 return start;
10563 }
10564 __ align(CodeEntryAlignment);
10565 StubCodeMark mark(this, stub_id);
10566 start = __ pc();
10567 __ generate_dsin_dcos(isCos, (address)StubRoutines::aarch64::_npio2_hw,
10568 (address)StubRoutines::aarch64::_two_over_pi,
10569 (address)StubRoutines::aarch64::_pio2,
10570 (address)StubRoutines::aarch64::_dsin_coef,
10571 (address)StubRoutines::aarch64::_dcos_coef);
10572
10573 // record the stub entry and end
10574 store_archive_data(stub_id, start, __ pc());
10575
10576 return start;
10577 }
10578
10579 // code for comparing 16 characters of strings with Latin1 and Utf16 encoding
10580 void compare_string_16_x_LU(Register tmpL, Register tmpU, Label &DIFF1,
10581 Label &DIFF2) {
10582 Register cnt1 = r2, tmp2 = r11, tmp3 = r12;
10583 FloatRegister vtmp = v1, vtmpZ = v0, vtmp3 = v2;
10584
10585 __ ldrq(vtmp, Address(__ post(tmp2, 16)));
10586 __ ldr(tmpU, Address(__ post(cnt1, 8)));
10587 __ zip1(vtmp3, __ T16B, vtmp, vtmpZ);
10588 // now we have 32 bytes of characters (converted to U) in vtmp:vtmp3
10589
10590 __ fmovd(tmpL, vtmp3);
10591 __ eor(rscratch2, tmp3, tmpL);
10592 __ cbnz(rscratch2, DIFF2);
10593
10594 __ ldr(tmp3, Address(__ post(cnt1, 8)));
10595 __ umov(tmpL, vtmp3, __ D, 1);
10596 __ eor(rscratch2, tmpU, tmpL);
10597 __ cbnz(rscratch2, DIFF1);
10598
10599 __ zip2(vtmp, __ T16B, vtmp, vtmpZ);
10600 __ ldr(tmpU, Address(__ post(cnt1, 8)));
10601 __ fmovd(tmpL, vtmp);
10602 __ eor(rscratch2, tmp3, tmpL);
10603 __ cbnz(rscratch2, DIFF2);
10604
10605 __ ldr(tmp3, Address(__ post(cnt1, 8)));
10606 __ umov(tmpL, vtmp, __ D, 1);
10607 __ eor(rscratch2, tmpU, tmpL);
10608 __ cbnz(rscratch2, DIFF1);
10609 }
10610
10611 // r0 = result
10612 // r1 = str1
10613 // r2 = cnt1
10614 // r3 = str2
10615 // r4 = cnt2
10616 // r10 = tmp1
10617 // r11 = tmp2
10618 address generate_compare_long_string_different_encoding(bool isLU) {
10619 StubId stub_id = (isLU ? StubId::stubgen_compare_long_string_LU_id : StubId::stubgen_compare_long_string_UL_id);
10620 int entry_count = StubInfo::entry_count(stub_id);
10621 assert(entry_count == 1, "sanity check");
10622 address start = load_archive_data(stub_id);
10623 if (start != nullptr) {
10624 return start;
10625 }
10626 __ align(CodeEntryAlignment);
10627 StubCodeMark mark(this, stub_id);
10628 address entry = __ pc();
10629 Label SMALL_LOOP, TAIL, TAIL_LOAD_16, LOAD_LAST, DIFF1, DIFF2,
10630 DONE, CALCULATE_DIFFERENCE, LARGE_LOOP_PREFETCH, NO_PREFETCH,
10631 LARGE_LOOP_PREFETCH_REPEAT1, LARGE_LOOP_PREFETCH_REPEAT2;
10632 Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
10633 tmp1 = r10, tmp2 = r11, tmp3 = r12, tmp4 = r14;
10634 FloatRegister vtmpZ = v0, vtmp = v1, vtmp3 = v2;
10635 RegSet spilled_regs = RegSet::of(tmp3, tmp4);
10636
10637 int prefetchLoopExitCondition = MAX2(64, SoftwarePrefetchHintDistance/2);
10638
10639 __ eor(vtmpZ, __ T16B, vtmpZ, vtmpZ);
10640 // cnt2 == amount of characters left to compare
10641 // Check already loaded first 4 symbols(vtmp and tmp2(LU)/tmp1(UL))
10642 __ zip1(vtmp, __ T8B, vtmp, vtmpZ);
10643 __ add(str1, str1, isLU ? wordSize/2 : wordSize);
10644 __ add(str2, str2, isLU ? wordSize : wordSize/2);
10645 __ fmovd(isLU ? tmp1 : tmp2, vtmp);
10646 __ subw(cnt2, cnt2, 8); // Already loaded 4 symbols. Last 4 is special case.
10647 __ eor(rscratch2, tmp1, tmp2);
10648 __ mov(rscratch1, tmp2);
10649 __ cbnz(rscratch2, CALCULATE_DIFFERENCE);
10650 Register tmpU = isLU ? rscratch1 : tmp1, // where to keep U for comparison
10651 tmpL = isLU ? tmp1 : rscratch1; // where to keep L for comparison
10652 __ push(spilled_regs, sp);
10653 __ mov(tmp2, isLU ? str1 : str2); // init the pointer to L next load
10654 __ mov(cnt1, isLU ? str2 : str1); // init the pointer to U next load
10655
10656 __ ldr(tmp3, Address(__ post(cnt1, 8)));
10657
10658 if (SoftwarePrefetchHintDistance >= 0) {
10659 __ subs(rscratch2, cnt2, prefetchLoopExitCondition);
10660 __ br(__ LT, NO_PREFETCH);
10661 __ bind(LARGE_LOOP_PREFETCH);
10662 __ prfm(Address(tmp2, SoftwarePrefetchHintDistance));
10663 __ mov(tmp4, 2);
10664 __ prfm(Address(cnt1, SoftwarePrefetchHintDistance));
10665 __ bind(LARGE_LOOP_PREFETCH_REPEAT1);
10666 compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2);
10667 __ subs(tmp4, tmp4, 1);
10668 __ br(__ GT, LARGE_LOOP_PREFETCH_REPEAT1);
10669 __ prfm(Address(cnt1, SoftwarePrefetchHintDistance));
10670 __ mov(tmp4, 2);
10671 __ bind(LARGE_LOOP_PREFETCH_REPEAT2);
10672 compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2);
10673 __ subs(tmp4, tmp4, 1);
10674 __ br(__ GT, LARGE_LOOP_PREFETCH_REPEAT2);
10675 __ sub(cnt2, cnt2, 64);
10676 __ subs(rscratch2, cnt2, prefetchLoopExitCondition);
10677 __ br(__ GE, LARGE_LOOP_PREFETCH);
10678 }
10679 __ cbz(cnt2, LOAD_LAST); // no characters left except last load
10680 __ bind(NO_PREFETCH);
10681 __ subs(cnt2, cnt2, 16);
10682 __ br(__ LT, TAIL);
10683 __ align(OptoLoopAlignment);
10684 __ bind(SMALL_LOOP); // smaller loop
10685 __ subs(cnt2, cnt2, 16);
10686 compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2);
10687 __ br(__ GE, SMALL_LOOP);
10688 __ cmn(cnt2, (u1)16);
10689 __ br(__ EQ, LOAD_LAST);
10690 __ bind(TAIL); // 1..15 characters left until last load (last 4 characters)
10691 __ add(cnt1, cnt1, cnt2, __ LSL, 1); // Address of 32 bytes before last 4 characters in UTF-16 string
10692 __ add(tmp2, tmp2, cnt2); // Address of 16 bytes before last 4 characters in Latin1 string
10693 __ ldr(tmp3, Address(cnt1, -8));
10694 compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2); // last 16 characters before last load
10695 __ b(LOAD_LAST);
10696 __ bind(DIFF2);
10697 __ mov(tmpU, tmp3);
10698 __ bind(DIFF1);
10699 __ pop(spilled_regs, sp);
10700 __ b(CALCULATE_DIFFERENCE);
10701 __ bind(LOAD_LAST);
10702 // Last 4 UTF-16 characters are already pre-loaded into tmp3 by compare_string_16_x_LU.
10703 // No need to load it again
10704 __ mov(tmpU, tmp3);
10705 __ pop(spilled_regs, sp);
10706
10707 // tmp2 points to the address of the last 4 Latin1 characters right now
10708 __ ldrs(vtmp, Address(tmp2));
10709 __ zip1(vtmp, __ T8B, vtmp, vtmpZ);
10710 __ fmovd(tmpL, vtmp);
10711
10712 __ eor(rscratch2, tmpU, tmpL);
10713 __ cbz(rscratch2, DONE);
10714
10715 // Find the first different characters in the longwords and
10716 // compute their difference.
10717 __ bind(CALCULATE_DIFFERENCE);
10718 __ rev(rscratch2, rscratch2);
10719 __ clz(rscratch2, rscratch2);
10720 __ andr(rscratch2, rscratch2, -16);
10721 __ lsrv(tmp1, tmp1, rscratch2);
10722 __ uxthw(tmp1, tmp1);
10723 __ lsrv(rscratch1, rscratch1, rscratch2);
10724 __ uxthw(rscratch1, rscratch1);
10725 __ subw(result, tmp1, rscratch1);
10726 __ bind(DONE);
10727 __ ret(lr);
10728
10729 // record the stub entry and end
10730 store_archive_data(stub_id, entry, __ pc());
10731
10732 return entry;
10733 }
10734
10735 // r0 = input (float16)
10736 // v0 = result (float)
10737 // v1 = temporary float register
10738 address generate_float16ToFloat() {
10739 StubId stub_id = StubId::stubgen_hf2f_id;
10740 int entry_count = StubInfo::entry_count(stub_id);
10741 assert(entry_count == 1, "sanity check");
10742 address start = load_archive_data(stub_id);
10743 if (start != nullptr) {
10744 return start;
10745 }
10746 __ align(CodeEntryAlignment);
10747 StubCodeMark mark(this, stub_id);
10748 address entry = __ pc();
10749 BLOCK_COMMENT("Entry:");
10750 __ flt16_to_flt(v0, r0, v1);
10751 __ ret(lr);
10752
10753 // record the stub entry and end
10754 store_archive_data(stub_id, entry, __ pc());
10755
10756 return entry;
10757 }
10758
10759 // v0 = input (float)
10760 // r0 = result (float16)
10761 // v1 = temporary float register
10762 address generate_floatToFloat16() {
10763 StubId stub_id = StubId::stubgen_f2hf_id;
10764 int entry_count = StubInfo::entry_count(stub_id);
10765 assert(entry_count == 1, "sanity check");
10766 address start = load_archive_data(stub_id);
10767 if (start != nullptr) {
10768 return start;
10769 }
10770 __ align(CodeEntryAlignment);
10771 StubCodeMark mark(this, stub_id);
10772 address entry = __ pc();
10773 BLOCK_COMMENT("Entry:");
10774 __ flt_to_flt16(r0, v0, v1);
10775 __ ret(lr);
10776
10777 // record the stub entry and end
10778 store_archive_data(stub_id, entry, __ pc());
10779
10780 return entry;
10781 }
10782
10783 address generate_method_entry_barrier() {
10784 StubId stub_id = StubId::stubgen_method_entry_barrier_id;
10785 int entry_count = StubInfo::entry_count(stub_id);
10786 assert(entry_count == 1, "sanity check");
10787 address start = load_archive_data(stub_id);
10788 if (start != nullptr) {
10789 return start;
10790 }
10791 __ align(CodeEntryAlignment);
10792 StubCodeMark mark(this, stub_id);
10793
10794 Label deoptimize_label;
10795
10796 start = __ pc();
10797
10798 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
10799
10800 if (bs_asm->nmethod_patching_type() == NMethodPatchingType::conc_instruction_and_data_patch) {
10801 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
10802 // We can get here despite the nmethod being good, if we have not
10803 // yet applied our cross modification fence (or data fence).
10804 Address thread_epoch_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()) + 4);
10805 __ lea(rscratch2, ExternalAddress(bs_asm->patching_epoch_addr()));
10806 __ ldrw(rscratch2, rscratch2);
10807 __ strw(rscratch2, thread_epoch_addr);
10808 __ isb();
10809 __ membar(__ LoadLoad);
10810 }
10811
10812 __ set_last_Java_frame(sp, rfp, lr, rscratch1);
10813
10814 __ enter();
10815 __ add(rscratch2, sp, wordSize); // rscratch2 points to the saved lr
10816
10817 __ sub(sp, sp, 4 * wordSize); // four words for the returned {sp, fp, lr, pc}
10818
10819 __ push_call_clobbered_registers();
10820
10821 __ mov(c_rarg0, rscratch2);
10822 __ call_VM_leaf
10823 (CAST_FROM_FN_PTR
10824 (address, BarrierSetNMethod::nmethod_stub_entry_barrier), 1);
10825
10826 __ reset_last_Java_frame(true);
10827
10828 __ mov(rscratch1, r0);
10829
10830 __ pop_call_clobbered_registers();
10831
10832 __ cbnz(rscratch1, deoptimize_label);
10833
10834 __ leave();
10835 __ ret(lr);
10836
10837 __ BIND(deoptimize_label);
10838
10839 __ ldp(/* new sp */ rscratch1, rfp, Address(sp, 0 * wordSize));
10840 __ ldp(lr, /* new pc*/ rscratch2, Address(sp, 2 * wordSize));
10841
10842 __ mov(sp, rscratch1);
10843 __ br(rscratch2);
10844
10845 // record the stub entry and end
10846 store_archive_data(stub_id, start, __ pc());
10847
10848 return start;
10849 }
10850
10851 // r0 = result
10852 // r1 = str1
10853 // r2 = cnt1
10854 // r3 = str2
10855 // r4 = cnt2
10856 // r10 = tmp1
10857 // r11 = tmp2
10858 address generate_compare_long_string_same_encoding(bool isLL) {
10859 StubId stub_id = (isLL ? StubId::stubgen_compare_long_string_LL_id : StubId::stubgen_compare_long_string_UU_id);
10860 int entry_count = StubInfo::entry_count(stub_id);
10861 assert(entry_count == 1, "sanity check");
10862 address start = load_archive_data(stub_id);
10863 if (start != nullptr) {
10864 return start;
10865 }
10866 __ align(CodeEntryAlignment);
10867 StubCodeMark mark(this, stub_id);
10868 address entry = __ pc();
10869 Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
10870 tmp1 = r10, tmp2 = r11, tmp1h = rscratch1, tmp2h = rscratch2;
10871
10872 Label LARGE_LOOP_PREFETCH, LOOP_COMPARE16, DIFF, LESS16, LESS8, CAL_DIFFERENCE, LENGTH_DIFF;
10873
10874 // exit from large loop when less than 64 bytes left to read or we're about
10875 // to prefetch memory behind array border
10876 int largeLoopExitCondition = MAX2(64, SoftwarePrefetchHintDistance)/(isLL ? 1 : 2);
10877
10878 // before jumping to stub, pre-load 8 bytes already, so do comparison directly
10879 __ eor(rscratch2, tmp1, tmp2);
10880 __ cbnz(rscratch2, CAL_DIFFERENCE);
10881
10882 __ sub(cnt2, cnt2, wordSize/(isLL ? 1 : 2));
10883 // update pointers, because of previous read
10884 __ add(str1, str1, wordSize);
10885 __ add(str2, str2, wordSize);
10886 if (SoftwarePrefetchHintDistance >= 0) {
10887 __ align(OptoLoopAlignment);
10888 __ bind(LARGE_LOOP_PREFETCH);
10889 __ prfm(Address(str1, SoftwarePrefetchHintDistance));
10890 __ prfm(Address(str2, SoftwarePrefetchHintDistance));
10891
10892 for (int i = 0; i < 4; i++) {
10893 __ ldp(tmp1, tmp1h, Address(str1, i * 16));
10894 __ ldp(tmp2, tmp2h, Address(str2, i * 16));
10895 __ cmp(tmp1, tmp2);
10896 __ ccmp(tmp1h, tmp2h, 0, Assembler::EQ);
10897 __ br(Assembler::NE, DIFF);
10898 }
10899 __ sub(cnt2, cnt2, isLL ? 64 : 32);
10900 __ add(str1, str1, 64);
10901 __ add(str2, str2, 64);
10902 __ subs(rscratch2, cnt2, largeLoopExitCondition);
10903 __ br(Assembler::GE, LARGE_LOOP_PREFETCH);
10904 __ cbz(cnt2, LENGTH_DIFF); // no more chars left?
10905 }
10906
10907 __ subs(rscratch1, cnt2, isLL ? 16 : 8);
10908 __ br(Assembler::LE, LESS16);
10909 __ align(OptoLoopAlignment);
10910 __ bind(LOOP_COMPARE16);
10911 __ ldp(tmp1, tmp1h, Address(__ post(str1, 16)));
10912 __ ldp(tmp2, tmp2h, Address(__ post(str2, 16)));
10913 __ cmp(tmp1, tmp2);
10914 __ ccmp(tmp1h, tmp2h, 0, Assembler::EQ);
10915 __ br(Assembler::NE, DIFF);
10916 __ sub(cnt2, cnt2, isLL ? 16 : 8);
10917 __ subs(rscratch2, cnt2, isLL ? 16 : 8);
10918 __ br(Assembler::LT, LESS16);
10919
10920 __ ldp(tmp1, tmp1h, Address(__ post(str1, 16)));
10921 __ ldp(tmp2, tmp2h, Address(__ post(str2, 16)));
10922 __ cmp(tmp1, tmp2);
10923 __ ccmp(tmp1h, tmp2h, 0, Assembler::EQ);
10924 __ br(Assembler::NE, DIFF);
10925 __ sub(cnt2, cnt2, isLL ? 16 : 8);
10926 __ subs(rscratch2, cnt2, isLL ? 16 : 8);
10927 __ br(Assembler::GE, LOOP_COMPARE16);
10928 __ cbz(cnt2, LENGTH_DIFF);
10929
10930 __ bind(LESS16);
10931 // each 8 compare
10932 __ subs(cnt2, cnt2, isLL ? 8 : 4);
10933 __ br(Assembler::LE, LESS8);
10934 __ ldr(tmp1, Address(__ post(str1, 8)));
10935 __ ldr(tmp2, Address(__ post(str2, 8)));
10936 __ eor(rscratch2, tmp1, tmp2);
10937 __ cbnz(rscratch2, CAL_DIFFERENCE);
10938 __ sub(cnt2, cnt2, isLL ? 8 : 4);
10939
10940 __ bind(LESS8); // directly load last 8 bytes
10941 if (!isLL) {
10942 __ add(cnt2, cnt2, cnt2);
10943 }
10944 __ ldr(tmp1, Address(str1, cnt2));
10945 __ ldr(tmp2, Address(str2, cnt2));
10946 __ eor(rscratch2, tmp1, tmp2);
10947 __ cbz(rscratch2, LENGTH_DIFF);
10948 __ b(CAL_DIFFERENCE);
10949
10950 __ bind(DIFF);
10951 __ cmp(tmp1, tmp2);
10952 __ csel(tmp1, tmp1, tmp1h, Assembler::NE);
10953 __ csel(tmp2, tmp2, tmp2h, Assembler::NE);
10954 // reuse rscratch2 register for the result of eor instruction
10955 __ eor(rscratch2, tmp1, tmp2);
10956
10957 __ bind(CAL_DIFFERENCE);
10958 __ rev(rscratch2, rscratch2);
10959 __ clz(rscratch2, rscratch2);
10960 __ andr(rscratch2, rscratch2, isLL ? -8 : -16);
10961 __ lsrv(tmp1, tmp1, rscratch2);
10962 __ lsrv(tmp2, tmp2, rscratch2);
10963 if (isLL) {
10964 __ uxtbw(tmp1, tmp1);
10965 __ uxtbw(tmp2, tmp2);
10966 } else {
10967 __ uxthw(tmp1, tmp1);
10968 __ uxthw(tmp2, tmp2);
10969 }
10970 __ subw(result, tmp1, tmp2);
10971
10972 __ bind(LENGTH_DIFF);
10973 __ ret(lr);
10974
10975 // record the stub entry and end
10976 store_archive_data(stub_id, entry, __ pc());
10977
10978 return entry;
10979 }
10980
10981 enum string_compare_mode {
10982 LL,
10983 LU,
10984 UL,
10985 UU,
10986 };
10987
10988 // The following registers are declared in aarch64.ad
10989 // r0 = result
10990 // r1 = str1
10991 // r2 = cnt1
10992 // r3 = str2
10993 // r4 = cnt2
10994 // r10 = tmp1
10995 // r11 = tmp2
10996 // z0 = ztmp1
10997 // z1 = ztmp2
10998 // p0 = pgtmp1
10999 // p1 = pgtmp2
11000 address generate_compare_long_string_sve(string_compare_mode mode) {
11001 StubId stub_id;
11002 switch (mode) {
11003 case LL: stub_id = StubId::stubgen_compare_long_string_LL_id; break;
11004 case LU: stub_id = StubId::stubgen_compare_long_string_LU_id; break;
11005 case UL: stub_id = StubId::stubgen_compare_long_string_UL_id; break;
11006 case UU: stub_id = StubId::stubgen_compare_long_string_UU_id; break;
11007 default: ShouldNotReachHere();
11008 }
11009 int entry_count = StubInfo::entry_count(stub_id);
11010 assert(entry_count == 1, "sanity check");
11011 address start = load_archive_data(stub_id);
11012 if (start != nullptr) {
11013 return start;
11014 }
11015 __ align(CodeEntryAlignment);
11016 StubCodeMark mark(this, stub_id);
11017 address entry = __ pc();
11018 Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
11019 tmp1 = r10, tmp2 = r11;
11020
11021 Label LOOP, DONE, MISMATCH;
11022 Register vec_len = tmp1;
11023 Register idx = tmp2;
11024 // The minimum of the string lengths has been stored in cnt2.
11025 Register cnt = cnt2;
11026 FloatRegister ztmp1 = z0, ztmp2 = z1;
11027 PRegister pgtmp1 = p0, pgtmp2 = p1;
11028
11029 #define LOAD_PAIR(ztmp1, ztmp2, pgtmp1, src1, src2, idx) \
11030 switch (mode) { \
11031 case LL: \
11032 __ sve_ld1b(ztmp1, __ B, pgtmp1, Address(str1, idx)); \
11033 __ sve_ld1b(ztmp2, __ B, pgtmp1, Address(str2, idx)); \
11034 break; \
11035 case LU: \
11036 __ sve_ld1b(ztmp1, __ H, pgtmp1, Address(str1, idx)); \
11037 __ sve_ld1h(ztmp2, __ H, pgtmp1, Address(str2, idx, Address::lsl(1))); \
11038 break; \
11039 case UL: \
11040 __ sve_ld1h(ztmp1, __ H, pgtmp1, Address(str1, idx, Address::lsl(1))); \
11041 __ sve_ld1b(ztmp2, __ H, pgtmp1, Address(str2, idx)); \
11042 break; \
11043 case UU: \
11044 __ sve_ld1h(ztmp1, __ H, pgtmp1, Address(str1, idx, Address::lsl(1))); \
11045 __ sve_ld1h(ztmp2, __ H, pgtmp1, Address(str2, idx, Address::lsl(1))); \
11046 break; \
11047 default: \
11048 ShouldNotReachHere(); \
11049 }
11050
11051 __ mov(idx, 0);
11052 __ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt);
11053
11054 if (mode == LL) {
11055 __ sve_cntb(vec_len);
11056 } else {
11057 __ sve_cnth(vec_len);
11058 }
11059
11060 __ sub(rscratch1, cnt, vec_len);
11061
11062 __ bind(LOOP);
11063
11064 // main loop
11065 LOAD_PAIR(ztmp1, ztmp2, pgtmp1, src1, src2, idx);
11066 __ add(idx, idx, vec_len);
11067 // Compare strings.
11068 __ sve_cmp(Assembler::NE, pgtmp2, mode == LL ? __ B : __ H, pgtmp1, ztmp1, ztmp2);
11069 __ br(__ NE, MISMATCH);
11070 __ cmp(idx, rscratch1);
11071 __ br(__ LT, LOOP);
11072
11073 // post loop, last iteration
11074 __ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt);
11075
11076 LOAD_PAIR(ztmp1, ztmp2, pgtmp1, src1, src2, idx);
11077 __ sve_cmp(Assembler::NE, pgtmp2, mode == LL ? __ B : __ H, pgtmp1, ztmp1, ztmp2);
11078 __ br(__ EQ, DONE);
11079
11080 __ bind(MISMATCH);
11081
11082 // Crop the vector to find its location.
11083 __ sve_brkb(pgtmp2, pgtmp1, pgtmp2, false /* isMerge */);
11084 // Extract the first different characters of each string.
11085 __ sve_lasta(rscratch1, mode == LL ? __ B : __ H, pgtmp2, ztmp1);
11086 __ sve_lasta(rscratch2, mode == LL ? __ B : __ H, pgtmp2, ztmp2);
11087
11088 // Compute the difference of the first different characters.
11089 __ sub(result, rscratch1, rscratch2);
11090
11091 __ bind(DONE);
11092 __ ret(lr);
11093 #undef LOAD_PAIR
11094
11095 // record the stub entry and end
11096 store_archive_data(stub_id, entry, __ pc());
11097
11098 return entry;
11099 }
11100
11101 void generate_compare_long_strings() {
11102 if (UseSVE == 0) {
11103 StubRoutines::aarch64::_compare_long_string_LL
11104 = generate_compare_long_string_same_encoding(true);
11105 StubRoutines::aarch64::_compare_long_string_UU
11106 = generate_compare_long_string_same_encoding(false);
11107 StubRoutines::aarch64::_compare_long_string_LU
11108 = generate_compare_long_string_different_encoding(true);
11109 StubRoutines::aarch64::_compare_long_string_UL
11110 = generate_compare_long_string_different_encoding(false);
11111 } else {
11112 StubRoutines::aarch64::_compare_long_string_LL
11113 = generate_compare_long_string_sve(LL);
11114 StubRoutines::aarch64::_compare_long_string_UU
11115 = generate_compare_long_string_sve(UU);
11116 StubRoutines::aarch64::_compare_long_string_LU
11117 = generate_compare_long_string_sve(LU);
11118 StubRoutines::aarch64::_compare_long_string_UL
11119 = generate_compare_long_string_sve(UL);
11120 }
11121 }
11122
11123 // R0 = result
11124 // R1 = str2
11125 // R2 = cnt1
11126 // R3 = str1
11127 // R4 = cnt2
11128 // Clobbers: rscratch1, rscratch2, v0, v1, rflags
11129 //
11130 // This generic linear code use few additional ideas, which makes it faster:
11131 // 1) we can safely keep at least 1st register of pattern(since length >= 8)
11132 // in order to skip initial loading(help in systems with 1 ld pipeline)
11133 // 2) we can use "fast" algorithm of finding single character to search for
11134 // first symbol with less branches(1 branch per each loaded register instead
11135 // of branch for each symbol), so, this is where constants like
11136 // 0x0101...01, 0x00010001...0001, 0x7f7f...7f, 0x7fff7fff...7fff comes from
11137 // 3) after loading and analyzing 1st register of source string, it can be
11138 // used to search for every 1st character entry, saving few loads in
11139 // comparison with "simplier-but-slower" implementation
11140 // 4) in order to avoid lots of push/pop operations, code below is heavily
11141 // re-using/re-initializing/compressing register values, which makes code
11142 // larger and a bit less readable, however, most of extra operations are
11143 // issued during loads or branches, so, penalty is minimal
11144 address generate_string_indexof_linear(bool str1_isL, bool str2_isL) {
11145 StubId stub_id;
11146 if (str1_isL) {
11147 if (str2_isL) {
11148 stub_id = StubId::stubgen_string_indexof_linear_ll_id;
11149 } else {
11150 stub_id = StubId::stubgen_string_indexof_linear_ul_id;
11151 }
11152 } else {
11153 if (str2_isL) {
11154 ShouldNotReachHere();
11155 } else {
11156 stub_id = StubId::stubgen_string_indexof_linear_uu_id;
11157 }
11158 }
11159 int entry_count = StubInfo::entry_count(stub_id);
11160 assert(entry_count == 1, "sanity check");
11161 address start = load_archive_data(stub_id);
11162 if (start != nullptr) {
11163 return start;
11164 }
11165 __ align(CodeEntryAlignment);
11166 StubCodeMark mark(this, stub_id);
11167 address entry = __ pc();
11168
11169 int str1_chr_size = str1_isL ? 1 : 2;
11170 int str2_chr_size = str2_isL ? 1 : 2;
11171 int str1_chr_shift = str1_isL ? 0 : 1;
11172 int str2_chr_shift = str2_isL ? 0 : 1;
11173 bool isL = str1_isL && str2_isL;
11174 // parameters
11175 Register result = r0, str2 = r1, cnt1 = r2, str1 = r3, cnt2 = r4;
11176 // temporary registers
11177 Register tmp1 = r20, tmp2 = r21, tmp3 = r22, tmp4 = r23;
11178 RegSet spilled_regs = RegSet::range(tmp1, tmp4);
11179 // redefinitions
11180 Register ch1 = rscratch1, ch2 = rscratch2, first = tmp3;
11181
11182 __ push(spilled_regs, sp);
11183 Label L_LOOP, L_LOOP_PROCEED, L_SMALL, L_HAS_ZERO,
11184 L_HAS_ZERO_LOOP, L_CMP_LOOP, L_CMP_LOOP_NOMATCH, L_SMALL_PROCEED,
11185 L_SMALL_HAS_ZERO_LOOP, L_SMALL_CMP_LOOP_NOMATCH, L_SMALL_CMP_LOOP,
11186 L_POST_LOOP, L_CMP_LOOP_LAST_CMP, L_HAS_ZERO_LOOP_NOMATCH,
11187 L_SMALL_CMP_LOOP_LAST_CMP, L_SMALL_CMP_LOOP_LAST_CMP2,
11188 L_CMP_LOOP_LAST_CMP2, DONE, NOMATCH;
11189 // Read whole register from str1. It is safe, because length >=8 here
11190 __ ldr(ch1, Address(str1));
11191 // Read whole register from str2. It is safe, because length >=8 here
11192 __ ldr(ch2, Address(str2));
11193 __ sub(cnt2, cnt2, cnt1);
11194 __ andr(first, ch1, str1_isL ? 0xFF : 0xFFFF);
11195 if (str1_isL != str2_isL) {
11196 __ eor(v0, __ T16B, v0, v0);
11197 }
11198 __ mov(tmp1, str2_isL ? 0x0101010101010101 : 0x0001000100010001);
11199 __ mul(first, first, tmp1);
11200 // check if we have less than 1 register to check
11201 __ subs(cnt2, cnt2, wordSize/str2_chr_size - 1);
11202 if (str1_isL != str2_isL) {
11203 __ fmovd(v1, ch1);
11204 }
11205 __ br(__ LE, L_SMALL);
11206 __ eor(ch2, first, ch2);
11207 if (str1_isL != str2_isL) {
11208 __ zip1(v1, __ T16B, v1, v0);
11209 }
11210 __ sub(tmp2, ch2, tmp1);
11211 __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
11212 __ bics(tmp2, tmp2, ch2);
11213 if (str1_isL != str2_isL) {
11214 __ fmovd(ch1, v1);
11215 }
11216 __ br(__ NE, L_HAS_ZERO);
11217 __ subs(cnt2, cnt2, wordSize/str2_chr_size);
11218 __ add(result, result, wordSize/str2_chr_size);
11219 __ add(str2, str2, wordSize);
11220 __ br(__ LT, L_POST_LOOP);
11221 __ BIND(L_LOOP);
11222 __ ldr(ch2, Address(str2));
11223 __ eor(ch2, first, ch2);
11224 __ sub(tmp2, ch2, tmp1);
11225 __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
11226 __ bics(tmp2, tmp2, ch2);
11227 __ br(__ NE, L_HAS_ZERO);
11228 __ BIND(L_LOOP_PROCEED);
11229 __ subs(cnt2, cnt2, wordSize/str2_chr_size);
11230 __ add(str2, str2, wordSize);
11231 __ add(result, result, wordSize/str2_chr_size);
11232 __ br(__ GE, L_LOOP);
11233 __ BIND(L_POST_LOOP);
11234 __ subs(zr, cnt2, -wordSize/str2_chr_size); // no extra characters to check
11235 __ br(__ LE, NOMATCH);
11236 __ ldr(ch2, Address(str2));
11237 __ sub(cnt2, zr, cnt2, __ LSL, LogBitsPerByte + str2_chr_shift);
11238 __ eor(ch2, first, ch2);
11239 __ sub(tmp2, ch2, tmp1);
11240 __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
11241 __ mov(tmp4, -1); // all bits set
11242 __ b(L_SMALL_PROCEED);
11243 __ align(OptoLoopAlignment);
11244 __ BIND(L_SMALL);
11245 __ sub(cnt2, zr, cnt2, __ LSL, LogBitsPerByte + str2_chr_shift);
11246 __ eor(ch2, first, ch2);
11247 if (str1_isL != str2_isL) {
11248 __ zip1(v1, __ T16B, v1, v0);
11249 }
11250 __ sub(tmp2, ch2, tmp1);
11251 __ mov(tmp4, -1); // all bits set
11252 __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
11253 if (str1_isL != str2_isL) {
11254 __ fmovd(ch1, v1); // move converted 4 symbols
11255 }
11256 __ BIND(L_SMALL_PROCEED);
11257 __ lsrv(tmp4, tmp4, cnt2); // mask. zeroes on useless bits.
11258 __ bic(tmp2, tmp2, ch2);
11259 __ ands(tmp2, tmp2, tmp4); // clear useless bits and check
11260 __ rbit(tmp2, tmp2);
11261 __ br(__ EQ, NOMATCH);
11262 __ BIND(L_SMALL_HAS_ZERO_LOOP);
11263 __ clz(tmp4, tmp2); // potentially long. Up to 4 cycles on some cpu's
11264 __ cmp(cnt1, u1(wordSize/str2_chr_size));
11265 __ br(__ LE, L_SMALL_CMP_LOOP_LAST_CMP2);
11266 if (str2_isL) { // LL
11267 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte); // address of "index"
11268 __ ldr(ch2, Address(str2)); // read whole register of str2. Safe.
11269 __ lslv(tmp2, tmp2, tmp4); // shift off leading zeroes from match info
11270 __ add(result, result, tmp4, __ LSR, LogBitsPerByte);
11271 __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
11272 } else {
11273 __ mov(ch2, 0xE); // all bits in byte set except last one
11274 __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
11275 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11276 __ lslv(tmp2, tmp2, tmp4);
11277 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11278 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11279 __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
11280 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11281 }
11282 __ cmp(ch1, ch2);
11283 __ mov(tmp4, wordSize/str2_chr_size);
11284 __ br(__ NE, L_SMALL_CMP_LOOP_NOMATCH);
11285 __ BIND(L_SMALL_CMP_LOOP);
11286 str1_isL ? __ ldrb(first, Address(str1, tmp4, Address::lsl(str1_chr_shift)))
11287 : __ ldrh(first, Address(str1, tmp4, Address::lsl(str1_chr_shift)));
11288 str2_isL ? __ ldrb(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)))
11289 : __ ldrh(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)));
11290 __ add(tmp4, tmp4, 1);
11291 __ cmp(tmp4, cnt1);
11292 __ br(__ GE, L_SMALL_CMP_LOOP_LAST_CMP);
11293 __ cmp(first, ch2);
11294 __ br(__ EQ, L_SMALL_CMP_LOOP);
11295 __ BIND(L_SMALL_CMP_LOOP_NOMATCH);
11296 __ cbz(tmp2, NOMATCH); // no more matches. exit
11297 __ clz(tmp4, tmp2);
11298 __ add(result, result, 1); // advance index
11299 __ add(str2, str2, str2_chr_size); // advance pointer
11300 __ b(L_SMALL_HAS_ZERO_LOOP);
11301 __ align(OptoLoopAlignment);
11302 __ BIND(L_SMALL_CMP_LOOP_LAST_CMP);
11303 __ cmp(first, ch2);
11304 __ br(__ NE, L_SMALL_CMP_LOOP_NOMATCH);
11305 __ b(DONE);
11306 __ align(OptoLoopAlignment);
11307 __ BIND(L_SMALL_CMP_LOOP_LAST_CMP2);
11308 if (str2_isL) { // LL
11309 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte); // address of "index"
11310 __ ldr(ch2, Address(str2)); // read whole register of str2. Safe.
11311 __ lslv(tmp2, tmp2, tmp4); // shift off leading zeroes from match info
11312 __ add(result, result, tmp4, __ LSR, LogBitsPerByte);
11313 __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
11314 } else {
11315 __ mov(ch2, 0xE); // all bits in byte set except last one
11316 __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
11317 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11318 __ lslv(tmp2, tmp2, tmp4);
11319 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11320 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11321 __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
11322 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11323 }
11324 __ cmp(ch1, ch2);
11325 __ br(__ NE, L_SMALL_CMP_LOOP_NOMATCH);
11326 __ b(DONE);
11327 __ align(OptoLoopAlignment);
11328 __ BIND(L_HAS_ZERO);
11329 __ rbit(tmp2, tmp2);
11330 __ clz(tmp4, tmp2); // potentially long. Up to 4 cycles on some CPU's
11331 // Now, perform compression of counters(cnt2 and cnt1) into one register.
11332 // It's fine because both counters are 32bit and are not changed in this
11333 // loop. Just restore it on exit. So, cnt1 can be re-used in this loop.
11334 __ orr(cnt2, cnt2, cnt1, __ LSL, BitsPerByte * wordSize / 2);
11335 __ sub(result, result, 1);
11336 __ BIND(L_HAS_ZERO_LOOP);
11337 __ mov(cnt1, wordSize/str2_chr_size);
11338 __ cmp(cnt1, cnt2, __ LSR, BitsPerByte * wordSize / 2);
11339 __ br(__ GE, L_CMP_LOOP_LAST_CMP2); // case of 8 bytes only to compare
11340 if (str2_isL) {
11341 __ lsr(ch2, tmp4, LogBitsPerByte + str2_chr_shift); // char index
11342 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11343 __ lslv(tmp2, tmp2, tmp4);
11344 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11345 __ add(tmp4, tmp4, 1);
11346 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11347 __ lsl(tmp2, tmp2, 1);
11348 __ mov(tmp4, wordSize/str2_chr_size);
11349 } else {
11350 __ mov(ch2, 0xE);
11351 __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
11352 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11353 __ lslv(tmp2, tmp2, tmp4);
11354 __ add(tmp4, tmp4, 1);
11355 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11356 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte);
11357 __ lsl(tmp2, tmp2, 1);
11358 __ mov(tmp4, wordSize/str2_chr_size);
11359 __ sub(str2, str2, str2_chr_size);
11360 }
11361 __ cmp(ch1, ch2);
11362 __ mov(tmp4, wordSize/str2_chr_size);
11363 __ br(__ NE, L_CMP_LOOP_NOMATCH);
11364 __ BIND(L_CMP_LOOP);
11365 str1_isL ? __ ldrb(cnt1, Address(str1, tmp4, Address::lsl(str1_chr_shift)))
11366 : __ ldrh(cnt1, Address(str1, tmp4, Address::lsl(str1_chr_shift)));
11367 str2_isL ? __ ldrb(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)))
11368 : __ ldrh(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)));
11369 __ add(tmp4, tmp4, 1);
11370 __ cmp(tmp4, cnt2, __ LSR, BitsPerByte * wordSize / 2);
11371 __ br(__ GE, L_CMP_LOOP_LAST_CMP);
11372 __ cmp(cnt1, ch2);
11373 __ br(__ EQ, L_CMP_LOOP);
11374 __ BIND(L_CMP_LOOP_NOMATCH);
11375 // here we're not matched
11376 __ cbz(tmp2, L_HAS_ZERO_LOOP_NOMATCH); // no more matches. Proceed to main loop
11377 __ clz(tmp4, tmp2);
11378 __ add(str2, str2, str2_chr_size); // advance pointer
11379 __ b(L_HAS_ZERO_LOOP);
11380 __ align(OptoLoopAlignment);
11381 __ BIND(L_CMP_LOOP_LAST_CMP);
11382 __ cmp(cnt1, ch2);
11383 __ br(__ NE, L_CMP_LOOP_NOMATCH);
11384 __ b(DONE);
11385 __ align(OptoLoopAlignment);
11386 __ BIND(L_CMP_LOOP_LAST_CMP2);
11387 if (str2_isL) {
11388 __ lsr(ch2, tmp4, LogBitsPerByte + str2_chr_shift); // char index
11389 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11390 __ lslv(tmp2, tmp2, tmp4);
11391 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11392 __ add(tmp4, tmp4, 1);
11393 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11394 __ lsl(tmp2, tmp2, 1);
11395 } else {
11396 __ mov(ch2, 0xE);
11397 __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
11398 __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
11399 __ lslv(tmp2, tmp2, tmp4);
11400 __ add(tmp4, tmp4, 1);
11401 __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
11402 __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte);
11403 __ lsl(tmp2, tmp2, 1);
11404 __ sub(str2, str2, str2_chr_size);
11405 }
11406 __ cmp(ch1, ch2);
11407 __ br(__ NE, L_CMP_LOOP_NOMATCH);
11408 __ b(DONE);
11409 __ align(OptoLoopAlignment);
11410 __ BIND(L_HAS_ZERO_LOOP_NOMATCH);
11411 // 1) Restore "result" index. Index was wordSize/str2_chr_size * N until
11412 // L_HAS_ZERO block. Byte octet was analyzed in L_HAS_ZERO_LOOP,
11413 // so, result was increased at max by wordSize/str2_chr_size - 1, so,
11414 // respective high bit wasn't changed. L_LOOP_PROCEED will increase
11415 // result by analyzed characters value, so, we can just reset lower bits
11416 // in result here. Clear 2 lower bits for UU/UL and 3 bits for LL
11417 // 2) restore cnt1 and cnt2 values from "compressed" cnt2
11418 // 3) advance str2 value to represent next str2 octet. result & 7/3 is
11419 // index of last analyzed substring inside current octet. So, str2 in at
11420 // respective start address. We need to advance it to next octet
11421 __ andr(tmp2, result, wordSize/str2_chr_size - 1); // symbols analyzed
11422 __ lsr(cnt1, cnt2, BitsPerByte * wordSize / 2);
11423 __ bfm(result, zr, 0, 2 - str2_chr_shift);
11424 __ sub(str2, str2, tmp2, __ LSL, str2_chr_shift); // restore str2
11425 __ movw(cnt2, cnt2);
11426 __ b(L_LOOP_PROCEED);
11427 __ align(OptoLoopAlignment);
11428 __ BIND(NOMATCH);
11429 __ mov(result, -1);
11430 __ BIND(DONE);
11431 __ pop(spilled_regs, sp);
11432 __ ret(lr);
11433
11434 // record the stub entry and end
11435 store_archive_data(stub_id, entry, __ pc());
11436
11437 return entry;
11438 }
11439
11440 void generate_string_indexof_stubs() {
11441 StubRoutines::aarch64::_string_indexof_linear_ll = generate_string_indexof_linear(true, true);
11442 StubRoutines::aarch64::_string_indexof_linear_uu = generate_string_indexof_linear(false, false);
11443 StubRoutines::aarch64::_string_indexof_linear_ul = generate_string_indexof_linear(true, false);
11444 }
11445
11446 void inflate_and_store_2_fp_registers(bool generatePrfm,
11447 FloatRegister src1, FloatRegister src2) {
11448 Register dst = r1;
11449 __ zip1(v1, __ T16B, src1, v0);
11450 __ zip2(v2, __ T16B, src1, v0);
11451 if (generatePrfm) {
11452 __ prfm(Address(dst, SoftwarePrefetchHintDistance), PSTL1STRM);
11453 }
11454 __ zip1(v3, __ T16B, src2, v0);
11455 __ zip2(v4, __ T16B, src2, v0);
11456 __ st1(v1, v2, v3, v4, __ T16B, Address(__ post(dst, 64)));
11457 }
11458
11459 // R0 = src
11460 // R1 = dst
11461 // R2 = len
11462 // R3 = len >> 3
11463 // V0 = 0
11464 // v1 = loaded 8 bytes
11465 // Clobbers: r0, r1, r3, rscratch1, rflags, v0-v6
11466 address generate_large_byte_array_inflate() {
11467 StubId stub_id = StubId::stubgen_large_byte_array_inflate_id;
11468 int entry_count = StubInfo::entry_count(stub_id);
11469 assert(entry_count == 1, "sanity check");
11470 address start = load_archive_data(stub_id);
11471 if (start != nullptr) {
11472 return start;
11473 }
11474 __ align(CodeEntryAlignment);
11475 StubCodeMark mark(this, stub_id);
11476 address entry = __ pc();
11477 Label LOOP, LOOP_START, LOOP_PRFM, LOOP_PRFM_START, DONE;
11478 Register src = r0, dst = r1, len = r2, octetCounter = r3;
11479 const int large_loop_threshold = MAX2(64, SoftwarePrefetchHintDistance)/8 + 4;
11480
11481 // do one more 8-byte read to have address 16-byte aligned in most cases
11482 // also use single store instruction
11483 __ ldrd(v2, __ post(src, 8));
11484 __ sub(octetCounter, octetCounter, 2);
11485 __ zip1(v1, __ T16B, v1, v0);
11486 __ zip1(v2, __ T16B, v2, v0);
11487 __ st1(v1, v2, __ T16B, __ post(dst, 32));
11488 __ ld1(v3, v4, v5, v6, __ T16B, Address(__ post(src, 64)));
11489 __ subs(rscratch1, octetCounter, large_loop_threshold);
11490 __ br(__ LE, LOOP_START);
11491 __ b(LOOP_PRFM_START);
11492 __ bind(LOOP_PRFM);
11493 __ ld1(v3, v4, v5, v6, __ T16B, Address(__ post(src, 64)));
11494 __ bind(LOOP_PRFM_START);
11495 __ prfm(Address(src, SoftwarePrefetchHintDistance));
11496 __ sub(octetCounter, octetCounter, 8);
11497 __ subs(rscratch1, octetCounter, large_loop_threshold);
11498 inflate_and_store_2_fp_registers(true, v3, v4);
11499 inflate_and_store_2_fp_registers(true, v5, v6);
11500 __ br(__ GT, LOOP_PRFM);
11501 __ cmp(octetCounter, (u1)8);
11502 __ br(__ LT, DONE);
11503 __ bind(LOOP);
11504 __ ld1(v3, v4, v5, v6, __ T16B, Address(__ post(src, 64)));
11505 __ bind(LOOP_START);
11506 __ sub(octetCounter, octetCounter, 8);
11507 __ cmp(octetCounter, (u1)8);
11508 inflate_and_store_2_fp_registers(false, v3, v4);
11509 inflate_and_store_2_fp_registers(false, v5, v6);
11510 __ br(__ GE, LOOP);
11511 __ bind(DONE);
11512 __ ret(lr);
11513
11514 // record the stub entry and end
11515 store_archive_data(stub_id, entry, __ pc());
11516
11517 return entry;
11518 }
11519
11520 /**
11521 * Arguments:
11522 *
11523 * Input:
11524 * c_rarg0 - current state address
11525 * c_rarg1 - H key address
11526 * c_rarg2 - data address
11527 * c_rarg3 - number of blocks
11528 *
11529 * Output:
11530 * Updated state at c_rarg0
11531 */
11532 address generate_ghash_processBlocks_small() {
11533 // Bafflingly, GCM uses little-endian for the byte order, but
11534 // big-endian for the bit order. For example, the polynomial 1 is
11535 // represented as the 16-byte string 80 00 00 00 | 12 bytes of 00.
11536 //
11537 // So, we must either reverse the bytes in each word and do
11538 // everything big-endian or reverse the bits in each byte and do
11539 // it little-endian. On AArch64 it's more idiomatic to reverse
11540 // the bits in each byte (we have an instruction, RBIT, to do
11541 // that) and keep the data in little-endian bit order through the
11542 // calculation, bit-reversing the inputs and outputs.
11543
11544 StubId stub_id = StubId::stubgen_ghash_processBlocks_small_id;
11545 int entry_count = StubInfo::entry_count(stub_id);
11546 assert(entry_count == 1, "sanity check");
11547 address start = load_archive_data(stub_id);
11548 if (start != nullptr) {
11549 return start;
11550 }
11551 __ align(CodeEntryAlignment);
11552 StubCodeMark mark(this, stub_id);
11553 Label polynomial; // local data generated at end of stub
11554 start = __ pc();
11555
11556 Register state = c_rarg0;
11557 Register subkeyH = c_rarg1;
11558 Register data = c_rarg2;
11559 Register blocks = c_rarg3;
11560
11561 FloatRegister vzr = v30;
11562 __ eor(vzr, __ T16B, vzr, vzr); // zero register
11563
11564 __ adr(rscratch1, polynomial);
11565 __ ldrq(v24, rscratch1); // The field polynomial
11566
11567 __ ldrq(v0, Address(state));
11568 __ ldrq(v1, Address(subkeyH));
11569
11570 __ rev64(v0, __ T16B, v0); // Bit-reverse words in state and subkeyH
11571 __ rbit(v0, __ T16B, v0);
11572 __ rev64(v1, __ T16B, v1);
11573 __ rbit(v1, __ T16B, v1);
11574
11575 __ ext(v4, __ T16B, v1, v1, 0x08); // long-swap subkeyH into v1
11576 __ eor(v4, __ T16B, v4, v1); // xor subkeyH into subkeyL (Karatsuba: (A1+A0))
11577
11578 {
11579 Label L_ghash_loop;
11580 __ bind(L_ghash_loop);
11581
11582 __ ldrq(v2, Address(__ post(data, 0x10))); // Load the data, bit
11583 // reversing each byte
11584 __ rbit(v2, __ T16B, v2);
11585 __ eor(v2, __ T16B, v0, v2); // bit-swapped data ^ bit-swapped state
11586
11587 // Multiply state in v2 by subkey in v1
11588 __ ghash_multiply(/*result_lo*/v5, /*result_hi*/v7,
11589 /*a*/v1, /*b*/v2, /*a1_xor_a0*/v4,
11590 /*temps*/v6, v3, /*reuse/clobber b*/v2);
11591 // Reduce v7:v5 by the field polynomial
11592 __ ghash_reduce(/*result*/v0, /*lo*/v5, /*hi*/v7, /*p*/v24, vzr, /*temp*/v3);
11593
11594 __ sub(blocks, blocks, 1);
11595 __ cbnz(blocks, L_ghash_loop);
11596 }
11597
11598 // The bit-reversed result is at this point in v0
11599 __ rev64(v0, __ T16B, v0);
11600 __ rbit(v0, __ T16B, v0);
11601
11602 __ st1(v0, __ T16B, state);
11603 __ ret(lr);
11604
11605 // bind label and generate local polynomial data
11606 __ align(wordSize * 2);
11607 __ bind(polynomial);
11608 __ emit_int64(0x87); // The low-order bits of the field
11609 // polynomial (i.e. p = z^7+z^2+z+1)
11610 // repeated in the low and high parts of a
11611 // 128-bit vector
11612 __ emit_int64(0x87);
11613
11614 // record the stub entry and end
11615 store_archive_data(stub_id, start, __ pc());
11616
11617 return start;
11618 }
11619
11620 address generate_ghash_processBlocks(address small) {
11621 StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
11622 int entry_count = StubInfo::entry_count(stub_id);
11623 assert(entry_count == 1, "sanity check");
11624 address start = load_archive_data(stub_id);
11625 if (start != nullptr) {
11626 return start;
11627 }
11628 Label polynomial; // local data generated after stub
11629 __ align(CodeEntryAlignment);
11630 StubCodeMark mark(this, stub_id);
11631 start = __ pc();
11632
11633 Register state = c_rarg0;
11634 Register subkeyH = c_rarg1;
11635 Register data = c_rarg2;
11636 Register blocks = c_rarg3;
11637
11638 const int unroll = 4;
11639
11640 __ cmp(blocks, (unsigned char)(unroll * 2));
11641 __ br(__ LT, small);
11642
11643 if (unroll > 1) {
11644 // Save state before entering routine
11645 __ sub(sp, sp, 4 * 16);
11646 __ st1(v12, v13, v14, v15, __ T16B, Address(sp));
11647 __ sub(sp, sp, 4 * 16);
11648 __ st1(v8, v9, v10, v11, __ T16B, Address(sp));
11649 }
11650
11651 __ ghash_processBlocks_wide(polynomial, state, subkeyH, data, blocks, unroll);
11652
11653 if (unroll > 1) {
11654 // And restore state
11655 __ ld1(v8, v9, v10, v11, __ T16B, __ post(sp, 4 * 16));
11656 __ ld1(v12, v13, v14, v15, __ T16B, __ post(sp, 4 * 16));
11657 }
11658
11659 __ cmp(blocks, (unsigned char)0);
11660 __ br(__ GT, small);
11661
11662 __ ret(lr);
11663
11664 // bind label and generate polynomial data
11665 __ align(wordSize * 2);
11666 __ bind(polynomial);
11667 __ emit_int64(0x87); // The low-order bits of the field
11668 // polynomial (i.e. p = z^7+z^2+z+1)
11669 // repeated in the low and high parts of a
11670 // 128-bit vector
11671 __ emit_int64(0x87);
11672
11673 // record the stub entry and end
11674 store_archive_data(stub_id, start, __ pc());
11675
11676 return start;
11677 }
11678
11679 void generate_base64_encode_simdround(Register src, Register dst,
11680 FloatRegister codec, u8 size) {
11681
11682 FloatRegister in0 = v4, in1 = v5, in2 = v6;
11683 FloatRegister out0 = v16, out1 = v17, out2 = v18, out3 = v19;
11684 FloatRegister ind0 = v20, ind1 = v21, ind2 = v22, ind3 = v23;
11685
11686 Assembler::SIMD_Arrangement arrangement = size == 16 ? __ T16B : __ T8B;
11687
11688 __ ld3(in0, in1, in2, arrangement, __ post(src, 3 * size));
11689
11690 __ ushr(ind0, arrangement, in0, 2);
11691
11692 __ ushr(ind1, arrangement, in1, 2);
11693 __ shl(in0, arrangement, in0, 6);
11694 __ orr(ind1, arrangement, ind1, in0);
11695 __ ushr(ind1, arrangement, ind1, 2);
11696
11697 __ ushr(ind2, arrangement, in2, 4);
11698 __ shl(in1, arrangement, in1, 4);
11699 __ orr(ind2, arrangement, in1, ind2);
11700 __ ushr(ind2, arrangement, ind2, 2);
11701
11702 __ shl(ind3, arrangement, in2, 2);
11703 __ ushr(ind3, arrangement, ind3, 2);
11704
11705 __ tbl(out0, arrangement, codec, 4, ind0);
11706 __ tbl(out1, arrangement, codec, 4, ind1);
11707 __ tbl(out2, arrangement, codec, 4, ind2);
11708 __ tbl(out3, arrangement, codec, 4, ind3);
11709
11710 __ st4(out0, out1, out2, out3, arrangement, __ post(dst, 4 * size));
11711 }
11712
11713 /**
11714 * Arguments:
11715 *
11716 * Input:
11717 * c_rarg0 - src_start
11718 * c_rarg1 - src_offset
11719 * c_rarg2 - src_length
11720 * c_rarg3 - dest_start
11721 * c_rarg4 - dest_offset
11722 * c_rarg5 - isURL
11723 *
11724 */
11725 address generate_base64_encodeBlock() {
11726
11727 StubId stub_id = StubId::stubgen_base64_encodeBlock_id;
11728 int entry_count = StubInfo::entry_count(stub_id);
11729 assert(entry_count == 1, "sanity check");
11730 address start = load_archive_data(stub_id);
11731 if (start != nullptr) {
11732 return start;
11733 }
11734 __ align(CodeEntryAlignment);
11735 StubCodeMark mark(this, stub_id);
11736 start = __ pc();
11737
11738 Register src = c_rarg0; // source array
11739 Register soff = c_rarg1; // source start offset
11740 Register send = c_rarg2; // source end offset
11741 Register dst = c_rarg3; // dest array
11742 Register doff = c_rarg4; // position for writing to dest array
11743 Register isURL = c_rarg5; // Base64 or URL character set
11744
11745 // c_rarg6 and c_rarg7 are free to use as temps
11746 Register codec = c_rarg6;
11747 Register length = c_rarg7;
11748
11749 Label ProcessData, Process48B, Process24B, Process3B, SIMDExit, Exit;
11750
11751 __ add(src, src, soff);
11752 __ add(dst, dst, doff);
11753 __ sub(length, send, soff);
11754
11755 // load the codec base address
11756 __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64));
11757 __ cbz(isURL, ProcessData);
11758 __ lea(codec, ExternalAddress((address) _encodeBlock_toBase64URL));
11759
11760 __ BIND(ProcessData);
11761
11762 // too short to formup a SIMD loop, roll back
11763 __ cmp(length, (u1)24);
11764 __ br(Assembler::LT, Process3B);
11765
11766 __ ld1(v0, v1, v2, v3, __ T16B, Address(codec));
11767
11768 __ BIND(Process48B);
11769 __ cmp(length, (u1)48);
11770 __ br(Assembler::LT, Process24B);
11771 generate_base64_encode_simdround(src, dst, v0, 16);
11772 __ sub(length, length, 48);
11773 __ b(Process48B);
11774
11775 __ BIND(Process24B);
11776 __ cmp(length, (u1)24);
11777 __ br(Assembler::LT, SIMDExit);
11778 generate_base64_encode_simdround(src, dst, v0, 8);
11779 __ sub(length, length, 24);
11780
11781 __ BIND(SIMDExit);
11782 __ cbz(length, Exit);
11783
11784 __ BIND(Process3B);
11785 // 3 src bytes, 24 bits
11786 __ ldrb(r10, __ post(src, 1));
11787 __ ldrb(r11, __ post(src, 1));
11788 __ ldrb(r12, __ post(src, 1));
11789 __ orrw(r11, r11, r10, Assembler::LSL, 8);
11790 __ orrw(r12, r12, r11, Assembler::LSL, 8);
11791 // codec index
11792 __ ubfmw(r15, r12, 18, 23);
11793 __ ubfmw(r14, r12, 12, 17);
11794 __ ubfmw(r13, r12, 6, 11);
11795 __ andw(r12, r12, 63);
11796 // get the code based on the codec
11797 __ ldrb(r15, Address(codec, r15, Address::uxtw(0)));
11798 __ ldrb(r14, Address(codec, r14, Address::uxtw(0)));
11799 __ ldrb(r13, Address(codec, r13, Address::uxtw(0)));
11800 __ ldrb(r12, Address(codec, r12, Address::uxtw(0)));
11801 __ strb(r15, __ post(dst, 1));
11802 __ strb(r14, __ post(dst, 1));
11803 __ strb(r13, __ post(dst, 1));
11804 __ strb(r12, __ post(dst, 1));
11805 __ sub(length, length, 3);
11806 __ cbnz(length, Process3B);
11807
11808 __ BIND(Exit);
11809 __ ret(lr);
11810
11811 // record the stub entry and end
11812 store_archive_data(stub_id, start, __ pc());
11813
11814 return start;
11815 }
11816
11817 void generate_base64_decode_simdround(Register src, Register dst,
11818 FloatRegister codecL, FloatRegister codecH, int size, Label& Exit) {
11819
11820 FloatRegister in0 = v16, in1 = v17, in2 = v18, in3 = v19;
11821 FloatRegister out0 = v20, out1 = v21, out2 = v22;
11822
11823 FloatRegister decL0 = v23, decL1 = v24, decL2 = v25, decL3 = v26;
11824 FloatRegister decH0 = v28, decH1 = v29, decH2 = v30, decH3 = v31;
11825
11826 Label NoIllegalData, ErrorInLowerHalf, StoreLegalData;
11827
11828 Assembler::SIMD_Arrangement arrangement = size == 16 ? __ T16B : __ T8B;
11829
11830 __ ld4(in0, in1, in2, in3, arrangement, __ post(src, 4 * size));
11831
11832 // we need unsigned saturating subtract, to make sure all input values
11833 // in range [0, 63] will have 0U value in the higher half lookup
11834 __ uqsubv(decH0, __ T16B, in0, v27);
11835 __ uqsubv(decH1, __ T16B, in1, v27);
11836 __ uqsubv(decH2, __ T16B, in2, v27);
11837 __ uqsubv(decH3, __ T16B, in3, v27);
11838
11839 // lower half lookup
11840 __ tbl(decL0, arrangement, codecL, 4, in0);
11841 __ tbl(decL1, arrangement, codecL, 4, in1);
11842 __ tbl(decL2, arrangement, codecL, 4, in2);
11843 __ tbl(decL3, arrangement, codecL, 4, in3);
11844
11845 // higher half lookup
11846 __ tbx(decH0, arrangement, codecH, 4, decH0);
11847 __ tbx(decH1, arrangement, codecH, 4, decH1);
11848 __ tbx(decH2, arrangement, codecH, 4, decH2);
11849 __ tbx(decH3, arrangement, codecH, 4, decH3);
11850
11851 // combine lower and higher
11852 __ orr(decL0, arrangement, decL0, decH0);
11853 __ orr(decL1, arrangement, decL1, decH1);
11854 __ orr(decL2, arrangement, decL2, decH2);
11855 __ orr(decL3, arrangement, decL3, decH3);
11856
11857 // check illegal inputs, value larger than 63 (maximum of 6 bits)
11858 __ cm(Assembler::HI, decH0, arrangement, decL0, v27);
11859 __ cm(Assembler::HI, decH1, arrangement, decL1, v27);
11860 __ cm(Assembler::HI, decH2, arrangement, decL2, v27);
11861 __ cm(Assembler::HI, decH3, arrangement, decL3, v27);
11862 __ orr(in0, arrangement, decH0, decH1);
11863 __ orr(in1, arrangement, decH2, decH3);
11864 __ orr(in2, arrangement, in0, in1);
11865 __ umaxv(in3, arrangement, in2);
11866 __ umov(rscratch2, in3, __ B, 0);
11867
11868 // get the data to output
11869 __ shl(out0, arrangement, decL0, 2);
11870 __ ushr(out1, arrangement, decL1, 4);
11871 __ orr(out0, arrangement, out0, out1);
11872 __ shl(out1, arrangement, decL1, 4);
11873 __ ushr(out2, arrangement, decL2, 2);
11874 __ orr(out1, arrangement, out1, out2);
11875 __ shl(out2, arrangement, decL2, 6);
11876 __ orr(out2, arrangement, out2, decL3);
11877
11878 __ cbz(rscratch2, NoIllegalData);
11879
11880 // handle illegal input
11881 __ umov(r10, in2, __ D, 0);
11882 if (size == 16) {
11883 __ cbnz(r10, ErrorInLowerHalf);
11884
11885 // illegal input is in higher half, store the lower half now.
11886 __ st3(out0, out1, out2, __ T8B, __ post(dst, 24));
11887
11888 __ umov(r10, in2, __ D, 1);
11889 __ umov(r11, out0, __ D, 1);
11890 __ umov(r12, out1, __ D, 1);
11891 __ umov(r13, out2, __ D, 1);
11892 __ b(StoreLegalData);
11893
11894 __ BIND(ErrorInLowerHalf);
11895 }
11896 __ umov(r11, out0, __ D, 0);
11897 __ umov(r12, out1, __ D, 0);
11898 __ umov(r13, out2, __ D, 0);
11899
11900 __ BIND(StoreLegalData);
11901 __ tbnz(r10, 5, Exit); // 0xff indicates illegal input
11902 __ strb(r11, __ post(dst, 1));
11903 __ strb(r12, __ post(dst, 1));
11904 __ strb(r13, __ post(dst, 1));
11905 __ lsr(r10, r10, 8);
11906 __ lsr(r11, r11, 8);
11907 __ lsr(r12, r12, 8);
11908 __ lsr(r13, r13, 8);
11909 __ b(StoreLegalData);
11910
11911 __ BIND(NoIllegalData);
11912 __ st3(out0, out1, out2, arrangement, __ post(dst, 3 * size));
11913 }
11914
11915
11916 /**
11917 * Arguments:
11918 *
11919 * Input:
11920 * c_rarg0 - src_start
11921 * c_rarg1 - src_offset
11922 * c_rarg2 - src_length
11923 * c_rarg3 - dest_start
11924 * c_rarg4 - dest_offset
11925 * c_rarg5 - isURL
11926 * c_rarg6 - isMIME
11927 *
11928 */
11929 address generate_base64_decodeBlock() {
11930
11931 // The SIMD part of this Base64 decode intrinsic is based on the algorithm outlined
11932 // on http://0x80.pl/articles/base64-simd-neon.html#encoding-quadwords, in section
11933 // titled "Base64 decoding".
11934
11935 StubId stub_id = StubId::stubgen_base64_decodeBlock_id;
11936 int entry_count = StubInfo::entry_count(stub_id);
11937 assert(entry_count == 1, "sanity check");
11938 address start = load_archive_data(stub_id);
11939 if (start != nullptr) {
11940 return start;
11941 }
11942 __ align(CodeEntryAlignment);
11943 StubCodeMark mark(this, stub_id);
11944 start = __ pc();
11945
11946 Register src = c_rarg0; // source array
11947 Register soff = c_rarg1; // source start offset
11948 Register send = c_rarg2; // source end offset
11949 Register dst = c_rarg3; // dest array
11950 Register doff = c_rarg4; // position for writing to dest array
11951 Register isURL = c_rarg5; // Base64 or URL character set
11952 Register isMIME = c_rarg6; // Decoding MIME block - unused in this implementation
11953
11954 Register length = send; // reuse send as length of source data to process
11955
11956 Register simd_codec = c_rarg6;
11957 Register nosimd_codec = c_rarg7;
11958
11959 Label ProcessData, Process64B, Process32B, Process4B, SIMDEnter, SIMDExit, Exit;
11960
11961 __ enter();
11962
11963 __ add(src, src, soff);
11964 __ add(dst, dst, doff);
11965
11966 __ mov(doff, dst);
11967
11968 __ sub(length, send, soff);
11969 __ bfm(length, zr, 0, 1);
11970
11971 __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForNoSIMD));
11972 __ cbz(isURL, ProcessData);
11973 __ lea(nosimd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForNoSIMD));
11974
11975 __ BIND(ProcessData);
11976 __ mov(rscratch1, length);
11977 __ cmp(length, (u1)144); // 144 = 80 + 64
11978 __ br(Assembler::LT, Process4B);
11979
11980 // In the MIME case, the line length cannot be more than 76
11981 // bytes (see RFC 2045). This is too short a block for SIMD
11982 // to be worthwhile, so we use non-SIMD here.
11983 __ movw(rscratch1, 79);
11984
11985 __ BIND(Process4B);
11986 __ ldrw(r14, __ post(src, 4));
11987 __ ubfxw(r10, r14, 0, 8);
11988 __ ubfxw(r11, r14, 8, 8);
11989 __ ubfxw(r12, r14, 16, 8);
11990 __ ubfxw(r13, r14, 24, 8);
11991 // get the de-code
11992 __ ldrb(r10, Address(nosimd_codec, r10, Address::uxtw(0)));
11993 __ ldrb(r11, Address(nosimd_codec, r11, Address::uxtw(0)));
11994 __ ldrb(r12, Address(nosimd_codec, r12, Address::uxtw(0)));
11995 __ ldrb(r13, Address(nosimd_codec, r13, Address::uxtw(0)));
11996 // error detection, 255u indicates an illegal input
11997 __ orrw(r14, r10, r11);
11998 __ orrw(r15, r12, r13);
11999 __ orrw(r14, r14, r15);
12000 __ tbnz(r14, 7, Exit);
12001 // recover the data
12002 __ lslw(r14, r10, 10);
12003 __ bfiw(r14, r11, 4, 6);
12004 __ bfmw(r14, r12, 2, 5);
12005 __ rev16w(r14, r14);
12006 __ bfiw(r13, r12, 6, 2);
12007 __ strh(r14, __ post(dst, 2));
12008 __ strb(r13, __ post(dst, 1));
12009 // non-simd loop
12010 __ subsw(rscratch1, rscratch1, 4);
12011 __ br(Assembler::GT, Process4B);
12012
12013 // if exiting from PreProcess80B, rscratch1 == -1;
12014 // otherwise, rscratch1 == 0.
12015 __ cbzw(rscratch1, Exit);
12016 __ sub(length, length, 80);
12017
12018 __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64ForSIMD));
12019 __ cbz(isURL, SIMDEnter);
12020 __ lea(simd_codec, ExternalAddress((address) _decodeBlock_fromBase64URLForSIMD));
12021
12022 __ BIND(SIMDEnter);
12023 __ ld1(v0, v1, v2, v3, __ T16B, __ post(simd_codec, 64));
12024 __ ld1(v4, v5, v6, v7, __ T16B, Address(simd_codec));
12025 __ mov(rscratch1, 63);
12026 __ dup(v27, __ T16B, rscratch1);
12027
12028 __ BIND(Process64B);
12029 __ cmp(length, (u1)64);
12030 __ br(Assembler::LT, Process32B);
12031 generate_base64_decode_simdround(src, dst, v0, v4, 16, Exit);
12032 __ sub(length, length, 64);
12033 __ b(Process64B);
12034
12035 __ BIND(Process32B);
12036 __ cmp(length, (u1)32);
12037 __ br(Assembler::LT, SIMDExit);
12038 generate_base64_decode_simdround(src, dst, v0, v4, 8, Exit);
12039 __ sub(length, length, 32);
12040 __ b(Process32B);
12041
12042 __ BIND(SIMDExit);
12043 __ cbz(length, Exit);
12044 __ movw(rscratch1, length);
12045 __ b(Process4B);
12046
12047 __ BIND(Exit);
12048 __ sub(c_rarg0, dst, doff);
12049
12050 __ leave();
12051 __ ret(lr);
12052
12053 // record the stub entry and end
12054 store_archive_data(stub_id, start, __ pc());
12055
12056 return start;
12057 }
12058
12059 // Support for spin waits.
12060 address generate_spin_wait() {
12061 StubId stub_id = StubId::stubgen_spin_wait_id;
12062 int entry_count = StubInfo::entry_count(stub_id);
12063 assert(entry_count == 1, "sanity check");
12064 address start = load_archive_data(stub_id);
12065 if (start != nullptr) {
12066 return start;
12067 }
12068 __ align(CodeEntryAlignment);
12069 StubCodeMark mark(this, stub_id);
12070 start = __ pc();
12071
12072 __ spin_wait();
12073 __ ret(lr);
12074
12075 // record the stub entry and end
12076 store_archive_data(stub_id, start, __ pc());
12077
12078 return start;
12079 }
12080
12081 void generate_lookup_secondary_supers_table_stub() {
12082 StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id;
12083 GrowableArray<address> entries;
12084 int entry_count = StubInfo::entry_count(stub_id);
12085 assert(entry_count == Klass::SECONDARY_SUPERS_TABLE_SIZE, "sanity check");
12086 address start = load_archive_data(stub_id, &entries);
12087 if (start != nullptr) {
12088 assert(entries.length() == Klass::SECONDARY_SUPERS_TABLE_SIZE - 1,
12089 "unexpected extra entry count %d", entries.length());
12090 StubRoutines::_lookup_secondary_supers_table_stubs[0] = start;
12091 for (int slot = 1; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
12092 StubRoutines::_lookup_secondary_supers_table_stubs[slot] = entries.at(slot - 1);
12093 }
12094 return;
12095 }
12096
12097 StubCodeMark mark(this, stub_id);
12098
12099 const Register
12100 r_super_klass = r0,
12101 r_array_base = r1,
12102 r_array_length = r2,
12103 r_array_index = r3,
12104 r_sub_klass = r4,
12105 r_bitmap = rscratch2,
12106 result = r5;
12107 const FloatRegister
12108 vtemp = v0;
12109
12110 for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
12111 address next_entry = __ pc();
12112 StubRoutines::_lookup_secondary_supers_table_stubs[slot] = next_entry;
12113 if (slot == 0) {
12114 start = next_entry;
12115 } else {
12116 entries.append(next_entry);
12117 }
12118 Label L_success;
12119 __ enter();
12120 __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass,
12121 r_array_base, r_array_length, r_array_index,
12122 vtemp, result, slot,
12123 /*stub_is_near*/true);
12124 __ leave();
12125 __ ret(lr);
12126 }
12127 // record the stub entry and end plus all the auxiliary entries
12128 store_archive_data(stub_id, start, __ pc(), &entries);
12129 }
12130
12131 // Slow path implementation for UseSecondarySupersTable.
12132 address generate_lookup_secondary_supers_table_slow_path_stub() {
12133 StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id;
12134 int entry_count = StubInfo::entry_count(stub_id);
12135 assert(entry_count == 1, "sanity check");
12136 address start = load_archive_data(stub_id);
12137 if (start != nullptr) {
12138 return start;
12139 }
12140 StubCodeMark mark(this, stub_id);
12141 start = __ pc();
12142 const Register
12143 r_super_klass = r0, // argument
12144 r_array_base = r1, // argument
12145 temp1 = r2, // temp
12146 r_array_index = r3, // argument
12147 r_bitmap = rscratch2, // argument
12148 result = r5; // argument
12149
12150 __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, result);
12151 __ ret(lr);
12152
12153 // record the stub entry and end
12154 store_archive_data(stub_id, start, __ pc());
12155
12156 return start;
12157 }
12158
12159 #if defined (LINUX) && !defined (__ARM_FEATURE_ATOMICS)
12160
12161 // ARMv8.1 LSE versions of the atomic stubs used by AtomicAccess::PlatformXX.
12162 //
12163 // If LSE is in use, generate LSE versions of all the stubs. The
12164 // non-LSE versions are in atomic_aarch64.S.
12165
12166 // class AtomicStubMark records the entry point of a stub and the
12167 // stub pointer which will point to it. The stub pointer is set to
12168 // the entry point when ~AtomicStubMark() is called, which must be
12169 // after ICache::invalidate_range. This ensures safe publication of
12170 // the generated code.
12171 class AtomicStubMark {
12172 address _entry_point;
12173 aarch64_atomic_stub_t *_stub;
12174 MacroAssembler *_masm;
12175 public:
12176 AtomicStubMark(MacroAssembler *masm, aarch64_atomic_stub_t *stub) {
12177 _masm = masm;
12178 __ align(32);
12179 _entry_point = __ pc();
12180 _stub = stub;
12181 }
12182 ~AtomicStubMark() {
12183 *_stub = (aarch64_atomic_stub_t)_entry_point;
12184 }
12185 };
12186
12187 // NB: For memory_order_conservative we need a trailing membar after
12188 // LSE atomic operations but not a leading membar.
12189 //
12190 // We don't need a leading membar because a clause in the Arm ARM
12191 // says:
12192 //
12193 // Barrier-ordered-before
12194 //
12195 // Barrier instructions order prior Memory effects before subsequent
12196 // Memory effects generated by the same Observer. A read or a write
12197 // RW1 is Barrier-ordered-before a read or a write RW 2 from the same
12198 // Observer if and only if RW1 appears in program order before RW 2
12199 // and [ ... ] at least one of RW 1 and RW 2 is generated by an atomic
12200 // instruction with both Acquire and Release semantics.
12201 //
12202 // All the atomic instructions {ldaddal, swapal, casal} have Acquire
12203 // and Release semantics, therefore we don't need a leading
12204 // barrier. However, there is no corresponding Barrier-ordered-after
12205 // relationship, therefore we need a trailing membar to prevent a
12206 // later store or load from being reordered with the store in an
12207 // atomic instruction.
12208 //
12209 // This was checked by using the herd7 consistency model simulator
12210 // (http://diy.inria.fr/) with this test case:
12211 //
12212 // AArch64 LseCas
12213 // { 0:X1=x; 0:X2=y; 1:X1=x; 1:X2=y; }
12214 // P0 | P1;
12215 // LDR W4, [X2] | MOV W3, #0;
12216 // DMB LD | MOV W4, #1;
12217 // LDR W3, [X1] | CASAL W3, W4, [X1];
12218 // | DMB ISH;
12219 // | STR W4, [X2];
12220 // exists
12221 // (0:X3=0 /\ 0:X4=1)
12222 //
12223 // If X3 == 0 && X4 == 1, the store to y in P1 has been reordered
12224 // with the store to x in P1. Without the DMB in P1 this may happen.
12225 //
12226 // At the time of writing we don't know of any AArch64 hardware that
12227 // reorders stores in this way, but the Reference Manual permits it.
12228
12229 void gen_cas_entry(Assembler::operand_size size,
12230 atomic_memory_order order) {
12231 Register prev = r3, ptr = c_rarg0, compare_val = c_rarg1,
12232 exchange_val = c_rarg2;
12233 bool acquire, release;
12234 switch (order) {
12235 case memory_order_relaxed:
12236 acquire = false;
12237 release = false;
12238 break;
12239 case memory_order_release:
12240 acquire = false;
12241 release = true;
12242 break;
12243 default:
12244 acquire = true;
12245 release = true;
12246 break;
12247 }
12248 __ mov(prev, compare_val);
12249 __ lse_cas(prev, exchange_val, ptr, size, acquire, release, /*not_pair*/true);
12250 if (order == memory_order_conservative) {
12251 __ membar(Assembler::StoreStore|Assembler::StoreLoad);
12252 }
12253 if (size == Assembler::xword) {
12254 __ mov(r0, prev);
12255 } else {
12256 __ movw(r0, prev);
12257 }
12258 __ ret(lr);
12259 }
12260
12261 void gen_ldadd_entry(Assembler::operand_size size, atomic_memory_order order) {
12262 Register prev = r2, addr = c_rarg0, incr = c_rarg1;
12263 // If not relaxed, then default to conservative. Relaxed is the only
12264 // case we use enough to be worth specializing.
12265 if (order == memory_order_relaxed) {
12266 __ ldadd(size, incr, prev, addr);
12267 } else {
12268 __ ldaddal(size, incr, prev, addr);
12269 __ membar(Assembler::StoreStore|Assembler::StoreLoad);
12270 }
12271 if (size == Assembler::xword) {
12272 __ mov(r0, prev);
12273 } else {
12274 __ movw(r0, prev);
12275 }
12276 __ ret(lr);
12277 }
12278
12279 void gen_swpal_entry(Assembler::operand_size size) {
12280 Register prev = r2, addr = c_rarg0, incr = c_rarg1;
12281 __ swpal(size, incr, prev, addr);
12282 __ membar(Assembler::StoreStore|Assembler::StoreLoad);
12283 if (size == Assembler::xword) {
12284 __ mov(r0, prev);
12285 } else {
12286 __ movw(r0, prev);
12287 }
12288 __ ret(lr);
12289 }
12290
12291 void generate_atomic_entry_points() {
12292 if (! UseLSE) {
12293 return;
12294 }
12295 StubId stub_id = StubId::stubgen_atomic_entry_points_id;
12296 GrowableArray<address> entries;
12297 int entry_count = StubInfo::entry_count(stub_id);
12298 address start = load_archive_data(stub_id, &entries);
12299 if (start != nullptr) {
12300 assert(entries.length() == entry_count - 1,
12301 "unexpected extra entry count %d", entries.length());
12302 aarch64_atomic_fetch_add_4_impl = (aarch64_atomic_stub_t)start;
12303 int idx = 0;
12304 aarch64_atomic_fetch_add_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12305 aarch64_atomic_fetch_add_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12306 aarch64_atomic_fetch_add_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12307 aarch64_atomic_xchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12308 aarch64_atomic_xchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12309 aarch64_atomic_cmpxchg_1_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12310 aarch64_atomic_cmpxchg_4_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12311 aarch64_atomic_cmpxchg_8_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12312 aarch64_atomic_cmpxchg_1_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12313 aarch64_atomic_cmpxchg_4_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12314 aarch64_atomic_cmpxchg_8_relaxed_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12315 aarch64_atomic_cmpxchg_4_release_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12316 aarch64_atomic_cmpxchg_8_release_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12317 aarch64_atomic_cmpxchg_4_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12318 aarch64_atomic_cmpxchg_8_seq_cst_impl = (aarch64_atomic_stub_t)entries.at(idx++);
12319 assert(idx == entries.length(), "sanity!");
12320 return;
12321 }
12322
12323 __ align(CodeEntryAlignment);
12324 StubCodeMark mark(this, stub_id);
12325 start = __ pc();
12326 address end;
12327 {
12328 // ADD, memory_order_conservative
12329 AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl);
12330 gen_ldadd_entry(Assembler::word, memory_order_conservative);
12331
12332 AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl);
12333 gen_ldadd_entry(Assembler::xword, memory_order_conservative);
12334
12335 // ADD, memory_order_relaxed
12336 AtomicStubMark mark_fetch_add_4_relaxed
12337 (_masm, &aarch64_atomic_fetch_add_4_relaxed_impl);
12338 gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed);
12339
12340 AtomicStubMark mark_fetch_add_8_relaxed
12341 (_masm, &aarch64_atomic_fetch_add_8_relaxed_impl);
12342 gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed);
12343
12344 // XCHG, memory_order_conservative
12345 AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl);
12346 gen_swpal_entry(Assembler::word);
12347
12348 AtomicStubMark mark_xchg_8(_masm, &aarch64_atomic_xchg_8_impl);
12349 gen_swpal_entry(Assembler::xword);
12350
12351 // CAS, memory_order_conservative
12352 AtomicStubMark mark_cmpxchg_1(_masm, &aarch64_atomic_cmpxchg_1_impl);
12353 gen_cas_entry(MacroAssembler::byte, memory_order_conservative);
12354
12355 AtomicStubMark mark_cmpxchg_4(_masm, &aarch64_atomic_cmpxchg_4_impl);
12356 gen_cas_entry(MacroAssembler::word, memory_order_conservative);
12357
12358 AtomicStubMark mark_cmpxchg_8(_masm, &aarch64_atomic_cmpxchg_8_impl);
12359 gen_cas_entry(MacroAssembler::xword, memory_order_conservative);
12360
12361 // CAS, memory_order_relaxed
12362 AtomicStubMark mark_cmpxchg_1_relaxed
12363 (_masm, &aarch64_atomic_cmpxchg_1_relaxed_impl);
12364 gen_cas_entry(MacroAssembler::byte, memory_order_relaxed);
12365
12366 AtomicStubMark mark_cmpxchg_4_relaxed
12367 (_masm, &aarch64_atomic_cmpxchg_4_relaxed_impl);
12368 gen_cas_entry(MacroAssembler::word, memory_order_relaxed);
12369
12370 AtomicStubMark mark_cmpxchg_8_relaxed
12371 (_masm, &aarch64_atomic_cmpxchg_8_relaxed_impl);
12372 gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
12373
12374 AtomicStubMark mark_cmpxchg_4_release
12375 (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
12376 gen_cas_entry(MacroAssembler::word, memory_order_release);
12377
12378 AtomicStubMark mark_cmpxchg_8_release
12379 (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
12380 gen_cas_entry(MacroAssembler::xword, memory_order_release);
12381
12382 AtomicStubMark mark_cmpxchg_4_seq_cst
12383 (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
12384 gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
12385
12386 AtomicStubMark mark_cmpxchg_8_seq_cst
12387 (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
12388 gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
12389
12390 end = __ pc();
12391
12392 ICache::invalidate_range(start, end - start);
12393 // exit block to force update of AtomicStubMark targets
12394 }
12395
12396 assert(start == (address)aarch64_atomic_fetch_add_4_impl,
12397 "atomic stub should be at start of buffer");
12398 // record the stub start and end plus all the entries saved by the
12399 // AtomicStubMark destructor
12400 entries.append((address)aarch64_atomic_fetch_add_8_impl);
12401 entries.append((address)aarch64_atomic_fetch_add_4_relaxed_impl);
12402 entries.append((address)aarch64_atomic_fetch_add_8_relaxed_impl);
12403 entries.append((address)aarch64_atomic_xchg_4_impl);
12404 entries.append((address)aarch64_atomic_xchg_8_impl);
12405 entries.append((address)aarch64_atomic_cmpxchg_1_impl);
12406 entries.append((address)aarch64_atomic_cmpxchg_4_impl);
12407 entries.append((address)aarch64_atomic_cmpxchg_8_impl);
12408 entries.append((address)aarch64_atomic_cmpxchg_1_relaxed_impl);
12409 entries.append((address)aarch64_atomic_cmpxchg_4_relaxed_impl);
12410 entries.append((address)aarch64_atomic_cmpxchg_8_relaxed_impl);
12411 entries.append((address)aarch64_atomic_cmpxchg_4_release_impl);
12412 entries.append((address)aarch64_atomic_cmpxchg_8_release_impl);
12413 entries.append((address)aarch64_atomic_cmpxchg_4_seq_cst_impl);
12414 entries.append((address)aarch64_atomic_cmpxchg_8_seq_cst_impl);
12415
12416 assert(entries.length() == entry_count - 1,
12417 "unexpected extra entry count %d", entries.length());
12418
12419 store_archive_data(stub_id, start, end, &entries);
12420 }
12421 #endif // LINUX
12422
12423 static void save_return_registers(MacroAssembler* masm) {
12424 if (InlineTypeReturnedAsFields) {
12425 masm->push(RegSet::range(r0, r7), sp);
12426 masm->sub(sp, sp, 4 * wordSize);
12427 masm->st1(v0, v1, v2, v3, masm->T1D, Address(sp));
12428 masm->sub(sp, sp, 4 * wordSize);
12429 masm->st1(v4, v5, v6, v7, masm->T1D, Address(sp));
12430 } else {
12431 masm->fmovd(rscratch1, v0);
12432 masm->stp(rscratch1, r0, Address(masm->pre(sp, -2 * wordSize)));
12433 }
12434 }
12435
12436 static void restore_return_registers(MacroAssembler* masm) {
12437 if (InlineTypeReturnedAsFields) {
12438 masm->ld1(v4, v5, v6, v7, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
12439 masm->ld1(v0, v1, v2, v3, masm->T1D, Address(masm->post(sp, 4 * wordSize)));
12440 masm->pop(RegSet::range(r0, r7), sp);
12441 } else {
12442 masm->ldp(rscratch1, r0, Address(masm->post(sp, 2 * wordSize)));
12443 masm->fmovd(v0, rscratch1);
12444 }
12445 }
12446
12447 address generate_cont_thaw(Continuation::thaw_kind kind) {
12448 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
12449 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
12450
12451 address start = __ pc();
12452
12453 if (return_barrier) {
12454 __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
12455 __ mov(sp, rscratch1);
12456 }
12457 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12458
12459 if (return_barrier) {
12460 // preserve possible return value from a method returning to the return barrier
12461 save_return_registers(_masm);
12462 }
12463
12464 __ movw(c_rarg1, (return_barrier ? 1 : 0));
12465 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
12466 __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
12467
12468 if (return_barrier) {
12469 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12470 restore_return_registers(_masm);
12471 }
12472 assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
12473
12474
12475 Label thaw_success;
12476 // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
12477 __ cbnz(rscratch2, thaw_success);
12478 __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
12479 __ br(rscratch1);
12480 __ bind(thaw_success);
12481
12482 // make room for the thawed frames
12483 __ sub(rscratch1, sp, rscratch2);
12484 __ andr(rscratch1, rscratch1, -16); // align
12485 __ mov(sp, rscratch1);
12486
12487 if (return_barrier) {
12488 // save original return value -- again
12489 save_return_registers(_masm);
12490 }
12491
12492 // If we want, we can templatize thaw by kind, and have three different entries
12493 __ movw(c_rarg1, (uint32_t)kind);
12494
12495 __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
12496 __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
12497
12498 if (return_barrier) {
12499 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
12500 restore_return_registers(_masm);
12501 } else {
12502 __ mov(r0, zr); // return 0 (success) from doYield
12503 }
12504
12505 // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
12506 __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
12507 __ mov(rfp, sp);
12508
12509 if (return_barrier_exception) {
12510 __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
12511 __ authenticate_return_address(c_rarg1);
12512 __ verify_oop(r0);
12513 // save return value containing the exception oop in callee-saved R19
12514 __ mov(r19, r0);
12515
12516 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
12517
12518 // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
12519 // __ reinitialize_ptrue();
12520
12521 // see OptoRuntime::generate_exception_blob: r0 -- exception oop, r3 -- exception pc
12522
12523 __ mov(r1, r0); // the exception handler
12524 __ mov(r0, r19); // restore return value containing the exception oop
12525 __ verify_oop(r0);
12526
12527 __ leave();
12528 __ mov(r3, lr);
12529 __ br(r1); // the exception handler
12530 } else {
12531 // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame
12532 __ leave();
12533 __ ret(lr);
12534 }
12535
12536 return start;
12537 }
12538
12539 address generate_cont_thaw() {
12540 if (!Continuations::enabled()) return nullptr;
12541
12542 StubId stub_id = StubId::stubgen_cont_thaw_id;
12543 int entry_count = StubInfo::entry_count(stub_id);
12544 assert(entry_count == 1, "sanity check");
12545 address start = load_archive_data(stub_id);
12546 if (start != nullptr) {
12547 return start;
12548 }
12549 StubCodeMark mark(this, stub_id);
12550 start = __ pc();
12551 generate_cont_thaw(Continuation::thaw_top);
12552
12553 // record the stub start and end
12554 store_archive_data(stub_id, start, __ pc());
12555
12556 return start;
12557 }
12558
12559 address generate_cont_returnBarrier() {
12560 if (!Continuations::enabled()) return nullptr;
12561
12562 // TODO: will probably need multiple return barriers depending on return type
12563 StubId stub_id = StubId::stubgen_cont_returnBarrier_id;
12564 int entry_count = StubInfo::entry_count(stub_id);
12565 assert(entry_count == 1, "sanity check");
12566 address start = load_archive_data(stub_id);
12567 if (start != nullptr) {
12568 return start;
12569 }
12570 StubCodeMark mark(this, stub_id);
12571 start = __ pc();
12572
12573 generate_cont_thaw(Continuation::thaw_return_barrier);
12574
12575 // record the stub start and end
12576 store_archive_data(stub_id, start, __ pc());
12577
12578 return start;
12579 }
12580
12581 address generate_cont_returnBarrier_exception() {
12582 if (!Continuations::enabled()) return nullptr;
12583
12584 StubId stub_id = StubId::stubgen_cont_returnBarrierExc_id;
12585 int entry_count = StubInfo::entry_count(stub_id);
12586 assert(entry_count == 1, "sanity check");
12587 address start = load_archive_data(stub_id);
12588 if (start != nullptr) {
12589 return start;
12590 }
12591 StubCodeMark mark(this, stub_id);
12592 start = __ pc();
12593
12594 generate_cont_thaw(Continuation::thaw_return_barrier_exception);
12595
12596 // record the stub start and end
12597 store_archive_data(stub_id, start, __ pc());
12598
12599 return start;
12600 }
12601
12602 address generate_cont_preempt_stub() {
12603 if (!Continuations::enabled()) return nullptr;
12604 StubId stub_id = StubId::stubgen_cont_preempt_id;
12605 int entry_count = StubInfo::entry_count(stub_id);
12606 assert(entry_count == 1, "sanity check");
12607 address start = load_archive_data(stub_id);
12608 if (start != nullptr) {
12609 return start;
12610 }
12611 StubCodeMark mark(this, stub_id);
12612 start = __ pc();
12613
12614 __ reset_last_Java_frame(true);
12615
12616 // Set sp to enterSpecial frame, i.e. remove all frames copied into the heap.
12617 __ ldr(rscratch2, Address(rthread, JavaThread::cont_entry_offset()));
12618 __ mov(sp, rscratch2);
12619
12620 Label preemption_cancelled;
12621 __ ldrb(rscratch1, Address(rthread, JavaThread::preemption_cancelled_offset()));
12622 __ cbnz(rscratch1, preemption_cancelled);
12623
12624 // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount.
12625 SharedRuntime::continuation_enter_cleanup(_masm);
12626 __ leave();
12627 __ ret(lr);
12628
12629 // We acquired the monitor after freezing the frames so call thaw to continue execution.
12630 __ bind(preemption_cancelled);
12631 __ strb(zr, Address(rthread, JavaThread::preemption_cancelled_offset()));
12632 __ lea(rfp, Address(sp, checked_cast<int32_t>(ContinuationEntry::size())));
12633 __ lea(rscratch1, ExternalAddress(ContinuationEntry::thaw_call_pc_address()));
12634 __ ldr(rscratch1, Address(rscratch1));
12635 __ br(rscratch1);
12636
12637 // record the stub start and end
12638 store_archive_data(stub_id, start, __ pc());
12639
12640 return start;
12641 }
12642
12643 // In sun.security.util.math.intpoly.IntegerPolynomial1305, integers
12644 // are represented as long[5], with BITS_PER_LIMB = 26.
12645 // Pack five 26-bit limbs into three 64-bit registers.
12646 void pack_26(Register dest0, Register dest1, Register dest2, Register src) {
12647 __ ldp(dest0, rscratch1, Address(src, 0)); // 26 bits
12648 __ add(dest0, dest0, rscratch1, Assembler::LSL, 26); // 26 bits
12649 __ ldp(rscratch1, rscratch2, Address(src, 2 * sizeof (jlong)));
12650 __ add(dest0, dest0, rscratch1, Assembler::LSL, 52); // 12 bits
12651
12652 __ add(dest1, zr, rscratch1, Assembler::LSR, 12); // 14 bits
12653 __ add(dest1, dest1, rscratch2, Assembler::LSL, 14); // 26 bits
12654 __ ldr(rscratch1, Address(src, 4 * sizeof (jlong)));
12655 __ add(dest1, dest1, rscratch1, Assembler::LSL, 40); // 24 bits
12656
12657 if (dest2->is_valid()) {
12658 __ add(dest2, zr, rscratch1, Assembler::LSR, 24); // 2 bits
12659 } else {
12660 #ifdef ASSERT
12661 Label OK;
12662 __ cmp(zr, rscratch1, Assembler::LSR, 24); // 2 bits
12663 __ br(__ EQ, OK);
12664 __ stop("high bits of Poly1305 integer should be zero");
12665 __ should_not_reach_here();
12666 __ bind(OK);
12667 #endif
12668 }
12669 }
12670
12671 // As above, but return only a 128-bit integer, packed into two
12672 // 64-bit registers.
12673 void pack_26(Register dest0, Register dest1, Register src) {
12674 pack_26(dest0, dest1, noreg, src);
12675 }
12676
12677 // Multiply and multiply-accumulate unsigned 64-bit registers.
12678 void wide_mul(Register prod_lo, Register prod_hi, Register n, Register m) {
12679 __ mul(prod_lo, n, m);
12680 __ umulh(prod_hi, n, m);
12681 }
12682 void wide_madd(Register sum_lo, Register sum_hi, Register n, Register m) {
12683 wide_mul(rscratch1, rscratch2, n, m);
12684 __ adds(sum_lo, sum_lo, rscratch1);
12685 __ adc(sum_hi, sum_hi, rscratch2);
12686 }
12687
12688 // Poly1305, RFC 7539
12689
12690 // See https://loup-vaillant.fr/tutorials/poly1305-design for a
12691 // description of the tricks used to simplify and accelerate this
12692 // computation.
12693
12694 address generate_poly1305_processBlocks() {
12695 StubId stub_id = StubId::stubgen_poly1305_processBlocks_id;
12696 int entry_count = StubInfo::entry_count(stub_id);
12697 assert(entry_count == 1, "sanity check");
12698 address start = load_archive_data(stub_id);
12699 if (start != nullptr) {
12700 return start;
12701 }
12702 __ align(CodeEntryAlignment);
12703 StubCodeMark mark(this, stub_id);
12704 start = __ pc();
12705 Label here;
12706 __ enter();
12707 RegSet callee_saved = RegSet::range(r19, r28);
12708 __ push(callee_saved, sp);
12709
12710 RegSetIterator<Register> regs = (RegSet::range(c_rarg0, r28) - r18_tls - rscratch1 - rscratch2).begin();
12711
12712 // Arguments
12713 const Register input_start = *regs, length = *++regs, acc_start = *++regs, r_start = *++regs;
12714
12715 // R_n is the 128-bit randomly-generated key, packed into two
12716 // registers. The caller passes this key to us as long[5], with
12717 // BITS_PER_LIMB = 26.
12718 const Register R_0 = *++regs, R_1 = *++regs;
12719 pack_26(R_0, R_1, r_start);
12720
12721 // RR_n is (R_n >> 2) * 5
12722 const Register RR_0 = *++regs, RR_1 = *++regs;
12723 __ lsr(RR_0, R_0, 2);
12724 __ add(RR_0, RR_0, RR_0, Assembler::LSL, 2);
12725 __ lsr(RR_1, R_1, 2);
12726 __ add(RR_1, RR_1, RR_1, Assembler::LSL, 2);
12727
12728 // U_n is the current checksum
12729 const Register U_0 = *++regs, U_1 = *++regs, U_2 = *++regs;
12730 pack_26(U_0, U_1, U_2, acc_start);
12731
12732 static constexpr int BLOCK_LENGTH = 16;
12733 Label DONE, LOOP;
12734
12735 __ cmp(length, checked_cast<u1>(BLOCK_LENGTH));
12736 __ br(Assembler::LT, DONE); {
12737 __ bind(LOOP);
12738
12739 // S_n is to be the sum of U_n and the next block of data
12740 const Register S_0 = *++regs, S_1 = *++regs, S_2 = *++regs;
12741 __ ldp(S_0, S_1, __ post(input_start, 2 * wordSize));
12742 __ adds(S_0, U_0, S_0);
12743 __ adcs(S_1, U_1, S_1);
12744 __ adc(S_2, U_2, zr);
12745 __ add(S_2, S_2, 1);
12746
12747 const Register U_0HI = *++regs, U_1HI = *++regs;
12748
12749 // NB: this logic depends on some of the special properties of
12750 // Poly1305 keys. In particular, because we know that the top
12751 // four bits of R_0 and R_1 are zero, we can add together
12752 // partial products without any risk of needing to propagate a
12753 // carry out.
12754 wide_mul(U_0, U_0HI, S_0, R_0); wide_madd(U_0, U_0HI, S_1, RR_1); wide_madd(U_0, U_0HI, S_2, RR_0);
12755 wide_mul(U_1, U_1HI, S_0, R_1); wide_madd(U_1, U_1HI, S_1, R_0); wide_madd(U_1, U_1HI, S_2, RR_1);
12756 __ andr(U_2, R_0, 3);
12757 __ mul(U_2, S_2, U_2);
12758
12759 // Recycle registers S_0, S_1, S_2
12760 regs = (regs.remaining() + S_0 + S_1 + S_2).begin();
12761
12762 // Partial reduction mod 2**130 - 5
12763 __ adds(U_1, U_0HI, U_1);
12764 __ adc(U_2, U_1HI, U_2);
12765 // Sum now in U_2:U_1:U_0.
12766 // Dead: U_0HI, U_1HI.
12767 regs = (regs.remaining() + U_0HI + U_1HI).begin();
12768
12769 // U_2:U_1:U_0 += (U_2 >> 2) * 5 in two steps
12770
12771 // First, U_2:U_1:U_0 += (U_2 >> 2)
12772 __ lsr(rscratch1, U_2, 2);
12773 __ andr(U_2, U_2, (u8)3);
12774 __ adds(U_0, U_0, rscratch1);
12775 __ adcs(U_1, U_1, zr);
12776 __ adc(U_2, U_2, zr);
12777 // Second, U_2:U_1:U_0 += (U_2 >> 2) << 2
12778 __ adds(U_0, U_0, rscratch1, Assembler::LSL, 2);
12779 __ adcs(U_1, U_1, zr);
12780 __ adc(U_2, U_2, zr);
12781
12782 __ sub(length, length, checked_cast<u1>(BLOCK_LENGTH));
12783 __ cmp(length, checked_cast<u1>(BLOCK_LENGTH));
12784 __ br(~ Assembler::LT, LOOP);
12785 }
12786
12787 // Further reduce modulo 2^130 - 5
12788 __ lsr(rscratch1, U_2, 2);
12789 __ add(rscratch1, rscratch1, rscratch1, Assembler::LSL, 2); // rscratch1 = U_2 * 5
12790 __ adds(U_0, U_0, rscratch1); // U_0 += U_2 * 5
12791 __ adcs(U_1, U_1, zr);
12792 __ andr(U_2, U_2, (u1)3);
12793 __ adc(U_2, U_2, zr);
12794
12795 // Unpack the sum into five 26-bit limbs and write to memory.
12796 __ ubfiz(rscratch1, U_0, 0, 26);
12797 __ ubfx(rscratch2, U_0, 26, 26);
12798 __ stp(rscratch1, rscratch2, Address(acc_start));
12799 __ ubfx(rscratch1, U_0, 52, 12);
12800 __ bfi(rscratch1, U_1, 12, 14);
12801 __ ubfx(rscratch2, U_1, 14, 26);
12802 __ stp(rscratch1, rscratch2, Address(acc_start, 2 * sizeof (jlong)));
12803 __ ubfx(rscratch1, U_1, 40, 24);
12804 __ bfi(rscratch1, U_2, 24, 3);
12805 __ str(rscratch1, Address(acc_start, 4 * sizeof (jlong)));
12806
12807 __ bind(DONE);
12808 __ pop(callee_saved, sp);
12809 __ leave();
12810 __ ret(lr);
12811
12812 // record the stub start and end
12813 store_archive_data(stub_id, start, __ pc());
12814
12815 return start;
12816 }
12817
12818 // exception handler for upcall stubs
12819 address generate_upcall_stub_exception_handler() {
12820 StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id;
12821 int entry_count = StubInfo::entry_count(stub_id);
12822 assert(entry_count == 1, "sanity check");
12823 address start = load_archive_data(stub_id);
12824 if (start != nullptr) {
12825 return start;
12826 }
12827 StubCodeMark mark(this, stub_id);
12828 start = __ pc();
12829
12830 // Native caller has no idea how to handle exceptions,
12831 // so we just crash here. Up to callee to catch exceptions.
12832 __ verify_oop(r0);
12833 __ movptr(rscratch1, CAST_FROM_FN_PTR(uint64_t, UpcallLinker::handle_uncaught_exception));
12834 __ blr(rscratch1);
12835 __ should_not_reach_here();
12836
12837 // record the stub start and end
12838 store_archive_data(stub_id, start, __ pc());
12839
12840 return start;
12841 }
12842
12843 // load Method* target of MethodHandle
12844 // j_rarg0 = jobject receiver
12845 // rmethod = result
12846 address generate_upcall_stub_load_target() {
12847 StubId stub_id = StubId::stubgen_upcall_stub_load_target_id;
12848 int entry_count = StubInfo::entry_count(stub_id);
12849 assert(entry_count == 1, "sanity check");
12850 address start = load_archive_data(stub_id);
12851 if (start != nullptr) {
12852 return start;
12853 }
12854 StubCodeMark mark(this, stub_id);
12855 start = __ pc();
12856
12857 __ resolve_global_jobject(j_rarg0, rscratch1, rscratch2);
12858 // Load target method from receiver
12859 __ load_heap_oop(rmethod, Address(j_rarg0, java_lang_invoke_MethodHandle::form_offset()), rscratch1, rscratch2);
12860 __ load_heap_oop(rmethod, Address(rmethod, java_lang_invoke_LambdaForm::vmentry_offset()), rscratch1, rscratch2);
12861 __ load_heap_oop(rmethod, Address(rmethod, java_lang_invoke_MemberName::method_offset()), rscratch1, rscratch2);
12862 __ access_load_at(T_ADDRESS, IN_HEAP, rmethod,
12863 Address(rmethod, java_lang_invoke_ResolvedMethodName::vmtarget_offset()),
12864 noreg, noreg);
12865 __ str(rmethod, Address(rthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
12866
12867 __ ret(lr);
12868
12869 // record the stub start and end
12870 store_archive_data(stub_id, start, __ pc());
12871
12872 return start;
12873 }
12874
12875 #undef __
12876 #define __ masm->
12877
12878 class MontgomeryMultiplyGenerator : public MacroAssembler {
12879
12880 Register Pa_base, Pb_base, Pn_base, Pm_base, inv, Rlen, Ra, Rb, Rm, Rn,
12881 Pa, Pb, Pn, Pm, Rhi_ab, Rlo_ab, Rhi_mn, Rlo_mn, t0, t1, t2, Ri, Rj;
12882
12883 RegSet _toSave;
12884 bool _squaring;
12885
12886 public:
12887 MontgomeryMultiplyGenerator (Assembler *as, bool squaring)
12888 : MacroAssembler(as->code()), _squaring(squaring) {
12889
12890 // Register allocation
12891
12892 RegSetIterator<Register> regs = (RegSet::range(r0, r26) - r18_tls).begin();
12893 Pa_base = *regs; // Argument registers
12894 if (squaring)
12895 Pb_base = Pa_base;
12896 else
12897 Pb_base = *++regs;
12898 Pn_base = *++regs;
12899 Rlen= *++regs;
12900 inv = *++regs;
12901 Pm_base = *++regs;
12902
12903 // Working registers:
12904 Ra = *++regs; // The current digit of a, b, n, and m.
12905 Rb = *++regs;
12906 Rm = *++regs;
12907 Rn = *++regs;
12908
12909 Pa = *++regs; // Pointers to the current/next digit of a, b, n, and m.
12910 Pb = *++regs;
12911 Pm = *++regs;
12912 Pn = *++regs;
12913
12914 t0 = *++regs; // Three registers which form a
12915 t1 = *++regs; // triple-precision accumuator.
12916 t2 = *++regs;
12917
12918 Ri = *++regs; // Inner and outer loop indexes.
12919 Rj = *++regs;
12920
12921 Rhi_ab = *++regs; // Product registers: low and high parts
12922 Rlo_ab = *++regs; // of a*b and m*n.
12923 Rhi_mn = *++regs;
12924 Rlo_mn = *++regs;
12925
12926 // r19 and up are callee-saved.
12927 _toSave = RegSet::range(r19, *regs) + Pm_base;
12928 }
12929
12930 private:
12931 void save_regs() {
12932 push(_toSave, sp);
12933 }
12934
12935 void restore_regs() {
12936 pop(_toSave, sp);
12937 }
12938
12939 template <typename T>
12940 void unroll_2(Register count, T block) {
12941 Label loop, end, odd;
12942 tbnz(count, 0, odd);
12943 cbz(count, end);
12944 align(16);
12945 bind(loop);
12946 (this->*block)();
12947 bind(odd);
12948 (this->*block)();
12949 subs(count, count, 2);
12950 br(Assembler::GT, loop);
12951 bind(end);
12952 }
12953
12954 template <typename T>
12955 void unroll_2(Register count, T block, Register d, Register s, Register tmp) {
12956 Label loop, end, odd;
12957 tbnz(count, 0, odd);
12958 cbz(count, end);
12959 align(16);
12960 bind(loop);
12961 (this->*block)(d, s, tmp);
12962 bind(odd);
12963 (this->*block)(d, s, tmp);
12964 subs(count, count, 2);
12965 br(Assembler::GT, loop);
12966 bind(end);
12967 }
12968
12969 void pre1(RegisterOrConstant i) {
12970 block_comment("pre1");
12971 // Pa = Pa_base;
12972 // Pb = Pb_base + i;
12973 // Pm = Pm_base;
12974 // Pn = Pn_base + i;
12975 // Ra = *Pa;
12976 // Rb = *Pb;
12977 // Rm = *Pm;
12978 // Rn = *Pn;
12979 ldr(Ra, Address(Pa_base));
12980 ldr(Rb, Address(Pb_base, i, Address::uxtw(LogBytesPerWord)));
12981 ldr(Rm, Address(Pm_base));
12982 ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
12983 lea(Pa, Address(Pa_base));
12984 lea(Pb, Address(Pb_base, i, Address::uxtw(LogBytesPerWord)));
12985 lea(Pm, Address(Pm_base));
12986 lea(Pn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
12987
12988 // Zero the m*n result.
12989 mov(Rhi_mn, zr);
12990 mov(Rlo_mn, zr);
12991 }
12992
12993 // The core multiply-accumulate step of a Montgomery
12994 // multiplication. The idea is to schedule operations as a
12995 // pipeline so that instructions with long latencies (loads and
12996 // multiplies) have time to complete before their results are
12997 // used. This most benefits in-order implementations of the
12998 // architecture but out-of-order ones also benefit.
12999 void step() {
13000 block_comment("step");
13001 // MACC(Ra, Rb, t0, t1, t2);
13002 // Ra = *++Pa;
13003 // Rb = *--Pb;
13004 umulh(Rhi_ab, Ra, Rb);
13005 mul(Rlo_ab, Ra, Rb);
13006 ldr(Ra, pre(Pa, wordSize));
13007 ldr(Rb, pre(Pb, -wordSize));
13008 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n from the
13009 // previous iteration.
13010 // MACC(Rm, Rn, t0, t1, t2);
13011 // Rm = *++Pm;
13012 // Rn = *--Pn;
13013 umulh(Rhi_mn, Rm, Rn);
13014 mul(Rlo_mn, Rm, Rn);
13015 ldr(Rm, pre(Pm, wordSize));
13016 ldr(Rn, pre(Pn, -wordSize));
13017 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
13018 }
13019
13020 void post1() {
13021 block_comment("post1");
13022
13023 // MACC(Ra, Rb, t0, t1, t2);
13024 // Ra = *++Pa;
13025 // Rb = *--Pb;
13026 umulh(Rhi_ab, Ra, Rb);
13027 mul(Rlo_ab, Ra, Rb);
13028 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n
13029 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
13030
13031 // *Pm = Rm = t0 * inv;
13032 mul(Rm, t0, inv);
13033 str(Rm, Address(Pm));
13034
13035 // MACC(Rm, Rn, t0, t1, t2);
13036 // t0 = t1; t1 = t2; t2 = 0;
13037 umulh(Rhi_mn, Rm, Rn);
13038
13039 #ifndef PRODUCT
13040 // assert(m[i] * n[0] + t0 == 0, "broken Montgomery multiply");
13041 {
13042 mul(Rlo_mn, Rm, Rn);
13043 add(Rlo_mn, t0, Rlo_mn);
13044 Label ok;
13045 cbz(Rlo_mn, ok); {
13046 stop("broken Montgomery multiply");
13047 } bind(ok);
13048 }
13049 #endif
13050 // We have very carefully set things up so that
13051 // m[i]*n[0] + t0 == 0 (mod b), so we don't have to calculate
13052 // the lower half of Rm * Rn because we know the result already:
13053 // it must be -t0. t0 + (-t0) must generate a carry iff
13054 // t0 != 0. So, rather than do a mul and an adds we just set
13055 // the carry flag iff t0 is nonzero.
13056 //
13057 // mul(Rlo_mn, Rm, Rn);
13058 // adds(zr, t0, Rlo_mn);
13059 subs(zr, t0, 1); // Set carry iff t0 is nonzero
13060 adcs(t0, t1, Rhi_mn);
13061 adc(t1, t2, zr);
13062 mov(t2, zr);
13063 }
13064
13065 void pre2(RegisterOrConstant i, RegisterOrConstant len) {
13066 block_comment("pre2");
13067 // Pa = Pa_base + i-len;
13068 // Pb = Pb_base + len;
13069 // Pm = Pm_base + i-len;
13070 // Pn = Pn_base + len;
13071
13072 if (i.is_register()) {
13073 sub(Rj, i.as_register(), len);
13074 } else {
13075 mov(Rj, i.as_constant());
13076 sub(Rj, Rj, len);
13077 }
13078 // Rj == i-len
13079
13080 lea(Pa, Address(Pa_base, Rj, Address::uxtw(LogBytesPerWord)));
13081 lea(Pb, Address(Pb_base, len, Address::uxtw(LogBytesPerWord)));
13082 lea(Pm, Address(Pm_base, Rj, Address::uxtw(LogBytesPerWord)));
13083 lea(Pn, Address(Pn_base, len, Address::uxtw(LogBytesPerWord)));
13084
13085 // Ra = *++Pa;
13086 // Rb = *--Pb;
13087 // Rm = *++Pm;
13088 // Rn = *--Pn;
13089 ldr(Ra, pre(Pa, wordSize));
13090 ldr(Rb, pre(Pb, -wordSize));
13091 ldr(Rm, pre(Pm, wordSize));
13092 ldr(Rn, pre(Pn, -wordSize));
13093
13094 mov(Rhi_mn, zr);
13095 mov(Rlo_mn, zr);
13096 }
13097
13098 void post2(RegisterOrConstant i, RegisterOrConstant len) {
13099 block_comment("post2");
13100 if (i.is_constant()) {
13101 mov(Rj, i.as_constant()-len.as_constant());
13102 } else {
13103 sub(Rj, i.as_register(), len);
13104 }
13105
13106 adds(t0, t0, Rlo_mn); // The pending m*n, low part
13107
13108 // As soon as we know the least significant digit of our result,
13109 // store it.
13110 // Pm_base[i-len] = t0;
13111 str(t0, Address(Pm_base, Rj, Address::uxtw(LogBytesPerWord)));
13112
13113 // t0 = t1; t1 = t2; t2 = 0;
13114 adcs(t0, t1, Rhi_mn); // The pending m*n, high part
13115 adc(t1, t2, zr);
13116 mov(t2, zr);
13117 }
13118
13119 // A carry in t0 after Montgomery multiplication means that we
13120 // should subtract multiples of n from our result in m. We'll
13121 // keep doing that until there is no carry.
13122 void normalize(RegisterOrConstant len) {
13123 block_comment("normalize");
13124 // while (t0)
13125 // t0 = sub(Pm_base, Pn_base, t0, len);
13126 Label loop, post, again;
13127 Register cnt = t1, i = t2; // Re-use registers; we're done with them now
13128 cbz(t0, post); {
13129 bind(again); {
13130 mov(i, zr);
13131 mov(cnt, len);
13132 ldr(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
13133 ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
13134 subs(zr, zr, zr); // set carry flag, i.e. no borrow
13135 align(16);
13136 bind(loop); {
13137 sbcs(Rm, Rm, Rn);
13138 str(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
13139 add(i, i, 1);
13140 ldr(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
13141 ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
13142 sub(cnt, cnt, 1);
13143 } cbnz(cnt, loop);
13144 sbc(t0, t0, zr);
13145 } cbnz(t0, again);
13146 } bind(post);
13147 }
13148
13149 // Move memory at s to d, reversing words.
13150 // Increments d to end of copied memory
13151 // Destroys tmp1, tmp2
13152 // Preserves len
13153 // Leaves s pointing to the address which was in d at start
13154 void reverse(Register d, Register s, Register len, Register tmp1, Register tmp2) {
13155 assert(tmp1->encoding() < r19->encoding(), "register corruption");
13156 assert(tmp2->encoding() < r19->encoding(), "register corruption");
13157
13158 lea(s, Address(s, len, Address::uxtw(LogBytesPerWord)));
13159 mov(tmp1, len);
13160 unroll_2(tmp1, &MontgomeryMultiplyGenerator::reverse1, d, s, tmp2);
13161 sub(s, d, len, ext::uxtw, LogBytesPerWord);
13162 }
13163 // where
13164 void reverse1(Register d, Register s, Register tmp) {
13165 ldr(tmp, pre(s, -wordSize));
13166 ror(tmp, tmp, 32);
13167 str(tmp, post(d, wordSize));
13168 }
13169
13170 void step_squaring() {
13171 // An extra ACC
13172 step();
13173 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
13174 }
13175
13176 void last_squaring(RegisterOrConstant i) {
13177 Label dont;
13178 // if ((i & 1) == 0) {
13179 tbnz(i.as_register(), 0, dont); {
13180 // MACC(Ra, Rb, t0, t1, t2);
13181 // Ra = *++Pa;
13182 // Rb = *--Pb;
13183 umulh(Rhi_ab, Ra, Rb);
13184 mul(Rlo_ab, Ra, Rb);
13185 acc(Rhi_ab, Rlo_ab, t0, t1, t2);
13186 } bind(dont);
13187 }
13188
13189 void extra_step_squaring() {
13190 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n
13191
13192 // MACC(Rm, Rn, t0, t1, t2);
13193 // Rm = *++Pm;
13194 // Rn = *--Pn;
13195 umulh(Rhi_mn, Rm, Rn);
13196 mul(Rlo_mn, Rm, Rn);
13197 ldr(Rm, pre(Pm, wordSize));
13198 ldr(Rn, pre(Pn, -wordSize));
13199 }
13200
13201 void post1_squaring() {
13202 acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n
13203
13204 // *Pm = Rm = t0 * inv;
13205 mul(Rm, t0, inv);
13206 str(Rm, Address(Pm));
13207
13208 // MACC(Rm, Rn, t0, t1, t2);
13209 // t0 = t1; t1 = t2; t2 = 0;
13210 umulh(Rhi_mn, Rm, Rn);
13211
13212 #ifndef PRODUCT
13213 // assert(m[i] * n[0] + t0 == 0, "broken Montgomery multiply");
13214 {
13215 mul(Rlo_mn, Rm, Rn);
13216 add(Rlo_mn, t0, Rlo_mn);
13217 Label ok;
13218 cbz(Rlo_mn, ok); {
13219 stop("broken Montgomery multiply");
13220 } bind(ok);
13221 }
13222 #endif
13223 // We have very carefully set things up so that
13224 // m[i]*n[0] + t0 == 0 (mod b), so we don't have to calculate
13225 // the lower half of Rm * Rn because we know the result already:
13226 // it must be -t0. t0 + (-t0) must generate a carry iff
13227 // t0 != 0. So, rather than do a mul and an adds we just set
13228 // the carry flag iff t0 is nonzero.
13229 //
13230 // mul(Rlo_mn, Rm, Rn);
13231 // adds(zr, t0, Rlo_mn);
13232 subs(zr, t0, 1); // Set carry iff t0 is nonzero
13233 adcs(t0, t1, Rhi_mn);
13234 adc(t1, t2, zr);
13235 mov(t2, zr);
13236 }
13237
13238 void acc(Register Rhi, Register Rlo,
13239 Register t0, Register t1, Register t2) {
13240 adds(t0, t0, Rlo);
13241 adcs(t1, t1, Rhi);
13242 adc(t2, t2, zr);
13243 }
13244
13245 public:
13246 /**
13247 * Fast Montgomery multiplication. The derivation of the
13248 * algorithm is in A Cryptographic Library for the Motorola
13249 * DSP56000, Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237.
13250 *
13251 * Arguments:
13252 *
13253 * Inputs for multiplication:
13254 * c_rarg0 - int array elements a
13255 * c_rarg1 - int array elements b
13256 * c_rarg2 - int array elements n (the modulus)
13257 * c_rarg3 - int length
13258 * c_rarg4 - int inv
13259 * c_rarg5 - int array elements m (the result)
13260 *
13261 * Inputs for squaring:
13262 * c_rarg0 - int array elements a
13263 * c_rarg1 - int array elements n (the modulus)
13264 * c_rarg2 - int length
13265 * c_rarg3 - int inv
13266 * c_rarg4 - int array elements m (the result)
13267 *
13268 */
13269 address generate_multiply() {
13270 Label argh, nothing;
13271
13272 align(CodeEntryAlignment);
13273 address entry = pc();
13274
13275 cbzw(Rlen, nothing);
13276
13277 enter();
13278
13279 // Make room.
13280 cmpw(Rlen, 512);
13281 br(Assembler::HI, argh);
13282 sub(Ra, sp, Rlen, ext::uxtw, exact_log2(4 * sizeof (jint)));
13283 andr(sp, Ra, -2 * wordSize);
13284
13285 lsrw(Rlen, Rlen, 1); // length in longwords = len/2
13286
13287 {
13288 // Copy input args, reversing as we go. We use Ra as a
13289 // temporary variable.
13290 reverse(Ra, Pa_base, Rlen, t0, t1);
13291 if (!_squaring)
13292 reverse(Ra, Pb_base, Rlen, t0, t1);
13293 reverse(Ra, Pn_base, Rlen, t0, t1);
13294 }
13295
13296 // Push all call-saved registers and also Pm_base which we'll need
13297 // at the end.
13298 save_regs();
13299
13300 #ifndef PRODUCT
13301 // assert(inv * n[0] == -1UL, "broken inverse in Montgomery multiply");
13302 {
13303 ldr(Rn, Address(Pn_base, 0));
13304 mul(Rlo_mn, Rn, inv);
13305 subs(zr, Rlo_mn, -1);
13306 Label ok;
13307 br(EQ, ok); {
13308 stop("broken inverse in Montgomery multiply");
13309 } bind(ok);
13310 }
13311 #endif
13312
13313 mov(Pm_base, Ra);
13314
13315 mov(t0, zr);
13316 mov(t1, zr);
13317 mov(t2, zr);
13318
13319 block_comment("for (int i = 0; i < len; i++) {");
13320 mov(Ri, zr); {
13321 Label loop, end;
13322 cmpw(Ri, Rlen);
13323 br(Assembler::GE, end);
13324
13325 bind(loop);
13326 pre1(Ri);
13327
13328 block_comment(" for (j = i; j; j--) {"); {
13329 movw(Rj, Ri);
13330 unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
13331 } block_comment(" } // j");
13332
13333 post1();
13334 addw(Ri, Ri, 1);
13335 cmpw(Ri, Rlen);
13336 br(Assembler::LT, loop);
13337 bind(end);
13338 block_comment("} // i");
13339 }
13340
13341 block_comment("for (int i = len; i < 2*len; i++) {");
13342 mov(Ri, Rlen); {
13343 Label loop, end;
13344 cmpw(Ri, Rlen, Assembler::LSL, 1);
13345 br(Assembler::GE, end);
13346
13347 bind(loop);
13348 pre2(Ri, Rlen);
13349
13350 block_comment(" for (j = len*2-i-1; j; j--) {"); {
13351 lslw(Rj, Rlen, 1);
13352 subw(Rj, Rj, Ri);
13353 subw(Rj, Rj, 1);
13354 unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
13355 } block_comment(" } // j");
13356
13357 post2(Ri, Rlen);
13358 addw(Ri, Ri, 1);
13359 cmpw(Ri, Rlen, Assembler::LSL, 1);
13360 br(Assembler::LT, loop);
13361 bind(end);
13362 }
13363 block_comment("} // i");
13364
13365 normalize(Rlen);
13366
13367 mov(Ra, Pm_base); // Save Pm_base in Ra
13368 restore_regs(); // Restore caller's Pm_base
13369
13370 // Copy our result into caller's Pm_base
13371 reverse(Pm_base, Ra, Rlen, t0, t1);
13372
13373 leave();
13374 bind(nothing);
13375 ret(lr);
13376
13377 // handler for error case
13378 bind(argh);
13379 stop("MontgomeryMultiply total_allocation must be <= 8192");
13380
13381 return entry;
13382 }
13383 // In C, approximately:
13384
13385 // void
13386 // montgomery_multiply(julong Pa_base[], julong Pb_base[],
13387 // julong Pn_base[], julong Pm_base[],
13388 // julong inv, int len) {
13389 // julong t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
13390 // julong *Pa, *Pb, *Pn, *Pm;
13391 // julong Ra, Rb, Rn, Rm;
13392
13393 // int i;
13394
13395 // assert(inv * Pn_base[0] == -1UL, "broken inverse in Montgomery multiply");
13396
13397 // for (i = 0; i < len; i++) {
13398 // int j;
13399
13400 // Pa = Pa_base;
13401 // Pb = Pb_base + i;
13402 // Pm = Pm_base;
13403 // Pn = Pn_base + i;
13404
13405 // Ra = *Pa;
13406 // Rb = *Pb;
13407 // Rm = *Pm;
13408 // Rn = *Pn;
13409
13410 // int iters = i;
13411 // for (j = 0; iters--; j++) {
13412 // assert(Ra == Pa_base[j] && Rb == Pb_base[i-j], "must be");
13413 // MACC(Ra, Rb, t0, t1, t2);
13414 // Ra = *++Pa;
13415 // Rb = *--Pb;
13416 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13417 // MACC(Rm, Rn, t0, t1, t2);
13418 // Rm = *++Pm;
13419 // Rn = *--Pn;
13420 // }
13421
13422 // assert(Ra == Pa_base[i] && Rb == Pb_base[0], "must be");
13423 // MACC(Ra, Rb, t0, t1, t2);
13424 // *Pm = Rm = t0 * inv;
13425 // assert(Rm == Pm_base[i] && Rn == Pn_base[0], "must be");
13426 // MACC(Rm, Rn, t0, t1, t2);
13427
13428 // assert(t0 == 0, "broken Montgomery multiply");
13429
13430 // t0 = t1; t1 = t2; t2 = 0;
13431 // }
13432
13433 // for (i = len; i < 2*len; i++) {
13434 // int j;
13435
13436 // Pa = Pa_base + i-len;
13437 // Pb = Pb_base + len;
13438 // Pm = Pm_base + i-len;
13439 // Pn = Pn_base + len;
13440
13441 // Ra = *++Pa;
13442 // Rb = *--Pb;
13443 // Rm = *++Pm;
13444 // Rn = *--Pn;
13445
13446 // int iters = len*2-i-1;
13447 // for (j = i-len+1; iters--; j++) {
13448 // assert(Ra == Pa_base[j] && Rb == Pb_base[i-j], "must be");
13449 // MACC(Ra, Rb, t0, t1, t2);
13450 // Ra = *++Pa;
13451 // Rb = *--Pb;
13452 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13453 // MACC(Rm, Rn, t0, t1, t2);
13454 // Rm = *++Pm;
13455 // Rn = *--Pn;
13456 // }
13457
13458 // Pm_base[i-len] = t0;
13459 // t0 = t1; t1 = t2; t2 = 0;
13460 // }
13461
13462 // while (t0)
13463 // t0 = sub(Pm_base, Pn_base, t0, len);
13464 // }
13465
13466 /**
13467 * Fast Montgomery squaring. This uses asymptotically 25% fewer
13468 * multiplies than Montgomery multiplication so it should be up to
13469 * 25% faster. However, its loop control is more complex and it
13470 * may actually run slower on some machines.
13471 *
13472 * Arguments:
13473 *
13474 * Inputs:
13475 * c_rarg0 - int array elements a
13476 * c_rarg1 - int array elements n (the modulus)
13477 * c_rarg2 - int length
13478 * c_rarg3 - int inv
13479 * c_rarg4 - int array elements m (the result)
13480 *
13481 */
13482 address generate_square() {
13483 Label argh;
13484
13485 align(CodeEntryAlignment);
13486 address entry = pc();
13487
13488 enter();
13489
13490 // Make room.
13491 cmpw(Rlen, 512);
13492 br(Assembler::HI, argh);
13493 sub(Ra, sp, Rlen, ext::uxtw, exact_log2(4 * sizeof (jint)));
13494 andr(sp, Ra, -2 * wordSize);
13495
13496 lsrw(Rlen, Rlen, 1); // length in longwords = len/2
13497
13498 {
13499 // Copy input args, reversing as we go. We use Ra as a
13500 // temporary variable.
13501 reverse(Ra, Pa_base, Rlen, t0, t1);
13502 reverse(Ra, Pn_base, Rlen, t0, t1);
13503 }
13504
13505 // Push all call-saved registers and also Pm_base which we'll need
13506 // at the end.
13507 save_regs();
13508
13509 mov(Pm_base, Ra);
13510
13511 mov(t0, zr);
13512 mov(t1, zr);
13513 mov(t2, zr);
13514
13515 block_comment("for (int i = 0; i < len; i++) {");
13516 mov(Ri, zr); {
13517 Label loop, end;
13518 bind(loop);
13519 cmp(Ri, Rlen);
13520 br(Assembler::GE, end);
13521
13522 pre1(Ri);
13523
13524 block_comment("for (j = (i+1)/2; j; j--) {"); {
13525 add(Rj, Ri, 1);
13526 lsr(Rj, Rj, 1);
13527 unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
13528 } block_comment(" } // j");
13529
13530 last_squaring(Ri);
13531
13532 block_comment(" for (j = i/2; j; j--) {"); {
13533 lsr(Rj, Ri, 1);
13534 unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
13535 } block_comment(" } // j");
13536
13537 post1_squaring();
13538 add(Ri, Ri, 1);
13539 cmp(Ri, Rlen);
13540 br(Assembler::LT, loop);
13541
13542 bind(end);
13543 block_comment("} // i");
13544 }
13545
13546 block_comment("for (int i = len; i < 2*len; i++) {");
13547 mov(Ri, Rlen); {
13548 Label loop, end;
13549 bind(loop);
13550 cmp(Ri, Rlen, Assembler::LSL, 1);
13551 br(Assembler::GE, end);
13552
13553 pre2(Ri, Rlen);
13554
13555 block_comment(" for (j = (2*len-i-1)/2; j; j--) {"); {
13556 lsl(Rj, Rlen, 1);
13557 sub(Rj, Rj, Ri);
13558 sub(Rj, Rj, 1);
13559 lsr(Rj, Rj, 1);
13560 unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
13561 } block_comment(" } // j");
13562
13563 last_squaring(Ri);
13564
13565 block_comment(" for (j = (2*len-i)/2; j; j--) {"); {
13566 lsl(Rj, Rlen, 1);
13567 sub(Rj, Rj, Ri);
13568 lsr(Rj, Rj, 1);
13569 unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
13570 } block_comment(" } // j");
13571
13572 post2(Ri, Rlen);
13573 add(Ri, Ri, 1);
13574 cmp(Ri, Rlen, Assembler::LSL, 1);
13575
13576 br(Assembler::LT, loop);
13577 bind(end);
13578 block_comment("} // i");
13579 }
13580
13581 normalize(Rlen);
13582
13583 mov(Ra, Pm_base); // Save Pm_base in Ra
13584 restore_regs(); // Restore caller's Pm_base
13585
13586 // Copy our result into caller's Pm_base
13587 reverse(Pm_base, Ra, Rlen, t0, t1);
13588
13589 leave();
13590 ret(lr);
13591
13592 // handler for error case
13593 bind(argh);
13594 stop("MontgomeryMultiply total_allocation must be <= 8192");
13595
13596 return entry;
13597 }
13598 // In C, approximately:
13599
13600 // void
13601 // montgomery_square(julong Pa_base[], julong Pn_base[],
13602 // julong Pm_base[], julong inv, int len) {
13603 // julong t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
13604 // julong *Pa, *Pb, *Pn, *Pm;
13605 // julong Ra, Rb, Rn, Rm;
13606
13607 // int i;
13608
13609 // assert(inv * Pn_base[0] == -1UL, "broken inverse in Montgomery multiply");
13610
13611 // for (i = 0; i < len; i++) {
13612 // int j;
13613
13614 // Pa = Pa_base;
13615 // Pb = Pa_base + i;
13616 // Pm = Pm_base;
13617 // Pn = Pn_base + i;
13618
13619 // Ra = *Pa;
13620 // Rb = *Pb;
13621 // Rm = *Pm;
13622 // Rn = *Pn;
13623
13624 // int iters = (i+1)/2;
13625 // for (j = 0; iters--; j++) {
13626 // assert(Ra == Pa_base[j] && Rb == Pa_base[i-j], "must be");
13627 // MACC2(Ra, Rb, t0, t1, t2);
13628 // Ra = *++Pa;
13629 // Rb = *--Pb;
13630 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13631 // MACC(Rm, Rn, t0, t1, t2);
13632 // Rm = *++Pm;
13633 // Rn = *--Pn;
13634 // }
13635 // if ((i & 1) == 0) {
13636 // assert(Ra == Pa_base[j], "must be");
13637 // MACC(Ra, Ra, t0, t1, t2);
13638 // }
13639 // iters = i/2;
13640 // assert(iters == i-j, "must be");
13641 // for (; iters--; j++) {
13642 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13643 // MACC(Rm, Rn, t0, t1, t2);
13644 // Rm = *++Pm;
13645 // Rn = *--Pn;
13646 // }
13647
13648 // *Pm = Rm = t0 * inv;
13649 // assert(Rm == Pm_base[i] && Rn == Pn_base[0], "must be");
13650 // MACC(Rm, Rn, t0, t1, t2);
13651
13652 // assert(t0 == 0, "broken Montgomery multiply");
13653
13654 // t0 = t1; t1 = t2; t2 = 0;
13655 // }
13656
13657 // for (i = len; i < 2*len; i++) {
13658 // int start = i-len+1;
13659 // int end = start + (len - start)/2;
13660 // int j;
13661
13662 // Pa = Pa_base + i-len;
13663 // Pb = Pa_base + len;
13664 // Pm = Pm_base + i-len;
13665 // Pn = Pn_base + len;
13666
13667 // Ra = *++Pa;
13668 // Rb = *--Pb;
13669 // Rm = *++Pm;
13670 // Rn = *--Pn;
13671
13672 // int iters = (2*len-i-1)/2;
13673 // assert(iters == end-start, "must be");
13674 // for (j = start; iters--; j++) {
13675 // assert(Ra == Pa_base[j] && Rb == Pa_base[i-j], "must be");
13676 // MACC2(Ra, Rb, t0, t1, t2);
13677 // Ra = *++Pa;
13678 // Rb = *--Pb;
13679 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13680 // MACC(Rm, Rn, t0, t1, t2);
13681 // Rm = *++Pm;
13682 // Rn = *--Pn;
13683 // }
13684 // if ((i & 1) == 0) {
13685 // assert(Ra == Pa_base[j], "must be");
13686 // MACC(Ra, Ra, t0, t1, t2);
13687 // }
13688 // iters = (2*len-i)/2;
13689 // assert(iters == len-j, "must be");
13690 // for (; iters--; j++) {
13691 // assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
13692 // MACC(Rm, Rn, t0, t1, t2);
13693 // Rm = *++Pm;
13694 // Rn = *--Pn;
13695 // }
13696 // Pm_base[i-len] = t0;
13697 // t0 = t1; t1 = t2; t2 = 0;
13698 // }
13699
13700 // while (t0)
13701 // t0 = sub(Pm_base, Pn_base, t0, len);
13702 // }
13703 };
13704
13705 // Initialization
13706 void generate_preuniverse_stubs() {
13707 // preuniverse stubs are not needed for aarch64
13708 }
13709
13710 void generate_initial_stubs() {
13711 // Generate initial stubs and initializes the entry points
13712
13713 // entry points that exist in all platforms Note: This is code
13714 // that could be shared among different platforms - however the
13715 // benefit seems to be smaller than the disadvantage of having a
13716 // much more complicated generator structure. See also comment in
13717 // stubRoutines.hpp.
13718
13719 StubRoutines::_forward_exception_entry = generate_forward_exception();
13720
13721 StubRoutines::_call_stub_entry =
13722 generate_call_stub(StubRoutines::_call_stub_return_address);
13723
13724 // is referenced by megamorphic call
13725 StubRoutines::_catch_exception_entry = generate_catch_exception();
13726
13727 // Initialize table for copy memory (arraycopy) check.
13728 if (UnsafeMemoryAccess::_table == nullptr) {
13729 UnsafeMemoryAccess::create_table(8 + 4); // 8 for copyMemory; 4 for setMemory
13730 }
13731
13732 if (UseCRC32Intrinsics) {
13733 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
13734 }
13735
13736 if (UseCRC32CIntrinsics) {
13737 StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
13738 }
13739
13740 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
13741 StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
13742 }
13743
13744 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
13745 StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
13746 }
13747
13748 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
13749 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
13750 StubRoutines::_hf2f = generate_float16ToFloat();
13751 StubRoutines::_f2hf = generate_floatToFloat16();
13752 }
13753 }
13754
13755 void generate_continuation_stubs() {
13756 // Continuation stubs:
13757 StubRoutines::_cont_thaw = generate_cont_thaw();
13758 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
13759 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
13760 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
13761 }
13762
13763 void generate_final_stubs() {
13764 // support for verify_oop (must happen after universe_init)
13765 if (VerifyOops) {
13766 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
13767 }
13768
13769 // arraycopy stubs used by compilers
13770 generate_arraycopy_stubs();
13771
13772 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
13773
13774 StubRoutines::aarch64::_spin_wait = generate_spin_wait();
13775
13776 StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler();
13777 StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target();
13778
13779 #if defined (LINUX) && !defined (__ARM_FEATURE_ATOMICS)
13780
13781 generate_atomic_entry_points();
13782
13783 #endif // LINUX
13784
13785 #ifdef COMPILER2
13786 if (UseSecondarySupersTable) {
13787 StubRoutines::_lookup_secondary_supers_table_slow_path_stub = generate_lookup_secondary_supers_table_slow_path_stub();
13788 if (! InlineSecondarySupersTest) {
13789 generate_lookup_secondary_supers_table_stub();
13790 }
13791 }
13792 #endif
13793
13794 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_setMemory)) {
13795 StubRoutines::_unsafe_setmemory = generate_unsafe_setmemory();
13796 }
13797
13798 StubRoutines::aarch64::set_completed(); // Inidicate that arraycopy and zero_blocks stubs are generated
13799 }
13800
13801 void generate_compiler_stubs() {
13802 #ifdef COMPILER2
13803
13804 if (UseSVE == 0) {
13805 generate_iota_indices(StubId::stubgen_vector_iota_indices_id);
13806 }
13807
13808 // array equals stub for large arrays.
13809 if (!UseSimpleArrayEquals) {
13810 StubRoutines::aarch64::_large_array_equals = generate_large_array_equals();
13811 }
13812
13813 // arrays_hascode stub for large arrays.
13814 StubRoutines::aarch64::_large_arrays_hashcode_boolean = generate_large_arrays_hashcode(T_BOOLEAN);
13815 StubRoutines::aarch64::_large_arrays_hashcode_byte = generate_large_arrays_hashcode(T_BYTE);
13816 StubRoutines::aarch64::_large_arrays_hashcode_char = generate_large_arrays_hashcode(T_CHAR);
13817 StubRoutines::aarch64::_large_arrays_hashcode_int = generate_large_arrays_hashcode(T_INT);
13818 StubRoutines::aarch64::_large_arrays_hashcode_short = generate_large_arrays_hashcode(T_SHORT);
13819
13820 // byte_array_inflate stub for large arrays.
13821 StubRoutines::aarch64::_large_byte_array_inflate = generate_large_byte_array_inflate();
13822
13823 // countPositives stub for large arrays.
13824 StubRoutines::aarch64::_count_positives = generate_count_positives(StubRoutines::aarch64::_count_positives_long);
13825
13826 generate_compare_long_strings();
13827
13828 generate_string_indexof_stubs();
13829
13830 if (UseMultiplyToLenIntrinsic) {
13831 StubRoutines::_multiplyToLen = generate_multiplyToLen();
13832 }
13833
13834 if (UseSquareToLenIntrinsic) {
13835 StubRoutines::_squareToLen = generate_squareToLen();
13836 }
13837
13838 if (UseMulAddIntrinsic) {
13839 StubRoutines::_mulAdd = generate_mulAdd();
13840 }
13841
13842 if (UseSIMDForBigIntegerShiftIntrinsics) {
13843 StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift();
13844 StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift();
13845 }
13846
13847 if (UseMontgomeryMultiplyIntrinsic) {
13848 StubId stub_id = StubId::stubgen_montgomeryMultiply_id;
13849 address start = load_archive_data(stub_id);
13850 if (start == nullptr) {
13851 // we have to generate it
13852 StubCodeMark mark(this, stub_id);
13853 MontgomeryMultiplyGenerator g(_masm, /*squaring*/false);
13854 start = g.generate_multiply();
13855 // record the stub start and end
13856 store_archive_data(stub_id, start, _masm->pc());
13857 }
13858 StubRoutines::_montgomeryMultiply = start;
13859 }
13860
13861 if (UseMontgomerySquareIntrinsic) {
13862 StubId stub_id = StubId::stubgen_montgomerySquare_id;
13863 address start = load_archive_data(stub_id);
13864 if (start == nullptr) {
13865 // we have to generate it
13866 StubCodeMark mark(this, stub_id);
13867 MontgomeryMultiplyGenerator g(_masm, /*squaring*/true);
13868 // We use generate_multiply() rather than generate_square()
13869 // because it's faster for the sizes of modulus we care about.
13870 start = g.generate_multiply();
13871 // record the stub start and end
13872 store_archive_data(stub_id, start, _masm->pc());
13873 }
13874 StubRoutines::_montgomerySquare = start;
13875 }
13876
13877 if (UseChaCha20Intrinsics) {
13878 StubRoutines::_chacha20Block = generate_chacha20Block_blockpar();
13879 }
13880
13881 if (UseIntPolyIntrinsics) {
13882 StubRoutines::_intpoly_montgomeryMult_P256 = generate_intpoly_montgomeryMult_P256();
13883 StubRoutines::_intpoly_assign = generate_intpoly_assign();
13884 }
13885
13886 if (UseKyberIntrinsics) {
13887 StubRoutines::_kyberNtt = generate_kyberNtt();
13888 StubRoutines::_kyberInverseNtt = generate_kyberInverseNtt();
13889 StubRoutines::_kyberNttMult = generate_kyberNttMult();
13890 StubRoutines::_kyberAddPoly_2 = generate_kyberAddPoly_2();
13891 StubRoutines::_kyberAddPoly_3 = generate_kyberAddPoly_3();
13892 StubRoutines::_kyber12To16 = generate_kyber12To16();
13893 StubRoutines::_kyberBarrettReduce = generate_kyberBarrettReduce();
13894 }
13895
13896 if (UseDilithiumIntrinsics) {
13897 StubRoutines::_dilithiumAlmostNtt = generate_dilithiumAlmostNtt();
13898 StubRoutines::_dilithiumAlmostInverseNtt = generate_dilithiumAlmostInverseNtt();
13899 StubRoutines::_dilithiumNttMult = generate_dilithiumNttMult();
13900 StubRoutines::_dilithiumMontMulByConstant = generate_dilithiumMontMulByConstant();
13901 StubRoutines::_dilithiumDecomposePoly = generate_dilithiumDecomposePoly();
13902 }
13903
13904 if (UseBASE64Intrinsics) {
13905 StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock();
13906 StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock();
13907 }
13908
13909 // data cache line writeback
13910 StubRoutines::_data_cache_writeback = generate_data_cache_writeback();
13911 StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
13912
13913 if (UseAESIntrinsics) {
13914 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
13915 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
13916 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
13917 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt();
13918 StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt();
13919 }
13920 if (UseGHASHIntrinsics) {
13921 // StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
13922 StubRoutines::aarch64::_ghash_processBlocks_small = generate_ghash_processBlocks_small();
13923 StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks(StubRoutines::aarch64::_ghash_processBlocks_small);
13924 }
13925 if (UseAESIntrinsics && UseGHASHIntrinsics) {
13926 StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt();
13927 }
13928
13929 if (UseMD5Intrinsics) {
13930 StubRoutines::_md5_implCompress = generate_md5_implCompress(StubId::stubgen_md5_implCompress_id);
13931 StubRoutines::_md5_implCompressMB = generate_md5_implCompress(StubId::stubgen_md5_implCompressMB_id);
13932 }
13933 if (UseSHA1Intrinsics) {
13934 StubRoutines::_sha1_implCompress = generate_sha1_implCompress(StubId::stubgen_sha1_implCompress_id);
13935 StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(StubId::stubgen_sha1_implCompressMB_id);
13936 }
13937 if (UseSHA256Intrinsics) {
13938 StubRoutines::_sha256_implCompress = generate_sha256_implCompress(StubId::stubgen_sha256_implCompress_id);
13939 StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(StubId::stubgen_sha256_implCompressMB_id);
13940 }
13941 if (UseSHA512Intrinsics) {
13942 StubRoutines::_sha512_implCompress = generate_sha512_implCompress(StubId::stubgen_sha512_implCompress_id);
13943 StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(StubId::stubgen_sha512_implCompressMB_id);
13944 }
13945 if (UseSHA3Intrinsics && UseSIMDForSHA3Intrinsic) {
13946 StubRoutines::_double_keccak = generate_double_keccak();
13947 StubRoutines::_sha3_implCompress = generate_sha3_implCompress(StubId::stubgen_sha3_implCompress_id);
13948 StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress(StubId::stubgen_sha3_implCompressMB_id);
13949 } else if (UseSHA3Intrinsics) {
13950 StubRoutines::_sha3_implCompress = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompress_id);
13951 StubRoutines::_sha3_implCompressMB = generate_sha3_implCompress_gpr(StubId::stubgen_sha3_implCompressMB_id);
13952 }
13953
13954 if (UsePoly1305Intrinsics) {
13955 StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks();
13956 }
13957
13958 // The difference between AArch64 vs. x86_64 intrinsics implementation
13959 // include the lack of square() intrinsics; usage caused a 3.3% performance
13960 // degradation due to the efficiencies of the symmetric squaring shape in
13961 // Java vs. the inefficiencies of the leaf calls and the additional cycles
13962 // required for 64 bit multiplication in AArch64.
13963 if (UseIntPoly25519Intrinsics) {
13964 StubRoutines::_intpoly_mult_25519 = generate_intpoly_mult_25519();
13965 }
13966
13967 // generate Adler32 intrinsics code
13968 if (UseAdler32Intrinsics) {
13969 StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
13970 }
13971
13972 #endif // COMPILER2
13973 }
13974
13975 public:
13976 StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
13977 switch(blob_id) {
13978 case BlobId::stubgen_preuniverse_id:
13979 generate_preuniverse_stubs();
13980 break;
13981 case BlobId::stubgen_initial_id:
13982 generate_initial_stubs();
13983 break;
13984 case BlobId::stubgen_continuation_id:
13985 generate_continuation_stubs();
13986 break;
13987 case BlobId::stubgen_compiler_id:
13988 generate_compiler_stubs();
13989 break;
13990 case BlobId::stubgen_final_id:
13991 generate_final_stubs();
13992 break;
13993 default:
13994 fatal("unexpected blob id: %s", StubInfo::name(blob_id));
13995 break;
13996 };
13997 }
13998
13999 #if INCLUDE_CDS
14000 static void init_AOTAddressTable(GrowableArray<address>& external_addresses) {
14001 // external data defined in this file
14002 #define ADD(addr) external_addresses.append((address)(addr));
14003 ADD(_sha256_round_consts);
14004 ADD(_sha512_round_consts);
14005 ADD(_sha3_round_consts);
14006 ADD(_double_keccak_round_consts);
14007 ADD(_modulus_P256);
14008 ADD(_encodeBlock_toBase64);
14009 ADD(_encodeBlock_toBase64URL);
14010 ADD(_decodeBlock_fromBase64ForNoSIMD);
14011 ADD(_decodeBlock_fromBase64URLForNoSIMD);
14012 ADD(_decodeBlock_fromBase64ForSIMD);
14013 ADD(_decodeBlock_fromBase64URLForSIMD);
14014 #undef ADD
14015 }
14016 #endif // INCLUDE_CDS
14017 }; // end class declaration
14018
14019 void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
14020 StubGenerator g(code, blob_id, stub_data);
14021 }
14022
14023 #if INCLUDE_CDS
14024 void StubGenerator_init_AOTAddressTable(GrowableArray<address>& addresses) {
14025 StubGenerator::init_AOTAddressTable(addresses);
14026 }
14027 #endif // INCLUDE_CDS
14028
14029 #if defined (LINUX)
14030
14031 // Define pointers to atomic stubs and initialize them to point to the
14032 // code in atomic_aarch64.S.
14033
14034 #define DEFAULT_ATOMIC_OP(OPNAME, SIZE, RELAXED) \
14035 extern "C" uint64_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl \
14036 (volatile void *ptr, uint64_t arg1, uint64_t arg2); \
14037 aarch64_atomic_stub_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _impl \
14038 = aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl;
14039
14040 DEFAULT_ATOMIC_OP(fetch_add, 4, )
14041 DEFAULT_ATOMIC_OP(fetch_add, 8, )
14042 DEFAULT_ATOMIC_OP(fetch_add, 4, _relaxed)
14043 DEFAULT_ATOMIC_OP(fetch_add, 8, _relaxed)
14044 DEFAULT_ATOMIC_OP(xchg, 4, )
14045 DEFAULT_ATOMIC_OP(xchg, 8, )
14046 DEFAULT_ATOMIC_OP(cmpxchg, 1, )
14047 DEFAULT_ATOMIC_OP(cmpxchg, 4, )
14048 DEFAULT_ATOMIC_OP(cmpxchg, 8, )
14049 DEFAULT_ATOMIC_OP(cmpxchg, 1, _relaxed)
14050 DEFAULT_ATOMIC_OP(cmpxchg, 4, _relaxed)
14051 DEFAULT_ATOMIC_OP(cmpxchg, 8, _relaxed)
14052 DEFAULT_ATOMIC_OP(cmpxchg, 4, _release)
14053 DEFAULT_ATOMIC_OP(cmpxchg, 8, _release)
14054 DEFAULT_ATOMIC_OP(cmpxchg, 4, _seq_cst)
14055 DEFAULT_ATOMIC_OP(cmpxchg, 8, _seq_cst)
14056
14057 #undef DEFAULT_ATOMIC_OP
14058
14059 #endif // LINUX