1 /*
2 * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2025, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2020, 2025, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "asm/macroAssembler.hpp"
28 #include "asm/macroAssembler.inline.hpp"
29 #include "compiler/oopMap.hpp"
30 #include "gc/shared/barrierSet.hpp"
31 #include "gc/shared/barrierSetAssembler.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "memory/universe.hpp"
34 #include "nativeInst_riscv.hpp"
35 #include "oops/instanceOop.hpp"
36 #include "oops/method.hpp"
37 #include "oops/objArrayKlass.hpp"
38 #include "oops/oop.inline.hpp"
39 #include "prims/methodHandles.hpp"
40 #include "prims/upcallLinker.hpp"
41 #include "runtime/continuation.hpp"
42 #include "runtime/continuationEntry.inline.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/handles.inline.hpp"
45 #include "runtime/javaThread.hpp"
46 #include "runtime/sharedRuntime.hpp"
47 #include "runtime/stubCodeGenerator.hpp"
48 #include "runtime/stubRoutines.hpp"
49 #include "utilities/align.hpp"
50 #include "utilities/powerOfTwo.hpp"
51 #ifdef COMPILER2
52 #include "opto/runtime.hpp"
53 #endif
54
55 // Declaration and definition of StubGenerator (no .hpp file).
56 // For a more detailed description of the stub routine structure
57 // see the comment in stubRoutines.hpp
58
59 #undef __
60 #define __ _masm->
61
62 #ifdef PRODUCT
63 #define BLOCK_COMMENT(str) /* nothing */
64 #else
65 #define BLOCK_COMMENT(str) __ block_comment(str)
66 #endif
67
68 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
69
70 // Stub Code definitions
71
72 class StubGenerator: public StubCodeGenerator {
73 private:
74
75 #ifdef PRODUCT
76 #define inc_counter_np(counter) ((void)0)
77 #else
78 void inc_counter_np_(uint& counter) {
79 __ incrementw(ExternalAddress((address)&counter));
80 }
81 #define inc_counter_np(counter) \
82 BLOCK_COMMENT("inc_counter " #counter); \
83 inc_counter_np_(counter);
84 #endif
85
86 // Call stubs are used to call Java from C
87 //
88 // Arguments:
89 // c_rarg0: call wrapper address address
90 // c_rarg1: result address
91 // c_rarg2: result type BasicType
92 // c_rarg3: method Method*
93 // c_rarg4: (interpreter) entry point address
94 // c_rarg5: parameters intptr_t*
95 // c_rarg6: parameter size (in words) int
96 // c_rarg7: thread Thread*
97 //
98 // There is no return from the stub itself as any Java result
99 // is written to result
100 //
101 // we save x1 (ra) as the return PC at the base of the frame and
102 // link x8 (fp) below it as the frame pointer installing sp (x2)
103 // into fp.
104 //
105 // we save x10-x17, which accounts for all the c arguments.
106 //
107 // TODO: strictly do we need to save them all? they are treated as
108 // volatile by C so could we omit saving the ones we are going to
109 // place in global registers (thread? method?) or those we only use
110 // during setup of the Java call?
111 //
112 // we don't need to save x5 which C uses as an indirect result location
113 // return register.
114 //
115 // we don't need to save x6-x7 and x28-x31 which both C and Java treat as
116 // volatile
117 //
118 // we save x9, x18-x27, f8-f9, and f18-f27 which Java uses as temporary
119 // registers and C expects to be callee-save
120 //
121 // so the stub frame looks like this when we enter Java code
122 //
123 // [ return_from_Java ] <--- sp
124 // [ argument word n ]
125 // ...
126 // -35 [ argument word 1 ]
127 // -34 [ saved FRM in Floating-point Control and Status Register ] <--- sp_after_call
128 // -33 [ saved f27 ]
129 // -32 [ saved f26 ]
130 // -31 [ saved f25 ]
131 // -30 [ saved f24 ]
132 // -29 [ saved f23 ]
133 // -28 [ saved f22 ]
134 // -27 [ saved f21 ]
135 // -26 [ saved f20 ]
136 // -25 [ saved f19 ]
137 // -24 [ saved f18 ]
138 // -23 [ saved f9 ]
139 // -22 [ saved f8 ]
140 // -21 [ saved x27 ]
141 // -20 [ saved x26 ]
142 // -19 [ saved x25 ]
143 // -18 [ saved x24 ]
144 // -17 [ saved x23 ]
145 // -16 [ saved x22 ]
146 // -15 [ saved x21 ]
147 // -14 [ saved x20 ]
148 // -13 [ saved x19 ]
149 // -12 [ saved x18 ]
150 // -11 [ saved x9 ]
151 // -10 [ call wrapper (x10) ]
152 // -9 [ result (x11) ]
153 // -8 [ result type (x12) ]
154 // -7 [ method (x13) ]
155 // -6 [ entry point (x14) ]
156 // -5 [ parameters (x15) ]
157 // -4 [ parameter size (x16) ]
158 // -3 [ thread (x17) ]
159 // -2 [ saved fp (x8) ]
160 // -1 [ saved ra (x1) ]
161 // 0 [ ] <--- fp == saved sp (x2)
162
163 // Call stub stack layout word offsets from fp
164 enum call_stub_layout {
165 sp_after_call_off = -34,
166
167 frm_off = sp_after_call_off,
168 f27_off = -33,
169 f26_off = -32,
170 f25_off = -31,
171 f24_off = -30,
172 f23_off = -29,
173 f22_off = -28,
174 f21_off = -27,
175 f20_off = -26,
176 f19_off = -25,
177 f18_off = -24,
178 f9_off = -23,
179 f8_off = -22,
180
181 x27_off = -21,
182 x26_off = -20,
183 x25_off = -19,
184 x24_off = -18,
185 x23_off = -17,
186 x22_off = -16,
187 x21_off = -15,
188 x20_off = -14,
189 x19_off = -13,
190 x18_off = -12,
191 x9_off = -11,
192
193 call_wrapper_off = -10,
194 result_off = -9,
195 result_type_off = -8,
196 method_off = -7,
197 entry_point_off = -6,
198 parameters_off = -5,
199 parameter_size_off = -4,
200 thread_off = -3,
201 fp_f = -2,
202 retaddr_off = -1,
203 };
204
205 address generate_call_stub(address& return_address) {
206 assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 &&
207 (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
208 "adjust this code");
209
210 StubId stub_id = StubId::stubgen_call_stub_id;
211 StubCodeMark mark(this, stub_id);
212 address start = __ pc();
213
214 const Address sp_after_call (fp, sp_after_call_off * wordSize);
215
216 const Address frm_save (fp, frm_off * wordSize);
217 const Address call_wrapper (fp, call_wrapper_off * wordSize);
218 const Address result (fp, result_off * wordSize);
219 const Address result_type (fp, result_type_off * wordSize);
220 const Address method (fp, method_off * wordSize);
221 const Address entry_point (fp, entry_point_off * wordSize);
222 const Address parameters (fp, parameters_off * wordSize);
223 const Address parameter_size(fp, parameter_size_off * wordSize);
224
225 const Address thread (fp, thread_off * wordSize);
226
227 const Address f27_save (fp, f27_off * wordSize);
228 const Address f26_save (fp, f26_off * wordSize);
229 const Address f25_save (fp, f25_off * wordSize);
230 const Address f24_save (fp, f24_off * wordSize);
231 const Address f23_save (fp, f23_off * wordSize);
232 const Address f22_save (fp, f22_off * wordSize);
233 const Address f21_save (fp, f21_off * wordSize);
234 const Address f20_save (fp, f20_off * wordSize);
235 const Address f19_save (fp, f19_off * wordSize);
236 const Address f18_save (fp, f18_off * wordSize);
237 const Address f9_save (fp, f9_off * wordSize);
238 const Address f8_save (fp, f8_off * wordSize);
239
240 const Address x27_save (fp, x27_off * wordSize);
241 const Address x26_save (fp, x26_off * wordSize);
242 const Address x25_save (fp, x25_off * wordSize);
243 const Address x24_save (fp, x24_off * wordSize);
244 const Address x23_save (fp, x23_off * wordSize);
245 const Address x22_save (fp, x22_off * wordSize);
246 const Address x21_save (fp, x21_off * wordSize);
247 const Address x20_save (fp, x20_off * wordSize);
248 const Address x19_save (fp, x19_off * wordSize);
249 const Address x18_save (fp, x18_off * wordSize);
250
251 const Address x9_save (fp, x9_off * wordSize);
252
253 // stub code
254
255 address riscv_entry = __ pc();
256
257 // set up frame and move sp to end of save area
258 __ enter();
259 __ addi(sp, fp, sp_after_call_off * wordSize);
260
261 // save register parameters and Java temporary/global registers
262 // n.b. we save thread even though it gets installed in
263 // xthread because we want to sanity check tp later
264 __ sd(c_rarg7, thread);
265 __ sw(c_rarg6, parameter_size);
266 __ sd(c_rarg5, parameters);
267 __ sd(c_rarg4, entry_point);
268 __ sd(c_rarg3, method);
269 __ sd(c_rarg2, result_type);
270 __ sd(c_rarg1, result);
271 __ sd(c_rarg0, call_wrapper);
272
273 __ sd(x9, x9_save);
274
275 __ sd(x18, x18_save);
276 __ sd(x19, x19_save);
277 __ sd(x20, x20_save);
278 __ sd(x21, x21_save);
279 __ sd(x22, x22_save);
280 __ sd(x23, x23_save);
281 __ sd(x24, x24_save);
282 __ sd(x25, x25_save);
283 __ sd(x26, x26_save);
284 __ sd(x27, x27_save);
285
286 __ fsd(f8, f8_save);
287 __ fsd(f9, f9_save);
288 __ fsd(f18, f18_save);
289 __ fsd(f19, f19_save);
290 __ fsd(f20, f20_save);
291 __ fsd(f21, f21_save);
292 __ fsd(f22, f22_save);
293 __ fsd(f23, f23_save);
294 __ fsd(f24, f24_save);
295 __ fsd(f25, f25_save);
296 __ fsd(f26, f26_save);
297 __ fsd(f27, f27_save);
298
299 __ frrm(t0);
300 __ sd(t0, frm_save);
301 // Set frm to the state we need. We do want Round to Nearest. We
302 // don't want non-IEEE rounding modes.
303 Label skip_fsrmi;
304 guarantee(__ RoundingMode::rne == 0, "must be");
305 __ beqz(t0, skip_fsrmi);
306 __ fsrmi(__ RoundingMode::rne);
307 __ bind(skip_fsrmi);
308
309 // install Java thread in global register now we have saved
310 // whatever value it held
311 __ mv(xthread, c_rarg7);
312
313 // And method
314 __ mv(xmethod, c_rarg3);
315
316 // set up the heapbase register
317 __ reinit_heapbase();
318
319 #ifdef ASSERT
320 // make sure we have no pending exceptions
321 {
322 Label L;
323 __ ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset())));
324 __ beqz(t0, L);
325 __ stop("StubRoutines::call_stub: entered with pending exception");
326 __ BIND(L);
327 }
328 #endif
329 // pass parameters if any
330 __ mv(esp, sp);
331 __ slli(t0, c_rarg6, LogBytesPerWord);
332 __ sub(t0, sp, t0); // Move SP out of the way
333 __ andi(sp, t0, -2 * wordSize);
334
335 BLOCK_COMMENT("pass parameters if any");
336 Label parameters_done;
337 // parameter count is still in c_rarg6
338 // and parameter pointer identifying param 1 is in c_rarg5
339 __ beqz(c_rarg6, parameters_done);
340
341 address loop = __ pc();
342 __ ld(t0, Address(c_rarg5, 0));
343 __ addi(c_rarg5, c_rarg5, wordSize);
344 __ subi(c_rarg6, c_rarg6, 1);
345 __ push_reg(t0);
346 __ bgtz(c_rarg6, loop);
347
348 __ BIND(parameters_done);
349
350 // call Java entry -- passing methdoOop, and current sp
351 // xmethod: Method*
352 // x19_sender_sp: sender sp
353 BLOCK_COMMENT("call Java function");
354 __ mv(x19_sender_sp, sp);
355 __ jalr(c_rarg4);
356
357 // save current address for use by exception handling code
358
359 return_address = __ pc();
360
361 // store result depending on type (everything that is not
362 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
363 // n.b. this assumes Java returns an integral result in x10
364 // and a floating result in j_farg0
365 __ ld(j_rarg2, result);
366 Label is_long, is_float, is_double, exit;
367 __ ld(j_rarg1, result_type);
368 __ mv(t0, (u1)T_OBJECT);
369 __ beq(j_rarg1, t0, is_long);
370 __ mv(t0, (u1)T_LONG);
371 __ beq(j_rarg1, t0, is_long);
372 __ mv(t0, (u1)T_FLOAT);
373 __ beq(j_rarg1, t0, is_float);
374 __ mv(t0, (u1)T_DOUBLE);
375 __ beq(j_rarg1, t0, is_double);
376
377 // handle T_INT case
378 __ sw(x10, Address(j_rarg2));
379
380 __ BIND(exit);
381
382 // pop parameters
383 __ addi(esp, fp, sp_after_call_off * wordSize);
384
385 #ifdef ASSERT
386 // verify that threads correspond
387 {
388 Label L, S;
389 __ ld(t0, thread);
390 __ bne(xthread, t0, S);
391 __ get_thread(t0);
392 __ beq(xthread, t0, L);
393 __ BIND(S);
394 __ stop("StubRoutines::call_stub: threads must correspond");
395 __ BIND(L);
396 }
397 #endif
398
399 __ pop_cont_fastpath(xthread);
400
401 // restore callee-save registers
402 __ fld(f27, f27_save);
403 __ fld(f26, f26_save);
404 __ fld(f25, f25_save);
405 __ fld(f24, f24_save);
406 __ fld(f23, f23_save);
407 __ fld(f22, f22_save);
408 __ fld(f21, f21_save);
409 __ fld(f20, f20_save);
410 __ fld(f19, f19_save);
411 __ fld(f18, f18_save);
412 __ fld(f9, f9_save);
413 __ fld(f8, f8_save);
414
415 __ ld(x27, x27_save);
416 __ ld(x26, x26_save);
417 __ ld(x25, x25_save);
418 __ ld(x24, x24_save);
419 __ ld(x23, x23_save);
420 __ ld(x22, x22_save);
421 __ ld(x21, x21_save);
422 __ ld(x20, x20_save);
423 __ ld(x19, x19_save);
424 __ ld(x18, x18_save);
425
426 __ ld(x9, x9_save);
427
428 // restore frm
429 Label skip_fsrm;
430 __ ld(t0, frm_save);
431 __ frrm(t1);
432 __ beq(t0, t1, skip_fsrm);
433 __ fsrm(t0);
434 __ bind(skip_fsrm);
435
436 __ ld(c_rarg0, call_wrapper);
437 __ ld(c_rarg1, result);
438 __ ld(c_rarg2, result_type);
439 __ ld(c_rarg3, method);
440 __ ld(c_rarg4, entry_point);
441 __ ld(c_rarg5, parameters);
442 __ ld(c_rarg6, parameter_size);
443 __ ld(c_rarg7, thread);
444
445 // leave frame and return to caller
446 __ leave();
447 __ ret();
448
449 // handle return types different from T_INT
450
451 __ BIND(is_long);
452 __ sd(x10, Address(j_rarg2, 0));
453 __ j(exit);
454
455 __ BIND(is_float);
456 __ fsw(j_farg0, Address(j_rarg2, 0), t0);
457 __ j(exit);
458
459 __ BIND(is_double);
460 __ fsd(j_farg0, Address(j_rarg2, 0), t0);
461 __ j(exit);
462
463 return start;
464 }
465
466 // Return point for a Java call if there's an exception thrown in
467 // Java code. The exception is caught and transformed into a
468 // pending exception stored in JavaThread that can be tested from
469 // within the VM.
470 //
471 // Note: Usually the parameters are removed by the callee. In case
472 // of an exception crossing an activation frame boundary, that is
473 // not the case if the callee is compiled code => need to setup the
474 // sp.
475 //
476 // x10: exception oop
477
478 address generate_catch_exception() {
479 StubId stub_id = StubId::stubgen_catch_exception_id;
480 StubCodeMark mark(this, stub_id);
481 address start = __ pc();
482
483 // same as in generate_call_stub():
484 const Address thread(fp, thread_off * wordSize);
485
486 #ifdef ASSERT
487 // verify that threads correspond
488 {
489 Label L, S;
490 __ ld(t0, thread);
491 __ bne(xthread, t0, S);
492 __ get_thread(t0);
493 __ beq(xthread, t0, L);
494 __ bind(S);
495 __ stop("StubRoutines::catch_exception: threads must correspond");
496 __ bind(L);
497 }
498 #endif
499
500 // set pending exception
501 __ verify_oop(x10);
502
503 __ sd(x10, Address(xthread, Thread::pending_exception_offset()));
504 __ mv(t0, (address)__FILE__);
505 __ sd(t0, Address(xthread, Thread::exception_file_offset()));
506 __ mv(t0, (int)__LINE__);
507 __ sw(t0, Address(xthread, Thread::exception_line_offset()));
508
509 // complete return to VM
510 assert(StubRoutines::_call_stub_return_address != nullptr,
511 "_call_stub_return_address must have been generated before");
512 __ j(RuntimeAddress(StubRoutines::_call_stub_return_address));
513
514 return start;
515 }
516
517 // Continuation point for runtime calls returning with a pending
518 // exception. The pending exception check happened in the runtime
519 // or native call stub. The pending exception in Thread is
520 // converted into a Java-level exception.
521 //
522 // Contract with Java-level exception handlers:
523 // x10: exception
524 // x13: throwing pc
525 //
526 // NOTE: At entry of this stub, exception-pc must be in RA !!
527
528 // NOTE: this is always used as a jump target within generated code
529 // so it just needs to be generated code with no x86 prolog
530
531 address generate_forward_exception() {
532 StubId stub_id = StubId::stubgen_forward_exception_id;
533 StubCodeMark mark(this, stub_id);
534 address start = __ pc();
535
536 // Upon entry, RA points to the return address returning into
537 // Java (interpreted or compiled) code; i.e., the return address
538 // becomes the throwing pc.
539 //
540 // Arguments pushed before the runtime call are still on the stack
541 // but the exception handler will reset the stack pointer ->
542 // ignore them. A potential result in registers can be ignored as
543 // well.
544
545 #ifdef ASSERT
546 // make sure this code is only executed if there is a pending exception
547 {
548 Label L;
549 __ ld(t0, Address(xthread, Thread::pending_exception_offset()));
550 __ bnez(t0, L);
551 __ stop("StubRoutines::forward exception: no pending exception (1)");
552 __ bind(L);
553 }
554 #endif
555
556 // compute exception handler into x9
557
558 // call the VM to find the handler address associated with the
559 // caller address. pass thread in x10 and caller pc (ret address)
560 // in x11. n.b. the caller pc is in ra, unlike x86 where it is on
561 // the stack.
562 __ mv(c_rarg1, ra);
563 // ra will be trashed by the VM call so we move it to x9
564 // (callee-saved) because we also need to pass it to the handler
565 // returned by this call.
566 __ mv(x9, ra);
567 BLOCK_COMMENT("call exception_handler_for_return_address");
568 __ call_VM_leaf(CAST_FROM_FN_PTR(address,
569 SharedRuntime::exception_handler_for_return_address),
570 xthread, c_rarg1);
571 // we should not really care that ra is no longer the callee
572 // address. we saved the value the handler needs in x9 so we can
573 // just copy it to x13. however, the C2 handler will push its own
574 // frame and then calls into the VM and the VM code asserts that
575 // the PC for the frame above the handler belongs to a compiled
576 // Java method. So, we restore ra here to satisfy that assert.
577 __ mv(ra, x9);
578 // setup x10 & x13 & clear pending exception
579 __ mv(x13, x9);
580 __ mv(x9, x10);
581 __ ld(x10, Address(xthread, Thread::pending_exception_offset()));
582 __ sd(zr, Address(xthread, Thread::pending_exception_offset()));
583
584 #ifdef ASSERT
585 // make sure exception is set
586 {
587 Label L;
588 __ bnez(x10, L);
589 __ stop("StubRoutines::forward exception: no pending exception (2)");
590 __ bind(L);
591 }
592 #endif
593
594 // continue at exception handler
595 // x10: exception
596 // x13: throwing pc
597 // x9: exception handler
598 __ verify_oop(x10);
599 __ jr(x9);
600
601 return start;
602 }
603
604 // Non-destructive plausibility checks for oops
605 //
606 // Arguments:
607 // x10: oop to verify
608 // t0: error message
609 //
610 // Stack after saving c_rarg3:
611 // [tos + 0]: saved c_rarg3
612 // [tos + 1]: saved c_rarg2
613 // [tos + 2]: saved ra
614 // [tos + 3]: saved t1
615 // [tos + 4]: saved x10
616 // [tos + 5]: saved t0
617 address generate_verify_oop() {
618
619 StubId stub_id = StubId::stubgen_verify_oop_id;
620 StubCodeMark mark(this, stub_id);
621 address start = __ pc();
622
623 Label exit, error;
624
625 __ push_reg(RegSet::of(c_rarg2, c_rarg3), sp); // save c_rarg2 and c_rarg3
626
627 __ la(c_rarg2, ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
628 __ ld(c_rarg3, Address(c_rarg2));
629 __ addi(c_rarg3, c_rarg3, 1);
630 __ sd(c_rarg3, Address(c_rarg2));
631
632 // object is in x10
633 // make sure object is 'reasonable'
634 __ beqz(x10, exit); // if obj is null it is OK
635
636 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
637 bs_asm->check_oop(_masm, x10, c_rarg2, c_rarg3, error);
638
639 // return if everything seems ok
640 __ bind(exit);
641
642 __ pop_reg(RegSet::of(c_rarg2, c_rarg3), sp); // pop c_rarg2 and c_rarg3
643 __ ret();
644
645 // handle errors
646 __ bind(error);
647 __ pop_reg(RegSet::of(c_rarg2, c_rarg3), sp); // pop c_rarg2 and c_rarg3
648
649 __ push_reg(RegSet::range(x0, x31), sp);
650 // debug(char* msg, int64_t pc, int64_t regs[])
651 __ mv(c_rarg0, t0); // pass address of error message
652 __ mv(c_rarg1, ra); // pass return address
653 __ mv(c_rarg2, sp); // pass address of regs on stack
654 #ifndef PRODUCT
655 assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
656 #endif
657 BLOCK_COMMENT("call MacroAssembler::debug");
658 __ rt_call(CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
659 __ ebreak();
660
661 return start;
662 }
663
664 // The inner part of zero_words().
665 //
666 // Inputs:
667 // x28: the HeapWord-aligned base address of an array to zero.
668 // x29: the count in HeapWords, x29 > 0.
669 //
670 // Returns x28 and x29, adjusted for the caller to clear.
671 // x28: the base address of the tail of words left to clear.
672 // x29: the number of words in the tail.
673 // x29 < MacroAssembler::zero_words_block_size.
674
675 address generate_zero_blocks() {
676 Label done;
677
678 const Register base = x28, cnt = x29, tmp1 = x30, tmp2 = x31;
679
680 __ align(CodeEntryAlignment);
681 StubId stub_id = StubId::stubgen_zero_blocks_id;
682 StubCodeMark mark(this, stub_id);
683 address start = __ pc();
684
685 if (UseBlockZeroing) {
686 int zicboz_block_size = VM_Version::zicboz_block_size.value();
687 // Ensure count >= 2 * zicboz_block_size so that it still deserves
688 // a cbo.zero after alignment.
689 Label small;
690 int low_limit = MAX2(2 * zicboz_block_size, (int)BlockZeroingLowLimit) / wordSize;
691 __ mv(tmp1, low_limit);
692 __ blt(cnt, tmp1, small);
693 __ zero_dcache_blocks(base, cnt, tmp1, tmp2);
694 __ bind(small);
695 }
696
697 {
698 // Clear the remaining blocks.
699 Label loop;
700 __ mv(tmp1, MacroAssembler::zero_words_block_size);
701 __ blt(cnt, tmp1, done);
702 __ bind(loop);
703 for (int i = 0; i < MacroAssembler::zero_words_block_size; i++) {
704 __ sd(zr, Address(base, i * wordSize));
705 }
706 __ addi(base, base, MacroAssembler::zero_words_block_size * wordSize);
707 __ subi(cnt, cnt, MacroAssembler::zero_words_block_size);
708 __ bge(cnt, tmp1, loop);
709 __ bind(done);
710 }
711
712 __ ret();
713
714 return start;
715 }
716
717 typedef enum {
718 copy_forwards = 1,
719 copy_backwards = -1
720 } copy_direction;
721
722 // Bulk copy of blocks of 8 words.
723 //
724 // count is a count of words.
725 //
726 // Precondition: count >= 8
727 //
728 // Postconditions:
729 //
730 // The least significant bit of count contains the remaining count
731 // of words to copy. The rest of count is trash.
732 //
733 // s and d are adjusted to point to the remaining words to copy
734 //
735 address generate_copy_longs(StubId stub_id, Register s, Register d, Register count) {
736 BasicType type;
737 copy_direction direction;
738 switch (stub_id) {
739 case StubId::stubgen_copy_byte_f_id:
740 direction = copy_forwards;
741 type = T_BYTE;
742 break;
743 case StubId::stubgen_copy_byte_b_id:
744 direction = copy_backwards;
745 type = T_BYTE;
746 break;
747 default:
748 ShouldNotReachHere();
749 }
750 int unit = wordSize * direction;
751 int bias = wordSize;
752
753 const Register tmp_reg0 = x13, tmp_reg1 = x14, tmp_reg2 = x15, tmp_reg3 = x16,
754 tmp_reg4 = x17, tmp_reg5 = x7, tmp_reg6 = x28, tmp_reg7 = x29;
755
756 const Register stride = x30;
757
758 assert_different_registers(t0, tmp_reg0, tmp_reg1, tmp_reg2, tmp_reg3,
759 tmp_reg4, tmp_reg5, tmp_reg6, tmp_reg7);
760 assert_different_registers(s, d, count, t0);
761
762 Label again, drain;
763 StubCodeMark mark(this, stub_id);
764 __ align(CodeEntryAlignment);
765 address start = __ pc();
766
767 if (direction == copy_forwards) {
768 __ sub(s, s, bias);
769 __ sub(d, d, bias);
770 }
771
772 #ifdef ASSERT
773 // Make sure we are never given < 8 words
774 {
775 Label L;
776
777 __ mv(t0, 8);
778 __ bge(count, t0, L);
779 __ stop("genrate_copy_longs called with < 8 words");
780 __ bind(L);
781 }
782 #endif
783
784 __ ld(tmp_reg0, Address(s, 1 * unit));
785 __ ld(tmp_reg1, Address(s, 2 * unit));
786 __ ld(tmp_reg2, Address(s, 3 * unit));
787 __ ld(tmp_reg3, Address(s, 4 * unit));
788 __ ld(tmp_reg4, Address(s, 5 * unit));
789 __ ld(tmp_reg5, Address(s, 6 * unit));
790 __ ld(tmp_reg6, Address(s, 7 * unit));
791 __ ld(tmp_reg7, Address(s, 8 * unit));
792 __ addi(s, s, 8 * unit);
793
794 __ subi(count, count, 16);
795 __ bltz(count, drain);
796
797 __ bind(again);
798
799 __ sd(tmp_reg0, Address(d, 1 * unit));
800 __ sd(tmp_reg1, Address(d, 2 * unit));
801 __ sd(tmp_reg2, Address(d, 3 * unit));
802 __ sd(tmp_reg3, Address(d, 4 * unit));
803 __ sd(tmp_reg4, Address(d, 5 * unit));
804 __ sd(tmp_reg5, Address(d, 6 * unit));
805 __ sd(tmp_reg6, Address(d, 7 * unit));
806 __ sd(tmp_reg7, Address(d, 8 * unit));
807
808 __ ld(tmp_reg0, Address(s, 1 * unit));
809 __ ld(tmp_reg1, Address(s, 2 * unit));
810 __ ld(tmp_reg2, Address(s, 3 * unit));
811 __ ld(tmp_reg3, Address(s, 4 * unit));
812 __ ld(tmp_reg4, Address(s, 5 * unit));
813 __ ld(tmp_reg5, Address(s, 6 * unit));
814 __ ld(tmp_reg6, Address(s, 7 * unit));
815 __ ld(tmp_reg7, Address(s, 8 * unit));
816
817 __ addi(s, s, 8 * unit);
818 __ addi(d, d, 8 * unit);
819
820 __ subi(count, count, 8);
821 __ bgez(count, again);
822
823 // Drain
824 __ bind(drain);
825
826 __ sd(tmp_reg0, Address(d, 1 * unit));
827 __ sd(tmp_reg1, Address(d, 2 * unit));
828 __ sd(tmp_reg2, Address(d, 3 * unit));
829 __ sd(tmp_reg3, Address(d, 4 * unit));
830 __ sd(tmp_reg4, Address(d, 5 * unit));
831 __ sd(tmp_reg5, Address(d, 6 * unit));
832 __ sd(tmp_reg6, Address(d, 7 * unit));
833 __ sd(tmp_reg7, Address(d, 8 * unit));
834 __ addi(d, d, 8 * unit);
835
836 {
837 Label L1, L2;
838 __ test_bit(t0, count, 2);
839 __ beqz(t0, L1);
840
841 __ ld(tmp_reg0, Address(s, 1 * unit));
842 __ ld(tmp_reg1, Address(s, 2 * unit));
843 __ ld(tmp_reg2, Address(s, 3 * unit));
844 __ ld(tmp_reg3, Address(s, 4 * unit));
845 __ addi(s, s, 4 * unit);
846
847 __ sd(tmp_reg0, Address(d, 1 * unit));
848 __ sd(tmp_reg1, Address(d, 2 * unit));
849 __ sd(tmp_reg2, Address(d, 3 * unit));
850 __ sd(tmp_reg3, Address(d, 4 * unit));
851 __ addi(d, d, 4 * unit);
852
853 __ bind(L1);
854
855 if (direction == copy_forwards) {
856 __ addi(s, s, bias);
857 __ addi(d, d, bias);
858 }
859
860 __ test_bit(t0, count, 1);
861 __ beqz(t0, L2);
862 if (direction == copy_backwards) {
863 __ addi(s, s, 2 * unit);
864 __ ld(tmp_reg0, Address(s));
865 __ ld(tmp_reg1, Address(s, wordSize));
866 __ addi(d, d, 2 * unit);
867 __ sd(tmp_reg0, Address(d));
868 __ sd(tmp_reg1, Address(d, wordSize));
869 } else {
870 __ ld(tmp_reg0, Address(s));
871 __ ld(tmp_reg1, Address(s, wordSize));
872 __ addi(s, s, 2 * unit);
873 __ sd(tmp_reg0, Address(d));
874 __ sd(tmp_reg1, Address(d, wordSize));
875 __ addi(d, d, 2 * unit);
876 }
877 __ bind(L2);
878 }
879
880 __ ret();
881
882 return start;
883 }
884
885 typedef void (MacroAssembler::*copy_insn)(Register Rd, const Address &adr, Register temp);
886
887 void copy_memory_v(Register s, Register d, Register count, int step) {
888 bool is_backward = step < 0;
889 int granularity = g_uabs(step);
890
891 const Register src = x30, dst = x31, vl = x14, cnt = x15, tmp1 = x16, tmp2 = x17;
892 assert_different_registers(s, d, cnt, vl, tmp1, tmp2);
893 Assembler::SEW sew = Assembler::elembytes_to_sew(granularity);
894 Label loop_forward, loop_backward, done;
895
896 __ mv(dst, d);
897 __ mv(src, s);
898 __ mv(cnt, count);
899
900 __ bind(loop_forward);
901 __ vsetvli(vl, cnt, sew, Assembler::m8);
902 if (is_backward) {
903 __ bne(vl, cnt, loop_backward);
904 }
905
906 __ vlex_v(v0, src, sew);
907 __ sub(cnt, cnt, vl);
908 if (sew != Assembler::e8) {
909 // when sew == e8 (e.g., elem size is 1 byte), slli R, R, 0 is a nop and unnecessary
910 __ slli(vl, vl, sew);
911 }
912 __ add(src, src, vl);
913
914 __ vsex_v(v0, dst, sew);
915 __ add(dst, dst, vl);
916 __ bnez(cnt, loop_forward);
917
918 if (is_backward) {
919 __ j(done);
920
921 __ bind(loop_backward);
922 __ sub(t0, cnt, vl);
923 if (sew != Assembler::e8) {
924 // when sew == e8 (e.g., elem size is 1 byte), slli R, R, 0 is a nop and unnecessary
925 __ slli(t0, t0, sew);
926 }
927 __ add(tmp1, s, t0);
928 __ vlex_v(v0, tmp1, sew);
929 __ add(tmp2, d, t0);
930 __ vsex_v(v0, tmp2, sew);
931 __ sub(cnt, cnt, vl);
932 __ bnez(cnt, loop_forward);
933 __ bind(done);
934 }
935 }
936
937 // All-singing all-dancing memory copy.
938 //
939 // Copy count units of memory from s to d. The size of a unit is
940 // step, which can be positive or negative depending on the direction
941 // of copy.
942 //
943 void copy_memory(DecoratorSet decorators, BasicType type, bool is_aligned,
944 Register s, Register d, Register count, int step) {
945 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
946 if (UseRVV && (!is_reference_type(type) || bs_asm->supports_rvv_arraycopy())) {
947 return copy_memory_v(s, d, count, step);
948 }
949
950 bool is_backwards = step < 0;
951 int granularity = g_uabs(step);
952
953 const Register src = x30, dst = x31, cnt = x15, tmp3 = x16, tmp4 = x17, tmp5 = x14, tmp6 = x13;
954 const Register gct1 = x28, gct2 = x29, gct3 = t2;
955
956 Label same_aligned;
957 Label copy_big, copy32_loop, copy8_loop, copy_small, done;
958
959 // The size of copy32_loop body increases significantly with ZGC GC barriers.
960 // Need conditional far branches to reach a point beyond the loop in this case.
961 bool is_far = UseZGC;
962
963 __ beqz(count, done, is_far);
964 __ slli(cnt, count, exact_log2(granularity));
965 if (is_backwards) {
966 __ add(src, s, cnt);
967 __ add(dst, d, cnt);
968 } else {
969 __ mv(src, s);
970 __ mv(dst, d);
971 }
972
973 if (is_aligned) {
974 __ subi(t0, cnt, 32);
975 __ bgez(t0, copy32_loop);
976 __ subi(t0, cnt, 8);
977 __ bgez(t0, copy8_loop, is_far);
978 __ j(copy_small);
979 } else {
980 __ mv(t0, 16);
981 __ blt(cnt, t0, copy_small, is_far);
982
983 __ xorr(t0, src, dst);
984 __ andi(t0, t0, 0b111);
985 __ bnez(t0, copy_small, is_far);
986
987 __ bind(same_aligned);
988 __ andi(t0, src, 0b111);
989 __ beqz(t0, copy_big);
990 if (is_backwards) {
991 __ addi(src, src, step);
992 __ addi(dst, dst, step);
993 }
994 bs_asm->copy_load_at(_masm, decorators, type, granularity, tmp3, Address(src), gct1);
995 bs_asm->copy_store_at(_masm, decorators, type, granularity, Address(dst), tmp3, gct1, gct2, gct3);
996 if (!is_backwards) {
997 __ addi(src, src, step);
998 __ addi(dst, dst, step);
999 }
1000 __ subi(cnt, cnt, granularity);
1001 __ beqz(cnt, done, is_far);
1002 __ j(same_aligned);
1003
1004 __ bind(copy_big);
1005 __ mv(t0, 32);
1006 __ blt(cnt, t0, copy8_loop, is_far);
1007 }
1008
1009 __ bind(copy32_loop);
1010 if (is_backwards) {
1011 __ subi(src, src, wordSize * 4);
1012 __ subi(dst, dst, wordSize * 4);
1013 }
1014 // we first load 32 bytes, then write it, so the direction here doesn't matter
1015 bs_asm->copy_load_at(_masm, decorators, type, 8, tmp3, Address(src), gct1);
1016 bs_asm->copy_load_at(_masm, decorators, type, 8, tmp4, Address(src, 8), gct1);
1017 bs_asm->copy_load_at(_masm, decorators, type, 8, tmp5, Address(src, 16), gct1);
1018 bs_asm->copy_load_at(_masm, decorators, type, 8, tmp6, Address(src, 24), gct1);
1019
1020 bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst), tmp3, gct1, gct2, gct3);
1021 bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 8), tmp4, gct1, gct2, gct3);
1022 bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 16), tmp5, gct1, gct2, gct3);
1023 bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 24), tmp6, gct1, gct2, gct3);
1024
1025 if (!is_backwards) {
1026 __ addi(src, src, wordSize * 4);
1027 __ addi(dst, dst, wordSize * 4);
1028 }
1029 __ subi(t0, cnt, 32 + wordSize * 4);
1030 __ subi(cnt, cnt, wordSize * 4);
1031 __ bgez(t0, copy32_loop); // cnt >= 32, do next loop
1032
1033 __ beqz(cnt, done); // if that's all - done
1034
1035 __ subi(t0, cnt, 8); // if not - copy the reminder
1036 __ bltz(t0, copy_small); // cnt < 8, go to copy_small, else fall through to copy8_loop
1037
1038 __ bind(copy8_loop);
1039 if (is_backwards) {
1040 __ subi(src, src, wordSize);
1041 __ subi(dst, dst, wordSize);
1042 }
1043 bs_asm->copy_load_at(_masm, decorators, type, 8, tmp3, Address(src), gct1);
1044 bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst), tmp3, gct1, gct2, gct3);
1045
1046 if (!is_backwards) {
1047 __ addi(src, src, wordSize);
1048 __ addi(dst, dst, wordSize);
1049 }
1050 __ subi(t0, cnt, 8 + wordSize);
1051 __ subi(cnt, cnt, wordSize);
1052 __ bgez(t0, copy8_loop); // cnt >= 8, do next loop
1053
1054 __ beqz(cnt, done); // if that's all - done
1055
1056 __ bind(copy_small);
1057 if (is_backwards) {
1058 __ addi(src, src, step);
1059 __ addi(dst, dst, step);
1060 }
1061
1062 bs_asm->copy_load_at(_masm, decorators, type, granularity, tmp3, Address(src), gct1);
1063 bs_asm->copy_store_at(_masm, decorators, type, granularity, Address(dst), tmp3, gct1, gct2, gct3);
1064
1065 if (!is_backwards) {
1066 __ addi(src, src, step);
1067 __ addi(dst, dst, step);
1068 }
1069 __ subi(cnt, cnt, granularity);
1070 __ bgtz(cnt, copy_small);
1071
1072 __ bind(done);
1073 }
1074
1075 // Scan over array at a for count oops, verifying each one.
1076 // Preserves a and count, clobbers t0 and t1.
1077 void verify_oop_array(size_t size, Register a, Register count, Register temp) {
1078 Label loop, end;
1079 __ mv(t1, zr);
1080 __ slli(t0, count, exact_log2(size));
1081 __ bind(loop);
1082 __ bgeu(t1, t0, end);
1083
1084 __ add(temp, a, t1);
1085 if (size == (size_t)wordSize) {
1086 __ ld(temp, Address(temp, 0));
1087 __ verify_oop(temp);
1088 } else {
1089 __ lwu(temp, Address(temp, 0));
1090 __ decode_heap_oop(temp); // calls verify_oop
1091 }
1092 __ add(t1, t1, size);
1093 __ j(loop);
1094 __ bind(end);
1095 }
1096
1097 // Arguments:
1098 // stub_id - is used to name the stub and identify all details of
1099 // how to perform the copy.
1100 //
1101 // nopush_entry - is assigned to the stub's post push entry point
1102 // unless it is null
1103 //
1104 // Inputs:
1105 // c_rarg0 - source array address
1106 // c_rarg1 - destination array address
1107 // c_rarg2 - element count, treated as ssize_t, can be zero
1108 //
1109 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1110 // the hardware handle it. The two dwords within qwords that span
1111 // cache line boundaries will still be loaded and stored atomically.
1112 //
1113 // Side Effects: nopush_entry is set to the (post push) entry point
1114 // so it can be used by the corresponding conjoint
1115 // copy method
1116 //
1117 address generate_disjoint_copy(StubId stub_id, address* nopush_entry) {
1118 size_t size;
1119 bool aligned;
1120 bool is_oop;
1121 bool dest_uninitialized;
1122 switch (stub_id) {
1123 case StubId::stubgen_jbyte_disjoint_arraycopy_id:
1124 size = sizeof(jbyte);
1125 aligned = false;
1126 is_oop = false;
1127 dest_uninitialized = false;
1128 break;
1129 case StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id:
1130 size = sizeof(jbyte);
1131 aligned = true;
1132 is_oop = false;
1133 dest_uninitialized = false;
1134 break;
1135 case StubId::stubgen_jshort_disjoint_arraycopy_id:
1136 size = sizeof(jshort);
1137 aligned = false;
1138 is_oop = false;
1139 dest_uninitialized = false;
1140 break;
1141 case StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id:
1142 size = sizeof(jshort);
1143 aligned = true;
1144 is_oop = false;
1145 dest_uninitialized = false;
1146 break;
1147 case StubId::stubgen_jint_disjoint_arraycopy_id:
1148 size = sizeof(jint);
1149 aligned = false;
1150 is_oop = false;
1151 dest_uninitialized = false;
1152 break;
1153 case StubId::stubgen_arrayof_jint_disjoint_arraycopy_id:
1154 size = sizeof(jint);
1155 aligned = true;
1156 is_oop = false;
1157 dest_uninitialized = false;
1158 break;
1159 case StubId::stubgen_jlong_disjoint_arraycopy_id:
1160 // since this is always aligned we can (should!) use the same
1161 // stub as for case arrayof_jlong_disjoint_arraycopy
1162 ShouldNotReachHere();
1163 break;
1164 case StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id:
1165 size = sizeof(jlong);
1166 aligned = true;
1167 is_oop = false;
1168 dest_uninitialized = false;
1169 break;
1170 case StubId::stubgen_oop_disjoint_arraycopy_id:
1171 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1172 aligned = !UseCompressedOops;
1173 is_oop = true;
1174 dest_uninitialized = false;
1175 break;
1176 case StubId::stubgen_arrayof_oop_disjoint_arraycopy_id:
1177 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1178 aligned = !UseCompressedOops;
1179 is_oop = true;
1180 dest_uninitialized = false;
1181 break;
1182 case StubId::stubgen_oop_disjoint_arraycopy_uninit_id:
1183 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1184 aligned = !UseCompressedOops;
1185 is_oop = true;
1186 dest_uninitialized = true;
1187 break;
1188 case StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id:
1189 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1190 aligned = !UseCompressedOops;
1191 is_oop = true;
1192 dest_uninitialized = true;
1193 break;
1194 default:
1195 ShouldNotReachHere();
1196 break;
1197 }
1198
1199 const Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1200 RegSet saved_reg = RegSet::of(s, d, count);
1201 __ align(CodeEntryAlignment);
1202 StubCodeMark mark(this, stub_id);
1203 address start = __ pc();
1204 __ enter();
1205
1206 if (nopush_entry != nullptr) {
1207 *nopush_entry = __ pc();
1208 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1209 BLOCK_COMMENT("Entry:");
1210 }
1211
1212 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
1213 if (dest_uninitialized) {
1214 decorators |= IS_DEST_UNINITIALIZED;
1215 }
1216 if (aligned) {
1217 decorators |= ARRAYCOPY_ALIGNED;
1218 }
1219
1220 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1221 bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_reg);
1222
1223 if (is_oop) {
1224 // save regs before copy_memory
1225 __ push_reg(RegSet::of(d, count), sp);
1226 }
1227
1228 {
1229 // UnsafeMemoryAccess page error: continue after unsafe access
1230 bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size);
1231 UnsafeMemoryAccessMark umam(this, add_entry, true);
1232 copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size);
1233 }
1234
1235 if (is_oop) {
1236 __ pop_reg(RegSet::of(d, count), sp);
1237 if (VerifyOops) {
1238 verify_oop_array(size, d, count, t2);
1239 }
1240 }
1241
1242 bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, t0);
1243
1244 __ leave();
1245 __ mv(x10, zr); // return 0
1246 __ ret();
1247 return start;
1248 }
1249
1250 // Arguments:
1251 // stub_id - is used to name the stub and identify all details of
1252 // how to perform the copy.
1253 //
1254 // nooverlap_target - identifes the (post push) entry for the
1255 // corresponding disjoint copy routine which can be
1256 // jumped to if the ranges do not actually overlap
1257 //
1258 // nopush_entry - is assigned to the stub's post push entry point
1259 // unless it is null
1260 //
1261 // Inputs:
1262 // c_rarg0 - source array address
1263 // c_rarg1 - destination array address
1264 // c_rarg2 - element count, treated as ssize_t, can be zero
1265 //
1266 // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1267 // the hardware handle it. The two dwords within qwords that span
1268 // cache line boundaries will still be loaded and stored atomically.
1269 //
1270 // Side Effects:
1271 // nopush_entry is set to the no-overlap entry point so it can be
1272 // used by some other conjoint copy method
1273 //
1274 address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) {
1275 const Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1276 RegSet saved_regs = RegSet::of(s, d, count);
1277 int size;
1278 bool aligned;
1279 bool is_oop;
1280 bool dest_uninitialized;
1281 switch (stub_id) {
1282 case StubId::stubgen_jbyte_arraycopy_id:
1283 size = sizeof(jbyte);
1284 aligned = false;
1285 is_oop = false;
1286 dest_uninitialized = false;
1287 break;
1288 case StubId::stubgen_arrayof_jbyte_arraycopy_id:
1289 size = sizeof(jbyte);
1290 aligned = true;
1291 is_oop = false;
1292 dest_uninitialized = false;
1293 break;
1294 case StubId::stubgen_jshort_arraycopy_id:
1295 size = sizeof(jshort);
1296 aligned = false;
1297 is_oop = false;
1298 dest_uninitialized = false;
1299 break;
1300 case StubId::stubgen_arrayof_jshort_arraycopy_id:
1301 size = sizeof(jshort);
1302 aligned = true;
1303 is_oop = false;
1304 dest_uninitialized = false;
1305 break;
1306 case StubId::stubgen_jint_arraycopy_id:
1307 size = sizeof(jint);
1308 aligned = false;
1309 is_oop = false;
1310 dest_uninitialized = false;
1311 break;
1312 case StubId::stubgen_arrayof_jint_arraycopy_id:
1313 size = sizeof(jint);
1314 aligned = true;
1315 is_oop = false;
1316 dest_uninitialized = false;
1317 break;
1318 case StubId::stubgen_jlong_arraycopy_id:
1319 // since this is always aligned we can (should!) use the same
1320 // stub as for case arrayof_jlong_disjoint_arraycopy
1321 ShouldNotReachHere();
1322 break;
1323 case StubId::stubgen_arrayof_jlong_arraycopy_id:
1324 size = sizeof(jlong);
1325 aligned = true;
1326 is_oop = false;
1327 dest_uninitialized = false;
1328 break;
1329 case StubId::stubgen_oop_arraycopy_id:
1330 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1331 aligned = !UseCompressedOops;
1332 is_oop = true;
1333 dest_uninitialized = false;
1334 break;
1335 case StubId::stubgen_arrayof_oop_arraycopy_id:
1336 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1337 aligned = !UseCompressedOops;
1338 is_oop = true;
1339 dest_uninitialized = false;
1340 break;
1341 case StubId::stubgen_oop_arraycopy_uninit_id:
1342 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1343 aligned = !UseCompressedOops;
1344 is_oop = true;
1345 dest_uninitialized = true;
1346 break;
1347 case StubId::stubgen_arrayof_oop_arraycopy_uninit_id:
1348 size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1349 aligned = !UseCompressedOops;
1350 is_oop = true;
1351 dest_uninitialized = true;
1352 break;
1353 default:
1354 ShouldNotReachHere();
1355 }
1356
1357 StubCodeMark mark(this, stub_id);
1358 address start = __ pc();
1359 __ enter();
1360
1361 if (nopush_entry != nullptr) {
1362 *nopush_entry = __ pc();
1363 // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1364 BLOCK_COMMENT("Entry:");
1365 }
1366
1367 // use fwd copy when (d-s) above_equal (count*size)
1368 __ sub(t0, d, s);
1369 __ slli(t1, count, exact_log2(size));
1370 Label L_continue;
1371 __ bltu(t0, t1, L_continue);
1372 __ j(RuntimeAddress(nooverlap_target));
1373 __ bind(L_continue);
1374
1375 DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1376 if (dest_uninitialized) {
1377 decorators |= IS_DEST_UNINITIALIZED;
1378 }
1379 if (aligned) {
1380 decorators |= ARRAYCOPY_ALIGNED;
1381 }
1382
1383 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1384 bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_regs);
1385
1386 if (is_oop) {
1387 // save regs before copy_memory
1388 __ push_reg(RegSet::of(d, count), sp);
1389 }
1390
1391 {
1392 // UnsafeMemoryAccess page error: continue after unsafe access
1393 bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size);
1394 UnsafeMemoryAccessMark umam(this, add_entry, true);
1395 copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size);
1396 }
1397
1398 if (is_oop) {
1399 __ pop_reg(RegSet::of(d, count), sp);
1400 if (VerifyOops) {
1401 verify_oop_array(size, d, count, t2);
1402 }
1403 }
1404 bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, t0);
1405 __ leave();
1406 __ mv(x10, zr); // return 0
1407 __ ret();
1408 return start;
1409 }
1410
1411 // Helper for generating a dynamic type check.
1412 // Smashes t0, t1.
1413 void generate_type_check(Register sub_klass,
1414 Register super_check_offset,
1415 Register super_klass,
1416 Register result,
1417 Register tmp1,
1418 Register tmp2,
1419 Label& L_success) {
1420 assert_different_registers(sub_klass, super_check_offset, super_klass);
1421
1422 BLOCK_COMMENT("type_check:");
1423
1424 Label L_miss;
1425
1426 __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg, &L_success, &L_miss, nullptr, super_check_offset);
1427 __ check_klass_subtype_slow_path(sub_klass, super_klass, tmp1, tmp2, &L_success, nullptr);
1428
1429 // Fall through on failure!
1430 __ BIND(L_miss);
1431 }
1432
1433 //
1434 // Generate checkcasting array copy stub
1435 //
1436 // Input:
1437 // c_rarg0 - source array address
1438 // c_rarg1 - destination array address
1439 // c_rarg2 - element count, treated as ssize_t, can be zero
1440 // c_rarg3 - size_t ckoff (super_check_offset)
1441 // c_rarg4 - oop ckval (super_klass)
1442 //
1443 // Output:
1444 // x10 == 0 - success
1445 // x10 == -1^K - failure, where K is partial transfer count
1446 //
1447 address generate_checkcast_copy(StubId stub_id, address* nopush_entry) {
1448 bool dest_uninitialized;
1449 switch (stub_id) {
1450 case StubId::stubgen_checkcast_arraycopy_id:
1451 dest_uninitialized = false;
1452 break;
1453 case StubId::stubgen_checkcast_arraycopy_uninit_id:
1454 dest_uninitialized = true;
1455 break;
1456 default:
1457 ShouldNotReachHere();
1458 }
1459
1460 Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop;
1461
1462 // Input registers (after setup_arg_regs)
1463 const Register from = c_rarg0; // source array address
1464 const Register to = c_rarg1; // destination array address
1465 const Register count = c_rarg2; // elementscount
1466 const Register ckoff = c_rarg3; // super_check_offset
1467 const Register ckval = c_rarg4; // super_klass
1468
1469 RegSet wb_pre_saved_regs = RegSet::range(c_rarg0, c_rarg4);
1470
1471 // Registers used as temps (x7, x9, x18 are save-on-entry)
1472 const Register count_save = x19; // orig elementscount
1473 const Register start_to = x18; // destination array start address
1474 const Register copied_oop = x7; // actual oop copied
1475 const Register r9_klass = x9; // oop._klass
1476
1477 // Registers used as gc temps (x15, x16, x17 are save-on-call)
1478 const Register gct1 = x15, gct2 = x16, gct3 = x17;
1479
1480 //---------------------------------------------------------------
1481 // Assembler stub will be used for this call to arraycopy
1482 // if the two arrays are subtypes of Object[] but the
1483 // destination array type is not equal to or a supertype
1484 // of the source type. Each element must be separately
1485 // checked.
1486
1487 assert_different_registers(from, to, count, ckoff, ckval, start_to,
1488 copied_oop, r9_klass, count_save);
1489
1490 __ align(CodeEntryAlignment);
1491 StubCodeMark mark(this, stub_id);
1492 address start = __ pc();
1493
1494 __ enter(); // required for proper stackwalking of RuntimeStub frame
1495
1496 // Caller of this entry point must set up the argument registers.
1497 if (nopush_entry != nullptr) {
1498 *nopush_entry = __ pc();
1499 BLOCK_COMMENT("Entry:");
1500 }
1501
1502 // Empty array: Nothing to do
1503 __ beqz(count, L_done);
1504
1505 __ push_reg(RegSet::of(x7, x9, x18, x19), sp);
1506
1507 #ifdef ASSERT
1508 BLOCK_COMMENT("assert consistent ckoff/ckval");
1509 // The ckoff and ckval must be mutually consistent,
1510 // even though caller generates both.
1511 { Label L;
1512 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1513 __ lwu(start_to, Address(ckval, sco_offset));
1514 __ beq(ckoff, start_to, L);
1515 __ stop("super_check_offset inconsistent");
1516 __ bind(L);
1517 }
1518 #endif //ASSERT
1519
1520 DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT;
1521 if (dest_uninitialized) {
1522 decorators |= IS_DEST_UNINITIALIZED;
1523 }
1524
1525 bool is_oop = true;
1526 int element_size = UseCompressedOops ? 4 : 8;
1527
1528 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1529 bs->arraycopy_prologue(_masm, decorators, is_oop, from, to, count, wb_pre_saved_regs);
1530
1531 // save the original count
1532 __ mv(count_save, count);
1533
1534 // Copy from low to high addresses
1535 __ mv(start_to, to); // Save destination array start address
1536 __ j(L_load_element);
1537
1538 // ======== begin loop ========
1539 // (Loop is rotated; its entry is L_load_element.)
1540 // Loop control:
1541 // for count to 0 do
1542 // copied_oop = load_heap_oop(from++)
1543 // ... generate_type_check ...
1544 // store_heap_oop(to++, copied_oop)
1545 // end
1546
1547 __ align(OptoLoopAlignment);
1548
1549 __ BIND(L_store_element);
1550 bs->copy_store_at(_masm, decorators, T_OBJECT, element_size,
1551 Address(to, 0), copied_oop,
1552 gct1, gct2, gct3);
1553 __ addi(to, to, UseCompressedOops ? 4 : 8);
1554 __ subi(count, count, 1);
1555 __ beqz(count, L_do_card_marks);
1556
1557 // ======== loop entry is here ========
1558 __ BIND(L_load_element);
1559 bs->copy_load_at(_masm, decorators, T_OBJECT, element_size,
1560 copied_oop, Address(from, 0),
1561 gct1);
1562 __ addi(from, from, UseCompressedOops ? 4 : 8);
1563 __ beqz(copied_oop, L_store_element);
1564
1565 __ load_klass(r9_klass, copied_oop);// query the object klass
1566
1567 BLOCK_COMMENT("type_check:");
1568 generate_type_check(r9_klass, /*sub_klass*/
1569 ckoff, /*super_check_offset*/
1570 ckval, /*super_klass*/
1571 x10, /*result*/
1572 gct1, /*tmp1*/
1573 gct2, /*tmp2*/
1574 L_store_element);
1575
1576 // Fall through on failure!
1577
1578 // ======== end loop ========
1579
1580 // It was a real error; we must depend on the caller to finish the job.
1581 // Register count = remaining oops, count_orig = total oops.
1582 // Emit GC store barriers for the oops we have copied and report
1583 // their number to the caller.
1584
1585 __ sub(count, count_save, count); // K = partially copied oop count
1586 __ xori(count, count, -1); // report (-1^K) to caller
1587 __ beqz(count, L_done_pop);
1588
1589 __ BIND(L_do_card_marks);
1590 bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, count_save, t0);
1591
1592 __ bind(L_done_pop);
1593 __ pop_reg(RegSet::of(x7, x9, x18, x19), sp);
1594 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1595
1596 __ bind(L_done);
1597 __ mv(x10, count);
1598 __ leave();
1599 __ ret();
1600
1601 return start;
1602 }
1603
1604 // Perform range checks on the proposed arraycopy.
1605 // Kills temp, but nothing else.
1606 // Also, clean the sign bits of src_pos and dst_pos.
1607 void arraycopy_range_checks(Register src, // source array oop (c_rarg0)
1608 Register src_pos, // source position (c_rarg1)
1609 Register dst, // destination array oo (c_rarg2)
1610 Register dst_pos, // destination position (c_rarg3)
1611 Register length,
1612 Register temp,
1613 Label& L_failed) {
1614 BLOCK_COMMENT("arraycopy_range_checks:");
1615
1616 assert_different_registers(t0, temp);
1617
1618 // if [src_pos + length > arrayOop(src)->length()] then FAIL
1619 __ lwu(t0, Address(src, arrayOopDesc::length_offset_in_bytes()));
1620 __ addw(temp, length, src_pos);
1621 __ bgtu(temp, t0, L_failed);
1622
1623 // if [dst_pos + length > arrayOop(dst)->length()] then FAIL
1624 __ lwu(t0, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1625 __ addw(temp, length, dst_pos);
1626 __ bgtu(temp, t0, L_failed);
1627
1628 // Have to clean up high 32 bits of 'src_pos' and 'dst_pos'.
1629 __ zext(src_pos, src_pos, 32);
1630 __ zext(dst_pos, dst_pos, 32);
1631
1632 BLOCK_COMMENT("arraycopy_range_checks done");
1633 }
1634
1635 address generate_unsafecopy_common_error_exit() {
1636 address start = __ pc();
1637 __ mv(x10, 0);
1638 __ leave();
1639 __ ret();
1640 return start;
1641 }
1642
1643 //
1644 // Generate 'unsafe' set memory stub
1645 // Though just as safe as the other stubs, it takes an unscaled
1646 // size_t (# bytes) argument instead of an element count.
1647 //
1648 // Input:
1649 // c_rarg0 - destination array address
1650 // c_rarg1 - byte count (size_t)
1651 // c_rarg2 - byte value
1652 //
1653 address generate_unsafe_setmemory() {
1654 __ align(CodeEntryAlignment);
1655 StubId stub_id = StubId::stubgen_unsafe_setmemory_id;
1656 StubCodeMark mark(this, stub_id);
1657 address start = __ pc();
1658
1659 // bump this on entry, not on exit:
1660 // inc_counter_np(SharedRuntime::_unsafe_set_memory_ctr);
1661
1662 Label L_fill_elements;
1663
1664 const Register dest = c_rarg0;
1665 const Register count = c_rarg1;
1666 const Register value = c_rarg2;
1667 const Register cnt_words = x28; // temp register
1668 const Register tmp_reg = x29; // temp register
1669
1670 // Mark remaining code as such which performs Unsafe accesses.
1671 UnsafeMemoryAccessMark umam(this, true, false);
1672
1673 __ enter(); // required for proper stackwalking of RuntimeStub frame
1674
1675 // if count < 8, jump to L_fill_elements
1676 __ mv(tmp_reg, 8); // 8 bytes fill by element
1677 __ bltu(count, tmp_reg, L_fill_elements);
1678
1679 // Propagate byte to 64-bit width
1680 // 8 bit -> 16 bit
1681 __ zext(value, value, 8);
1682 __ slli(tmp_reg, value, 8);
1683 __ orr(value, value, tmp_reg);
1684 // 16 bit -> 32 bit
1685 __ slli(tmp_reg, value, 16);
1686 __ orr(value, value, tmp_reg);
1687 // 32 bit -> 64 bit
1688 __ slli(tmp_reg, value, 32);
1689 __ orr(value, value, tmp_reg);
1690
1691 // Align source address at 8 bytes address boundary.
1692 Label L_skip_align1, L_skip_align2, L_skip_align4;
1693 // One byte misalignment happens.
1694 __ test_bit(tmp_reg, dest, 0);
1695 __ beqz(tmp_reg, L_skip_align1);
1696 __ sb(value, Address(dest, 0));
1697 __ addi(dest, dest, 1);
1698 __ subi(count, count, 1);
1699
1700 __ bind(L_skip_align1);
1701 // Two bytes misalignment happens.
1702 __ test_bit(tmp_reg, dest, 1);
1703 __ beqz(tmp_reg, L_skip_align2);
1704 __ sh(value, Address(dest, 0));
1705 __ addi(dest, dest, 2);
1706 __ subi(count, count, 2);
1707
1708 __ bind(L_skip_align2);
1709 // Four bytes misalignment happens.
1710 __ test_bit(tmp_reg, dest, 2);
1711 __ beqz(tmp_reg, L_skip_align4);
1712 __ sw(value, Address(dest, 0));
1713 __ addi(dest, dest, 4);
1714 __ subi(count, count, 4);
1715 __ bind(L_skip_align4);
1716
1717 // Fill large chunks
1718 __ srli(cnt_words, count, 3); // number of words
1719 __ slli(tmp_reg, cnt_words, 3);
1720 __ sub(count, count, tmp_reg);
1721 {
1722 __ fill_words(dest, cnt_words, value);
1723 }
1724
1725 // Handle copies less than 8 bytes
1726 __ bind(L_fill_elements);
1727 Label L_fill_2, L_fill_1, L_exit;
1728 __ test_bit(tmp_reg, count, 2);
1729 __ beqz(tmp_reg, L_fill_2);
1730 __ sb(value, Address(dest, 0));
1731 __ sb(value, Address(dest, 1));
1732 __ sb(value, Address(dest, 2));
1733 __ sb(value, Address(dest, 3));
1734 __ addi(dest, dest, 4);
1735
1736 __ bind(L_fill_2);
1737 __ test_bit(tmp_reg, count, 1);
1738 __ beqz(tmp_reg, L_fill_1);
1739 __ sb(value, Address(dest, 0));
1740 __ sb(value, Address(dest, 1));
1741 __ addi(dest, dest, 2);
1742
1743 __ bind(L_fill_1);
1744 __ test_bit(tmp_reg, count, 0);
1745 __ beqz(tmp_reg, L_exit);
1746 __ sb(value, Address(dest, 0));
1747
1748 __ bind(L_exit);
1749 __ leave();
1750 __ ret();
1751
1752 return start;
1753 }
1754
1755 //
1756 // Generate 'unsafe' array copy stub
1757 // Though just as safe as the other stubs, it takes an unscaled
1758 // size_t argument instead of an element count.
1759 //
1760 // Input:
1761 // c_rarg0 - source array address
1762 // c_rarg1 - destination array address
1763 // c_rarg2 - byte count, treated as ssize_t, can be zero
1764 //
1765 // Examines the alignment of the operands and dispatches
1766 // to a long, int, short, or byte copy loop.
1767 //
1768 address generate_unsafe_copy(address byte_copy_entry,
1769 address short_copy_entry,
1770 address int_copy_entry,
1771 address long_copy_entry) {
1772 assert_cond(byte_copy_entry != nullptr && short_copy_entry != nullptr &&
1773 int_copy_entry != nullptr && long_copy_entry != nullptr);
1774 Label L_long_aligned, L_int_aligned, L_short_aligned;
1775 const Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1776
1777 __ align(CodeEntryAlignment);
1778 StubId stub_id = StubId::stubgen_unsafe_arraycopy_id;
1779 StubCodeMark mark(this, stub_id);
1780 address start = __ pc();
1781 __ enter(); // required for proper stackwalking of RuntimeStub frame
1782
1783 // bump this on entry, not on exit:
1784 inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
1785
1786 __ orr(t0, s, d);
1787 __ orr(t0, t0, count);
1788
1789 __ andi(t0, t0, BytesPerLong - 1);
1790 __ beqz(t0, L_long_aligned);
1791 __ andi(t0, t0, BytesPerInt - 1);
1792 __ beqz(t0, L_int_aligned);
1793 __ test_bit(t0, t0, 0);
1794 __ beqz(t0, L_short_aligned);
1795 __ j(RuntimeAddress(byte_copy_entry));
1796
1797 __ BIND(L_short_aligned);
1798 __ srli(count, count, LogBytesPerShort); // size => short_count
1799 __ j(RuntimeAddress(short_copy_entry));
1800 __ BIND(L_int_aligned);
1801 __ srli(count, count, LogBytesPerInt); // size => int_count
1802 __ j(RuntimeAddress(int_copy_entry));
1803 __ BIND(L_long_aligned);
1804 __ srli(count, count, LogBytesPerLong); // size => long_count
1805 __ j(RuntimeAddress(long_copy_entry));
1806
1807 return start;
1808 }
1809
1810 //
1811 // Generate generic array copy stubs
1812 //
1813 // Input:
1814 // c_rarg0 - src oop
1815 // c_rarg1 - src_pos (32-bits)
1816 // c_rarg2 - dst oop
1817 // c_rarg3 - dst_pos (32-bits)
1818 // c_rarg4 - element count (32-bits)
1819 //
1820 // Output:
1821 // x10 == 0 - success
1822 // x10 == -1^K - failure, where K is partial transfer count
1823 //
1824 address generate_generic_copy(address byte_copy_entry, address short_copy_entry,
1825 address int_copy_entry, address oop_copy_entry,
1826 address long_copy_entry, address checkcast_copy_entry) {
1827 assert_cond(byte_copy_entry != nullptr && short_copy_entry != nullptr &&
1828 int_copy_entry != nullptr && oop_copy_entry != nullptr &&
1829 long_copy_entry != nullptr && checkcast_copy_entry != nullptr);
1830 Label L_failed, L_failed_0, L_objArray;
1831 Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
1832
1833 // Input registers
1834 const Register src = c_rarg0; // source array oop
1835 const Register src_pos = c_rarg1; // source position
1836 const Register dst = c_rarg2; // destination array oop
1837 const Register dst_pos = c_rarg3; // destination position
1838 const Register length = c_rarg4;
1839
1840 // Registers used as temps
1841 const Register dst_klass = c_rarg5;
1842
1843 __ align(CodeEntryAlignment);
1844
1845 StubId stub_id = StubId::stubgen_generic_arraycopy_id;
1846 StubCodeMark mark(this, stub_id);
1847
1848 address start = __ pc();
1849
1850 __ enter(); // required for proper stackwalking of RuntimeStub frame
1851
1852 // bump this on entry, not on exit:
1853 inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
1854
1855 //-----------------------------------------------------------------------
1856 // Assembler stub will be used for this call to arraycopy
1857 // if the following conditions are met:
1858 //
1859 // (1) src and dst must not be null.
1860 // (2) src_pos must not be negative.
1861 // (3) dst_pos must not be negative.
1862 // (4) length must not be negative.
1863 // (5) src klass and dst klass should be the same and not null.
1864 // (6) src and dst should be arrays.
1865 // (7) src_pos + length must not exceed length of src.
1866 // (8) dst_pos + length must not exceed length of dst.
1867 //
1868
1869 // if src is null then return -1
1870 __ beqz(src, L_failed);
1871
1872 // if [src_pos < 0] then return -1
1873 __ sext(t0, src_pos, 32);
1874 __ bltz(t0, L_failed);
1875
1876 // if dst is null then return -1
1877 __ beqz(dst, L_failed);
1878
1879 // if [dst_pos < 0] then return -1
1880 __ sext(t0, dst_pos, 32);
1881 __ bltz(t0, L_failed);
1882
1883 // registers used as temp
1884 const Register scratch_length = x28; // elements count to copy
1885 const Register scratch_src_klass = x29; // array klass
1886 const Register lh = x30; // layout helper
1887
1888 // if [length < 0] then return -1
1889 __ sext(scratch_length, length, 32); // length (elements count, 32-bits value)
1890 __ bltz(scratch_length, L_failed);
1891
1892 __ load_narrow_klass(scratch_src_klass, src);
1893 #ifdef ASSERT
1894 {
1895 BLOCK_COMMENT("assert klasses not null {");
1896 Label L1, L2;
1897 __ bnez(scratch_src_klass, L2); // it is broken if klass is null
1898 __ bind(L1);
1899 __ stop("broken null klass");
1900 __ bind(L2);
1901 __ load_narrow_klass(t0, dst);
1902 __ beqz(t0, L1); // this would be broken also
1903 BLOCK_COMMENT("} assert klasses not null done");
1904 }
1905 #endif
1906 __ decode_klass_not_null(scratch_src_klass, t0);
1907
1908 // Load layout helper (32-bits)
1909 //
1910 // |array_tag| | header_size | element_type | |log2_element_size|
1911 // 32 30 24 16 8 2 0
1912 //
1913 // array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
1914 //
1915
1916 const int lh_offset = in_bytes(Klass::layout_helper_offset());
1917
1918 // Handle objArrays completely differently...
1919 const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
1920 __ lw(lh, Address(scratch_src_klass, lh_offset));
1921 __ mv(t0, objArray_lh);
1922 __ beq(lh, t0, L_objArray);
1923
1924 // if [src->klass() != dst->klass()] then return -1
1925 __ load_klass(t1, dst);
1926 __ bne(t1, scratch_src_klass, L_failed);
1927
1928 // Check for flat inline type array -> return -1
1929 __ test_flat_array_oop(src, t1, L_failed);
1930
1931 // Check for null-free (non-flat) inline type array -> handle as object array
1932 __ test_null_free_array_oop(src, t1, L_objArray);
1933
1934 // if src->is_Array() isn't null then return -1
1935 // i.e. (lh >= 0)
1936 __ bgez(lh, L_failed);
1937
1938 // At this point, it is known to be a typeArray (array_tag 0x3).
1939 #ifdef ASSERT
1940 {
1941 BLOCK_COMMENT("assert primitive array {");
1942 Label L;
1943 __ mv(t1, (int32_t)(Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
1944 __ bge(lh, t1, L);
1945 __ stop("must be a primitive array");
1946 __ bind(L);
1947 BLOCK_COMMENT("} assert primitive array done");
1948 }
1949 #endif
1950
1951 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
1952 t1, L_failed);
1953
1954 // TypeArrayKlass
1955 //
1956 // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize)
1957 // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize)
1958 //
1959
1960 const Register t0_offset = t0; // array offset
1961 const Register x30_elsize = lh; // element size
1962
1963 // Get array_header_in_bytes()
1964 int lh_header_size_width = exact_log2(Klass::_lh_header_size_mask + 1);
1965 int lh_header_size_msb = Klass::_lh_header_size_shift + lh_header_size_width;
1966 __ slli(t0_offset, lh, XLEN - lh_header_size_msb); // left shift to remove 24 ~ 32;
1967 __ srli(t0_offset, t0_offset, XLEN - lh_header_size_width); // array_offset
1968
1969 __ add(src, src, t0_offset); // src array offset
1970 __ add(dst, dst, t0_offset); // dst array offset
1971 BLOCK_COMMENT("choose copy loop based on element size");
1972
1973 // next registers should be set before the jump to corresponding stub
1974 const Register from = c_rarg0; // source array address
1975 const Register to = c_rarg1; // destination array address
1976 const Register count = c_rarg2; // elements count
1977
1978 // 'from', 'to', 'count' registers should be set in such order
1979 // since they are the same as 'src', 'src_pos', 'dst'.
1980
1981 assert(Klass::_lh_log2_element_size_shift == 0, "fix this code");
1982
1983 // The possible values of elsize are 0-3, i.e. exact_log2(element
1984 // size in bytes). We do a simple bitwise binary search.
1985 __ BIND(L_copy_bytes);
1986 __ test_bit(t0, x30_elsize, 1);
1987 __ bnez(t0, L_copy_ints);
1988 __ test_bit(t0, x30_elsize, 0);
1989 __ bnez(t0, L_copy_shorts);
1990 __ add(from, src, src_pos); // src_addr
1991 __ add(to, dst, dst_pos); // dst_addr
1992 __ sext(count, scratch_length, 32); // length
1993 __ j(RuntimeAddress(byte_copy_entry));
1994
1995 __ BIND(L_copy_shorts);
1996 __ shadd(from, src_pos, src, t0, 1); // src_addr
1997 __ shadd(to, dst_pos, dst, t0, 1); // dst_addr
1998 __ sext(count, scratch_length, 32); // length
1999 __ j(RuntimeAddress(short_copy_entry));
2000
2001 __ BIND(L_copy_ints);
2002 __ test_bit(t0, x30_elsize, 0);
2003 __ bnez(t0, L_copy_longs);
2004 __ shadd(from, src_pos, src, t0, 2); // src_addr
2005 __ shadd(to, dst_pos, dst, t0, 2); // dst_addr
2006 __ sext(count, scratch_length, 32); // length
2007 __ j(RuntimeAddress(int_copy_entry));
2008
2009 __ BIND(L_copy_longs);
2010 #ifdef ASSERT
2011 {
2012 BLOCK_COMMENT("assert long copy {");
2013 Label L;
2014 __ andi(lh, lh, Klass::_lh_log2_element_size_mask); // lh -> x30_elsize
2015 __ sext(lh, lh, 32);
2016 __ mv(t0, LogBytesPerLong);
2017 __ beq(x30_elsize, t0, L);
2018 __ stop("must be long copy, but elsize is wrong");
2019 __ bind(L);
2020 BLOCK_COMMENT("} assert long copy done");
2021 }
2022 #endif
2023 __ shadd(from, src_pos, src, t0, 3); // src_addr
2024 __ shadd(to, dst_pos, dst, t0, 3); // dst_addr
2025 __ sext(count, scratch_length, 32); // length
2026 __ j(RuntimeAddress(long_copy_entry));
2027
2028 // ObjArrayKlass
2029 __ BIND(L_objArray);
2030 // live at this point: scratch_src_klass, scratch_length, src[_pos], dst[_pos]
2031
2032 Label L_plain_copy, L_checkcast_copy;
2033 // test array classes for subtyping
2034 __ load_klass(t2, dst);
2035 __ bne(scratch_src_klass, t2, L_checkcast_copy); // usual case is exact equality
2036
2037 // Identically typed arrays can be copied without element-wise checks.
2038 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2039 t1, L_failed);
2040
2041 __ shadd(from, src_pos, src, t0, LogBytesPerHeapOop);
2042 __ addi(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2043 __ shadd(to, dst_pos, dst, t0, LogBytesPerHeapOop);
2044 __ addi(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2045 __ sext(count, scratch_length, 32); // length
2046 __ BIND(L_plain_copy);
2047 __ j(RuntimeAddress(oop_copy_entry));
2048
2049 __ BIND(L_checkcast_copy);
2050 // live at this point: scratch_src_klass, scratch_length, t2 (dst_klass)
2051 {
2052 // Before looking at dst.length, make sure dst is also an objArray.
2053 __ lwu(t0, Address(t2, lh_offset));
2054 __ mv(t1, objArray_lh);
2055 __ bne(t0, t1, L_failed);
2056
2057 // It is safe to examine both src.length and dst.length.
2058 arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2059 t2, L_failed);
2060
2061 __ load_klass(dst_klass, dst); // reload
2062
2063 // Marshal the base address arguments now, freeing registers.
2064 __ shadd(from, src_pos, src, t0, LogBytesPerHeapOop);
2065 __ addi(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2066 __ shadd(to, dst_pos, dst, t0, LogBytesPerHeapOop);
2067 __ addi(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2068 __ sext(count, length, 32); // length (reloaded)
2069 const Register sco_temp = c_rarg3; // this register is free now
2070 assert_different_registers(from, to, count, sco_temp,
2071 dst_klass, scratch_src_klass);
2072
2073 // Generate the type check.
2074 const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2075 __ lwu(sco_temp, Address(dst_klass, sco_offset));
2076
2077 // Smashes t0, t1
2078 generate_type_check(scratch_src_klass, sco_temp, dst_klass, noreg, noreg, noreg, L_plain_copy);
2079
2080 // Fetch destination element klass from the ObjArrayKlass header.
2081 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2082 __ ld(dst_klass, Address(dst_klass, ek_offset));
2083 __ lwu(sco_temp, Address(dst_klass, sco_offset));
2084
2085 // the checkcast_copy loop needs two extra arguments:
2086 assert(c_rarg3 == sco_temp, "#3 already in place");
2087 // Set up arguments for checkcast_copy_entry.
2088 __ mv(c_rarg4, dst_klass); // dst.klass.element_klass
2089 __ j(RuntimeAddress(checkcast_copy_entry));
2090 }
2091
2092 __ BIND(L_failed);
2093 __ mv(x10, -1);
2094 __ leave(); // required for proper stackwalking of RuntimeStub frame
2095 __ ret();
2096
2097 return start;
2098 }
2099
2100 //
2101 // Generate stub for array fill. If "aligned" is true, the
2102 // "to" address is assumed to be heapword aligned.
2103 //
2104 // Arguments for generated stub:
2105 // to: c_rarg0
2106 // value: c_rarg1
2107 // count: c_rarg2 treated as signed
2108 //
2109 address generate_fill(StubId stub_id) {
2110 BasicType t;
2111 bool aligned;
2112
2113 switch (stub_id) {
2114 case StubId::stubgen_jbyte_fill_id:
2115 t = T_BYTE;
2116 aligned = false;
2117 break;
2118 case StubId::stubgen_jshort_fill_id:
2119 t = T_SHORT;
2120 aligned = false;
2121 break;
2122 case StubId::stubgen_jint_fill_id:
2123 t = T_INT;
2124 aligned = false;
2125 break;
2126 case StubId::stubgen_arrayof_jbyte_fill_id:
2127 t = T_BYTE;
2128 aligned = true;
2129 break;
2130 case StubId::stubgen_arrayof_jshort_fill_id:
2131 t = T_SHORT;
2132 aligned = true;
2133 break;
2134 case StubId::stubgen_arrayof_jint_fill_id:
2135 t = T_INT;
2136 aligned = true;
2137 break;
2138 default:
2139 ShouldNotReachHere();
2140 };
2141
2142 __ align(CodeEntryAlignment);
2143 StubCodeMark mark(this, stub_id);
2144 address start = __ pc();
2145
2146 BLOCK_COMMENT("Entry:");
2147
2148 const Register to = c_rarg0; // source array address
2149 const Register value = c_rarg1; // value
2150 const Register count = c_rarg2; // elements count
2151
2152 const Register bz_base = x28; // base for block_zero routine
2153 const Register cnt_words = x29; // temp register
2154 const Register tmp_reg = t1;
2155
2156 __ enter();
2157
2158 Label L_fill_elements;
2159
2160 int shift = -1;
2161 switch (t) {
2162 case T_BYTE:
2163 shift = 0;
2164 // Short arrays (< 8 bytes) fill by element
2165 __ mv(tmp_reg, 8 >> shift);
2166 __ bltu(count, tmp_reg, L_fill_elements);
2167
2168 // Zero extend value
2169 // 8 bit -> 16 bit
2170 __ zext(value, value, 8);
2171 __ slli(tmp_reg, value, 8);
2172 __ orr(value, value, tmp_reg);
2173
2174 // 16 bit -> 32 bit
2175 __ slli(tmp_reg, value, 16);
2176 __ orr(value, value, tmp_reg);
2177 break;
2178 case T_SHORT:
2179 shift = 1;
2180 // Short arrays (< 8 bytes) fill by element
2181 __ mv(tmp_reg, 8 >> shift);
2182 __ bltu(count, tmp_reg, L_fill_elements);
2183
2184 // Zero extend value
2185 // 16 bit -> 32 bit
2186 __ zext(value, value, 16);
2187 __ slli(tmp_reg, value, 16);
2188 __ orr(value, value, tmp_reg);
2189 break;
2190 case T_INT:
2191 shift = 2;
2192 // Short arrays (< 8 bytes) fill by element
2193 __ mv(tmp_reg, 8 >> shift);
2194 __ bltu(count, tmp_reg, L_fill_elements);
2195 break;
2196 default: ShouldNotReachHere();
2197 }
2198
2199 // Align source address at 8 bytes address boundary.
2200 Label L_skip_align1, L_skip_align2, L_skip_align4;
2201 if (!aligned) {
2202 switch (t) {
2203 case T_BYTE:
2204 // One byte misalignment happens only for byte arrays.
2205 __ test_bit(tmp_reg, to, 0);
2206 __ beqz(tmp_reg, L_skip_align1);
2207 __ sb(value, Address(to, 0));
2208 __ addi(to, to, 1);
2209 __ subiw(count, count, 1);
2210 __ bind(L_skip_align1);
2211 // Fallthrough
2212 case T_SHORT:
2213 // Two bytes misalignment happens only for byte and short (char) arrays.
2214 __ test_bit(tmp_reg, to, 1);
2215 __ beqz(tmp_reg, L_skip_align2);
2216 __ sh(value, Address(to, 0));
2217 __ addi(to, to, 2);
2218 __ subiw(count, count, 2 >> shift);
2219 __ bind(L_skip_align2);
2220 // Fallthrough
2221 case T_INT:
2222 // Align to 8 bytes, we know we are 4 byte aligned to start.
2223 __ test_bit(tmp_reg, to, 2);
2224 __ beqz(tmp_reg, L_skip_align4);
2225 __ sw(value, Address(to, 0));
2226 __ addi(to, to, 4);
2227 __ subiw(count, count, 4 >> shift);
2228 __ bind(L_skip_align4);
2229 break;
2230 default: ShouldNotReachHere();
2231 }
2232 }
2233
2234 //
2235 // Fill large chunks
2236 //
2237 __ srliw(cnt_words, count, 3 - shift); // number of words
2238
2239 // 32 bit -> 64 bit
2240 __ zext(value, value, 32);
2241 __ slli(tmp_reg, value, 32);
2242 __ orr(value, value, tmp_reg);
2243
2244 __ slli(tmp_reg, cnt_words, 3 - shift);
2245 __ subw(count, count, tmp_reg);
2246 {
2247 __ fill_words(to, cnt_words, value);
2248 }
2249
2250 // Handle copies less than 8 bytes.
2251 // Address may not be heapword aligned.
2252 Label L_fill_1, L_fill_2, L_exit;
2253 __ bind(L_fill_elements);
2254 switch (t) {
2255 case T_BYTE:
2256 __ test_bit(tmp_reg, count, 2);
2257 __ beqz(tmp_reg, L_fill_2);
2258 __ sb(value, Address(to, 0));
2259 __ sb(value, Address(to, 1));
2260 __ sb(value, Address(to, 2));
2261 __ sb(value, Address(to, 3));
2262 __ addi(to, to, 4);
2263
2264 __ bind(L_fill_2);
2265 __ test_bit(tmp_reg, count, 1);
2266 __ beqz(tmp_reg, L_fill_1);
2267 __ sb(value, Address(to, 0));
2268 __ sb(value, Address(to, 1));
2269 __ addi(to, to, 2);
2270
2271 __ bind(L_fill_1);
2272 __ test_bit(tmp_reg, count, 0);
2273 __ beqz(tmp_reg, L_exit);
2274 __ sb(value, Address(to, 0));
2275 break;
2276 case T_SHORT:
2277 __ test_bit(tmp_reg, count, 1);
2278 __ beqz(tmp_reg, L_fill_2);
2279 __ sh(value, Address(to, 0));
2280 __ sh(value, Address(to, 2));
2281 __ addi(to, to, 4);
2282
2283 __ bind(L_fill_2);
2284 __ test_bit(tmp_reg, count, 0);
2285 __ beqz(tmp_reg, L_exit);
2286 __ sh(value, Address(to, 0));
2287 break;
2288 case T_INT:
2289 __ beqz(count, L_exit);
2290 __ sw(value, Address(to, 0));
2291 break;
2292 default: ShouldNotReachHere();
2293 }
2294 __ bind(L_exit);
2295 __ leave();
2296 __ ret();
2297
2298 return start;
2299 }
2300
2301 void generate_arraycopy_stubs() {
2302 // Some copy stubs publish a normal entry and then a 2nd 'fallback'
2303 // entry immediately following their stack push. This can be used
2304 // as a post-push branch target for compatible stubs when they
2305 // identify a special case that can be handled by the fallback
2306 // stub e.g a disjoint copy stub may be use as a special case
2307 // fallback for its compatible conjoint copy stub.
2308 //
2309 // A no push entry is always returned in the following local and
2310 // then published by assigning to the appropriate entry field in
2311 // class StubRoutines. The entry value is then passed to the
2312 // generator for the compatible stub. That means the entry must be
2313 // listed when saving to/restoring from the AOT cache, ensuring
2314 // that the inter-stub jumps are noted at AOT-cache save and
2315 // relocated at AOT cache load.
2316 address nopush_entry = nullptr;
2317
2318 // generate the common exit first so later stubs can rely on it if
2319 // they want an UnsafeMemoryAccess exit non-local to the stub
2320 StubRoutines::_unsafecopy_common_exit = generate_unsafecopy_common_error_exit();
2321 // register the stub as the default exit with class UnsafeMemoryAccess
2322 UnsafeMemoryAccess::set_common_exit_stub_pc(StubRoutines::_unsafecopy_common_exit);
2323
2324 // generate and publish riscv-specific bulk copy routines first
2325 // so we can call them from other copy stubs
2326 StubRoutines::riscv::_copy_byte_f = generate_copy_longs(StubId::stubgen_copy_byte_f_id, c_rarg0, c_rarg1, t1);
2327 StubRoutines::riscv::_copy_byte_b = generate_copy_longs(StubId::stubgen_copy_byte_b_id, c_rarg0, c_rarg1, t1);
2328
2329 StubRoutines::riscv::_zero_blocks = generate_zero_blocks();
2330
2331 //*** jbyte
2332 // Always need aligned and unaligned versions
2333 StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &nopush_entry);
2334 // disjoint nopush entry is needed by conjoint copy
2335 StubRoutines::_jbyte_disjoint_arraycopy_nopush = nopush_entry;
2336 StubRoutines::_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, StubRoutines::_jbyte_disjoint_arraycopy_nopush, &nopush_entry);
2337 // conjoint nopush entry is needed by generic/unsafe copy
2338 StubRoutines::_jbyte_arraycopy_nopush = nopush_entry;
2339 StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &nopush_entry);
2340 // disjoint arrayof nopush entry is needed by conjoint copy
2341 StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush = nopush_entry;
2342 StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush, nullptr);
2343
2344 //*** jshort
2345 // Always need aligned and unaligned versions
2346 StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &nopush_entry);
2347 // disjoint nopush entry is needed by conjoint copy
2348 StubRoutines::_jshort_disjoint_arraycopy_nopush = nopush_entry;
2349 StubRoutines::_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, StubRoutines::_jshort_disjoint_arraycopy_nopush, &nopush_entry);
2350 // conjoint nopush entry is used by generic/unsafe copy
2351 StubRoutines::_jshort_arraycopy_nopush = nopush_entry;
2352 StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &nopush_entry);
2353 // disjoint arrayof nopush entry is needed by conjoint copy
2354 StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush = nopush_entry;
2355 StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush, nullptr);
2356
2357 //*** jint
2358 // Aligned versions
2359 StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &nopush_entry);
2360 // disjoint arrayof nopush entry is needed by conjoint copy
2361 StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush = nopush_entry;
2362 StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush, nullptr);
2363 // In 64 bit we need both aligned and unaligned versions of jint arraycopy.
2364 // entry_jint_arraycopy always points to the unaligned version
2365 StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &nopush_entry);
2366 // disjoint nopush entry is needed by conjoint copy
2367 StubRoutines::_jint_disjoint_arraycopy_nopush = nopush_entry;
2368 StubRoutines::_jint_arraycopy = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, StubRoutines::_jint_disjoint_arraycopy_nopush, &nopush_entry);
2369 // conjoint nopush entry is needed by generic/unsafe copy
2370 StubRoutines::_jint_arraycopy_nopush = nopush_entry;
2371
2372 //*** jlong
2373 // It is always aligned
2374 StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &nopush_entry);
2375 // disjoint arrayof nopush entry is needed by conjoint copy
2376 StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush = nopush_entry;
2377 StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush, &nopush_entry);
2378 // conjoint nopush entry is needed by generic/unsafe copy
2379 StubRoutines::_jlong_arraycopy_nopush = nopush_entry;
2380 // disjoint normal/nopush and conjoint normal entries are not
2381 // generated since the arrayof versions are the same
2382 StubRoutines::_jlong_disjoint_arraycopy = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
2383 StubRoutines::_jlong_disjoint_arraycopy_nopush = StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush;
2384 StubRoutines::_jlong_arraycopy = StubRoutines::_arrayof_jlong_arraycopy;
2385
2386 //*** oops
2387 StubRoutines::_arrayof_oop_disjoint_arraycopy
2388 = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &nopush_entry);
2389 // disjoint arrayof nopush entry is needed by conjoint copy
2390 StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush = nopush_entry;
2391 StubRoutines::_arrayof_oop_arraycopy
2392 = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush, &nopush_entry);
2393 // conjoint arrayof nopush entry is needed by generic/unsafe copy
2394 StubRoutines::_oop_arraycopy_nopush = nopush_entry;
2395 // Aligned versions without pre-barriers
2396 StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit
2397 = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
2398 // disjoint arrayof+uninit nopush entry is needed by conjoint copy
2399 StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush = nopush_entry;
2400
2401 // note that we don't need a returned nopush entry because the
2402 // generic/unsafe copy does not cater for uninit arrays.
2403 StubRoutines::_arrayof_oop_arraycopy_uninit
2404 = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush, nullptr);
2405
2406 // for oop copies reuse arrayof entries for non-arrayof cases
2407 StubRoutines::_oop_disjoint_arraycopy = StubRoutines::_arrayof_oop_disjoint_arraycopy;
2408 StubRoutines::_oop_disjoint_arraycopy_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush;
2409 StubRoutines::_oop_arraycopy = StubRoutines::_arrayof_oop_arraycopy;
2410 StubRoutines::_oop_disjoint_arraycopy_uninit = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
2411 StubRoutines::_oop_disjoint_arraycopy_uninit_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush;
2412 StubRoutines::_oop_arraycopy_uninit = StubRoutines::_arrayof_oop_arraycopy_uninit;
2413
2414 StubRoutines::_checkcast_arraycopy = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &nopush_entry);
2415 // checkcast nopush entry is needed by generic copy
2416 StubRoutines::_checkcast_arraycopy_nopush = nopush_entry;
2417 // note that we don't need a returned nopush entry because the
2418 // generic copy does not cater for uninit arrays.
2419 StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_uninit_id, nullptr);
2420
2421
2422 // unsafe arraycopy may fallback on conjoint stubs
2423 StubRoutines::_unsafe_arraycopy = generate_unsafe_copy(StubRoutines::_jbyte_arraycopy_nopush,
2424 StubRoutines::_jshort_arraycopy_nopush,
2425 StubRoutines::_jint_arraycopy_nopush,
2426 StubRoutines::_jlong_arraycopy_nopush);
2427
2428 // generic arraycopy may fallback on conjoint stubs
2429 StubRoutines::_generic_arraycopy = generate_generic_copy(StubRoutines::_jbyte_arraycopy_nopush,
2430 StubRoutines::_jshort_arraycopy_nopush,
2431 StubRoutines::_jint_arraycopy_nopush,
2432 StubRoutines::_oop_arraycopy_nopush,
2433 StubRoutines::_jlong_arraycopy_nopush,
2434 StubRoutines::_checkcast_arraycopy_nopush);
2435
2436 StubRoutines::_jbyte_fill = generate_fill(StubId::stubgen_jbyte_fill_id);
2437 StubRoutines::_jshort_fill = generate_fill(StubId::stubgen_jshort_fill_id);
2438 StubRoutines::_jint_fill = generate_fill(StubId::stubgen_jint_fill_id);
2439 StubRoutines::_arrayof_jbyte_fill = generate_fill(StubId::stubgen_arrayof_jbyte_fill_id);
2440 StubRoutines::_arrayof_jshort_fill = generate_fill(StubId::stubgen_arrayof_jshort_fill_id);
2441 StubRoutines::_arrayof_jint_fill = generate_fill(StubId::stubgen_arrayof_jint_fill_id);
2442
2443 StubRoutines::_unsafe_setmemory = generate_unsafe_setmemory();
2444 }
2445
2446 void aes_load_keys(const Register &key, VectorRegister *working_vregs, int rounds) {
2447 const int step = 16;
2448 for (int i = 0; i < rounds; i++) {
2449 __ vle32_v(working_vregs[i], key);
2450 // The keys are stored in little-endian array, while we need
2451 // to operate in big-endian.
2452 // So performing an endian-swap here with vrev8.v instruction
2453 __ vrev8_v(working_vregs[i], working_vregs[i]);
2454 __ addi(key, key, step);
2455 }
2456 }
2457
2458 void aes_encrypt(const VectorRegister &res, VectorRegister *working_vregs, int rounds) {
2459 assert(rounds <= 15, "rounds should be less than or equal to working_vregs size");
2460
2461 __ vxor_vv(res, res, working_vregs[0]);
2462 for (int i = 1; i < rounds - 1; i++) {
2463 __ vaesem_vv(res, working_vregs[i]);
2464 }
2465 __ vaesef_vv(res, working_vregs[rounds - 1]);
2466 }
2467
2468 // Arguments:
2469 //
2470 // Inputs:
2471 // c_rarg0 - source byte array address
2472 // c_rarg1 - destination byte array address
2473 // c_rarg2 - sessionKe (key) in little endian int array
2474 //
2475 address generate_aescrypt_encryptBlock() {
2476 assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support");
2477
2478 __ align(CodeEntryAlignment);
2479 StubId stub_id = StubId::stubgen_aescrypt_encryptBlock_id;
2480 StubCodeMark mark(this, stub_id);
2481
2482 Label L_aes128, L_aes192;
2483
2484 const Register from = c_rarg0; // source array address
2485 const Register to = c_rarg1; // destination array address
2486 const Register key = c_rarg2; // key array address
2487 const Register keylen = c_rarg3;
2488
2489 VectorRegister working_vregs[] = {
2490 v4, v5, v6, v7, v8, v9, v10, v11,
2491 v12, v13, v14, v15, v16, v17, v18
2492 };
2493 const VectorRegister res = v19;
2494
2495 address start = __ pc();
2496 __ enter();
2497
2498 __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2499
2500 __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
2501 __ vle32_v(res, from);
2502
2503 __ mv(t2, 52); // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
2504 __ bltu(keylen, t2, L_aes128);
2505 __ beq(keylen, t2, L_aes192);
2506 // Else we fallthrough to the biggest case (256-bit key size)
2507
2508 // Note: the following function performs key += 15*16
2509 aes_load_keys(key, working_vregs, 15);
2510 aes_encrypt(res, working_vregs, 15);
2511 __ vse32_v(res, to);
2512 __ mv(c_rarg0, 0);
2513 __ leave();
2514 __ ret();
2515
2516 __ bind(L_aes192);
2517 // Note: the following function performs key += 13*16
2518 aes_load_keys(key, working_vregs, 13);
2519 aes_encrypt(res, working_vregs, 13);
2520 __ vse32_v(res, to);
2521 __ mv(c_rarg0, 0);
2522 __ leave();
2523 __ ret();
2524
2525 __ bind(L_aes128);
2526 // Note: the following function performs key += 11*16
2527 aes_load_keys(key, working_vregs, 11);
2528 aes_encrypt(res, working_vregs, 11);
2529 __ vse32_v(res, to);
2530 __ mv(c_rarg0, 0);
2531 __ leave();
2532 __ ret();
2533
2534 return start;
2535 }
2536
2537 void aes_decrypt(const VectorRegister &res, VectorRegister *working_vregs, int rounds) {
2538 assert(rounds <= 15, "rounds should be less than or equal to working_vregs size");
2539
2540 __ vxor_vv(res, res, working_vregs[rounds - 1]);
2541 for (int i = rounds - 2; i > 0; i--) {
2542 __ vaesdm_vv(res, working_vregs[i]);
2543 }
2544 __ vaesdf_vv(res, working_vregs[0]);
2545 }
2546
2547 // Arguments:
2548 //
2549 // Inputs:
2550 // c_rarg0 - source byte array address
2551 // c_rarg1 - destination byte array address
2552 // c_rarg2 - sessionKe (key) in little endian int array
2553 //
2554 address generate_aescrypt_decryptBlock() {
2555 assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support");
2556
2557 __ align(CodeEntryAlignment);
2558 StubId stub_id = StubId::stubgen_aescrypt_decryptBlock_id;
2559 StubCodeMark mark(this, stub_id);
2560
2561 Label L_aes128, L_aes192;
2562
2563 const Register from = c_rarg0; // source array address
2564 const Register to = c_rarg1; // destination array address
2565 const Register key = c_rarg2; // key array address
2566 const Register keylen = c_rarg3;
2567
2568 VectorRegister working_vregs[] = {
2569 v4, v5, v6, v7, v8, v9, v10, v11,
2570 v12, v13, v14, v15, v16, v17, v18
2571 };
2572 const VectorRegister res = v19;
2573
2574 address start = __ pc();
2575 __ enter(); // required for proper stackwalking of RuntimeStub frame
2576
2577 __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2578
2579 __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
2580 __ vle32_v(res, from);
2581
2582 __ mv(t2, 52); // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
2583 __ bltu(keylen, t2, L_aes128);
2584 __ beq(keylen, t2, L_aes192);
2585 // Else we fallthrough to the biggest case (256-bit key size)
2586
2587 // Note: the following function performs key += 15*16
2588 aes_load_keys(key, working_vregs, 15);
2589 aes_decrypt(res, working_vregs, 15);
2590 __ vse32_v(res, to);
2591 __ mv(c_rarg0, 0);
2592 __ leave();
2593 __ ret();
2594
2595 __ bind(L_aes192);
2596 // Note: the following function performs key += 13*16
2597 aes_load_keys(key, working_vregs, 13);
2598 aes_decrypt(res, working_vregs, 13);
2599 __ vse32_v(res, to);
2600 __ mv(c_rarg0, 0);
2601 __ leave();
2602 __ ret();
2603
2604 __ bind(L_aes128);
2605 // Note: the following function performs key += 11*16
2606 aes_load_keys(key, working_vregs, 11);
2607 aes_decrypt(res, working_vregs, 11);
2608 __ vse32_v(res, to);
2609 __ mv(c_rarg0, 0);
2610 __ leave();
2611 __ ret();
2612
2613 return start;
2614 }
2615
2616 void cipherBlockChaining_encryptAESCrypt(int round, Register from, Register to, Register key,
2617 Register rvec, Register input_len) {
2618 const Register len = x29;
2619
2620 VectorRegister working_vregs[] = {
2621 v1, v2, v3, v4, v5, v6, v7, v8,
2622 v9, v10, v11, v12, v13, v14, v15
2623 };
2624
2625 const unsigned int BLOCK_SIZE = 16;
2626
2627 __ mv(len, input_len);
2628 // load init rvec
2629 __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
2630 __ vle32_v(v16, rvec);
2631
2632 aes_load_keys(key, working_vregs, round);
2633 Label L_enc_loop;
2634 __ bind(L_enc_loop);
2635 // Encrypt from source by block size
2636 __ vle32_v(v17, from);
2637 __ addi(from, from, BLOCK_SIZE);
2638 __ vxor_vv(v16, v16, v17);
2639 aes_encrypt(v16, working_vregs, round);
2640 __ vse32_v(v16, to);
2641 __ addi(to, to, BLOCK_SIZE);
2642 __ subi(len, len, BLOCK_SIZE);
2643 __ bnez(len, L_enc_loop);
2644
2645 // save current rvec and return
2646 __ vse32_v(v16, rvec);
2647 __ mv(x10, input_len);
2648 __ leave();
2649 __ ret();
2650 }
2651
2652 // Arguments:
2653 //
2654 // Inputs:
2655 // c_rarg0 - source byte array address
2656 // c_rarg1 - destination byte array address
2657 // c_rarg2 - K (key) in little endian int array
2658 // c_rarg3 - r vector byte array address
2659 // c_rarg4 - input length
2660 //
2661 // Output:
2662 // x10 - input length
2663 //
2664 address generate_cipherBlockChaining_encryptAESCrypt() {
2665 assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support");
2666 __ align(CodeEntryAlignment);
2667 StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id;
2668 StubCodeMark mark(this, stub_id);
2669
2670 const Register from = c_rarg0;
2671 const Register to = c_rarg1;
2672 const Register key = c_rarg2;
2673 const Register rvec = c_rarg3;
2674 const Register input_len = c_rarg4;
2675
2676 const Register keylen = x28;
2677
2678 address start = __ pc();
2679 __ enter();
2680
2681 Label L_aes128, L_aes192;
2682 // Compute #rounds for AES based on the length of the key array
2683 __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2684 __ mv(t0, 52);
2685 __ bltu(keylen, t0, L_aes128);
2686 __ beq(keylen, t0, L_aes192);
2687 // Else we fallthrough to the biggest case (256-bit key size)
2688
2689 // Note: the following function performs key += 15*16
2690 cipherBlockChaining_encryptAESCrypt(15, from, to, key, rvec, input_len);
2691
2692 // Note: the following function performs key += 11*16
2693 __ bind(L_aes128);
2694 cipherBlockChaining_encryptAESCrypt(11, from, to, key, rvec, input_len);
2695
2696 // Note: the following function performs key += 13*16
2697 __ bind(L_aes192);
2698 cipherBlockChaining_encryptAESCrypt(13, from, to, key, rvec, input_len);
2699
2700 return start;
2701 }
2702
2703 void cipherBlockChaining_decryptAESCrypt(int round, Register from, Register to, Register key,
2704 Register rvec, Register input_len) {
2705 const Register len = x29;
2706
2707 VectorRegister working_vregs[] = {
2708 v1, v2, v3, v4, v5, v6, v7, v8,
2709 v9, v10, v11, v12, v13, v14, v15
2710 };
2711
2712 const unsigned int BLOCK_SIZE = 16;
2713
2714 __ mv(len, input_len);
2715 // load init rvec
2716 __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
2717 __ vle32_v(v16, rvec);
2718
2719 aes_load_keys(key, working_vregs, round);
2720 Label L_dec_loop;
2721 // Decrypt from source by block size
2722 __ bind(L_dec_loop);
2723 __ vle32_v(v17, from);
2724 __ addi(from, from, BLOCK_SIZE);
2725 __ vmv_v_v(v18, v17);
2726 aes_decrypt(v17, working_vregs, round);
2727 __ vxor_vv(v17, v17, v16);
2728 __ vse32_v(v17, to);
2729 __ vmv_v_v(v16, v18);
2730 __ addi(to, to, BLOCK_SIZE);
2731 __ subi(len, len, BLOCK_SIZE);
2732 __ bnez(len, L_dec_loop);
2733
2734 // save current rvec and return
2735 __ vse32_v(v16, rvec);
2736 __ mv(x10, input_len);
2737 __ leave();
2738 __ ret();
2739 }
2740
2741 // Arguments:
2742 //
2743 // Inputs:
2744 // c_rarg0 - source byte array address
2745 // c_rarg1 - destination byte array address
2746 // c_rarg2 - K (key) in little endian int array
2747 // c_rarg3 - r vector byte array address
2748 // c_rarg4 - input length
2749 //
2750 // Output:
2751 // x10 - input length
2752 //
2753 address generate_cipherBlockChaining_decryptAESCrypt() {
2754 assert(UseAESIntrinsics, "need AES instructions (Zvkned extension) support");
2755 __ align(CodeEntryAlignment);
2756 StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
2757 StubCodeMark mark(this, stub_id);
2758
2759 const Register from = c_rarg0;
2760 const Register to = c_rarg1;
2761 const Register key = c_rarg2;
2762 const Register rvec = c_rarg3;
2763 const Register input_len = c_rarg4;
2764
2765 const Register keylen = x28;
2766
2767 address start = __ pc();
2768 __ enter();
2769
2770 Label L_aes128, L_aes192, L_aes128_loop, L_aes192_loop, L_aes256_loop;
2771 // Compute #rounds for AES based on the length of the key array
2772 __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2773 __ mv(t0, 52);
2774 __ bltu(keylen, t0, L_aes128);
2775 __ beq(keylen, t0, L_aes192);
2776 // Else we fallthrough to the biggest case (256-bit key size)
2777
2778 // Note: the following function performs key += 15*16
2779 cipherBlockChaining_decryptAESCrypt(15, from, to, key, rvec, input_len);
2780
2781 // Note: the following function performs key += 11*16
2782 __ bind(L_aes128);
2783 cipherBlockChaining_decryptAESCrypt(11, from, to, key, rvec, input_len);
2784
2785 // Note: the following function performs key += 13*16
2786 __ bind(L_aes192);
2787 cipherBlockChaining_decryptAESCrypt(13, from, to, key, rvec, input_len);
2788
2789 return start;
2790 }
2791
2792 // Load big-endian 128-bit from memory.
2793 void be_load_counter_128(Register counter_hi, Register counter_lo, Register counter) {
2794 __ ld(counter_lo, Address(counter, 8)); // Load 128-bits from counter
2795 __ ld(counter_hi, Address(counter));
2796 __ rev8(counter_lo, counter_lo); // Convert big-endian to little-endian
2797 __ rev8(counter_hi, counter_hi);
2798 }
2799
2800 // Little-endian 128-bit + 64-bit -> 128-bit addition.
2801 void add_counter_128(Register counter_hi, Register counter_lo) {
2802 assert_different_registers(counter_hi, counter_lo, t0);
2803 __ addi(counter_lo, counter_lo, 1);
2804 __ seqz(t0, counter_lo); // Check for result overflow
2805 __ add(counter_hi, counter_hi, t0); // Add 1 if overflow otherwise 0
2806 }
2807
2808 // Store big-endian 128-bit to memory.
2809 void be_store_counter_128(Register counter_hi, Register counter_lo, Register counter) {
2810 assert_different_registers(counter_hi, counter_lo, t0, t1);
2811 __ rev8(t0, counter_lo); // Convert little-endian to big-endian
2812 __ rev8(t1, counter_hi);
2813 __ sd(t0, Address(counter, 8)); // Store 128-bits to counter
2814 __ sd(t1, Address(counter));
2815 }
2816
2817 void counterMode_AESCrypt(int round, Register in, Register out, Register key, Register counter,
2818 Register input_len, Register saved_encrypted_ctr, Register used_ptr) {
2819 // Algorithm:
2820 //
2821 // aes_load_keys();
2822 // load_counter_128(counter_hi, counter_lo, counter);
2823 //
2824 // L_next:
2825 // if (used >= BLOCK_SIZE) goto L_main_loop;
2826 //
2827 // L_encrypt_next:
2828 // *out = *in ^ saved_encrypted_ctr[used]);
2829 // out++; in++; used++; len--;
2830 // if (len == 0) goto L_exit;
2831 // goto L_next;
2832 //
2833 // L_main_loop:
2834 // if (len == 0) goto L_exit;
2835 // saved_encrypted_ctr = aes_encrypt(counter);
2836 //
2837 // add_counter_128(counter_hi, counter_lo);
2838 // be_store_counter_128(counter_hi, counter_lo, counter);
2839 // used = 0;
2840 //
2841 // if(len < BLOCK_SIZE) goto L_encrypt_next;
2842 //
2843 // v_in = load_16Byte(in);
2844 // v_out = load_16Byte(out);
2845 // v_saved_encrypted_ctr = load_16Byte(saved_encrypted_ctr);
2846 // v_out = v_in ^ v_saved_encrypted_ctr;
2847 // out += BLOCK_SIZE;
2848 // in += BLOCK_SIZE;
2849 // len -= BLOCK_SIZE;
2850 // used = BLOCK_SIZE;
2851 // goto L_main_loop;
2852 //
2853 //
2854 // L_exit:
2855 // store(used);
2856 // result = input_len
2857 // return result;
2858
2859 const Register used = x28;
2860 const Register len = x29;
2861 const Register counter_hi = x30;
2862 const Register counter_lo = x31;
2863 const Register block_size = t2;
2864
2865 const unsigned int BLOCK_SIZE = 16;
2866
2867 VectorRegister working_vregs[] = {
2868 v1, v2, v3, v4, v5, v6, v7, v8,
2869 v9, v10, v11, v12, v13, v14, v15
2870 };
2871
2872 __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
2873
2874 __ lwu(used, Address(used_ptr));
2875 __ mv(len, input_len);
2876 __ mv(block_size, BLOCK_SIZE);
2877
2878 // load keys to working_vregs according to round
2879 aes_load_keys(key, working_vregs, round);
2880
2881 // 128-bit big-endian load
2882 be_load_counter_128(counter_hi, counter_lo, counter);
2883
2884 Label L_next, L_encrypt_next, L_main_loop, L_exit;
2885 // Check the last saved_encrypted_ctr used value, we fall through
2886 // to L_encrypt_next when the used value lower than block_size
2887 __ bind(L_next);
2888 __ bgeu(used, block_size, L_main_loop);
2889
2890 // There is still data left fewer than block_size after L_main_loop
2891 // or last used, we encrypt them one by one.
2892 __ bind(L_encrypt_next);
2893 __ add(t0, saved_encrypted_ctr, used);
2894 __ lbu(t1, Address(t0));
2895 __ lbu(t0, Address(in));
2896 __ xorr(t1, t1, t0);
2897 __ sb(t1, Address(out));
2898 __ addi(in, in, 1);
2899 __ addi(out, out, 1);
2900 __ addi(used, used, 1);
2901 __ subi(len, len, 1);
2902 __ beqz(len, L_exit);
2903 __ j(L_next);
2904
2905 // We will calculate the next saved_encrypted_ctr and encrypt the blocks of data
2906 // one by one until there is less than a full block remaining if len not zero
2907 __ bind(L_main_loop);
2908 __ beqz(len, L_exit);
2909 __ vle32_v(v16, counter);
2910
2911 // encrypt counter according to round
2912 aes_encrypt(v16, working_vregs, round);
2913
2914 __ vse32_v(v16, saved_encrypted_ctr);
2915
2916 // 128-bit little-endian increment
2917 add_counter_128(counter_hi, counter_lo);
2918 // 128-bit big-endian store
2919 be_store_counter_128(counter_hi, counter_lo, counter);
2920
2921 __ mv(used, 0);
2922 // Check if we have a full block_size
2923 __ bltu(len, block_size, L_encrypt_next);
2924
2925 // We have one full block to encrypt at least
2926 __ vle32_v(v17, in);
2927 __ vxor_vv(v16, v16, v17);
2928 __ vse32_v(v16, out);
2929 __ add(out, out, block_size);
2930 __ add(in, in, block_size);
2931 __ sub(len, len, block_size);
2932 __ mv(used, block_size);
2933 __ j(L_main_loop);
2934
2935 __ bind(L_exit);
2936 __ sw(used, Address(used_ptr));
2937 __ mv(x10, input_len);
2938 __ leave();
2939 __ ret();
2940 };
2941
2942 // CTR AES crypt.
2943 // Arguments:
2944 //
2945 // Inputs:
2946 // c_rarg0 - source byte array address
2947 // c_rarg1 - destination byte array address
2948 // c_rarg2 - K (key) in little endian int array
2949 // c_rarg3 - counter vector byte array address
2950 // c_rarg4 - input length
2951 // c_rarg5 - saved encryptedCounter start
2952 // c_rarg6 - saved used length
2953 //
2954 // Output:
2955 // x10 - input length
2956 //
2957 address generate_counterMode_AESCrypt() {
2958 assert(UseAESCTRIntrinsics, "need AES instructions (Zvkned extension) and Zbb extension support");
2959
2960 __ align(CodeEntryAlignment);
2961 StubId stub_id = StubId::stubgen_counterMode_AESCrypt_id;
2962 StubCodeMark mark(this, stub_id);
2963
2964 const Register in = c_rarg0;
2965 const Register out = c_rarg1;
2966 const Register key = c_rarg2;
2967 const Register counter = c_rarg3;
2968 const Register input_len = c_rarg4;
2969 const Register saved_encrypted_ctr = c_rarg5;
2970 const Register used_len_ptr = c_rarg6;
2971
2972 const Register keylen = c_rarg7; // temporary register
2973
2974 const address start = __ pc();
2975 __ enter();
2976
2977 Label L_exit;
2978 __ beqz(input_len, L_exit);
2979
2980 Label L_aes128, L_aes192;
2981 // Compute #rounds for AES based on the length of the key array
2982 __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2983 __ mv(t0, 52); // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
2984 __ bltu(keylen, t0, L_aes128);
2985 __ beq(keylen, t0, L_aes192);
2986 // Else we fallthrough to the biggest case (256-bit key size)
2987
2988 // Note: the following function performs crypt with key += 15*16
2989 counterMode_AESCrypt(15, in, out, key, counter, input_len, saved_encrypted_ctr, used_len_ptr);
2990
2991 // Note: the following function performs crypt with key += 13*16
2992 __ bind(L_aes192);
2993 counterMode_AESCrypt(13, in, out, key, counter, input_len, saved_encrypted_ctr, used_len_ptr);
2994
2995 // Note: the following function performs crypt with key += 11*16
2996 __ bind(L_aes128);
2997 counterMode_AESCrypt(11, in, out, key, counter, input_len, saved_encrypted_ctr, used_len_ptr);
2998
2999 __ bind(L_exit);
3000 __ mv(x10, input_len);
3001 __ leave();
3002 __ ret();
3003
3004 return start;
3005 }
3006
3007 void ghash_loop(Register state, Register subkeyH, Register data, Register blocks,
3008 VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3) {
3009 VectorRegister partial_hash = vtmp1;
3010 VectorRegister hash_subkey = vtmp2;
3011 VectorRegister cipher_text = vtmp3;
3012
3013 const unsigned int BLOCK_SIZE = 16;
3014
3015 __ vsetivli(x0, 2, Assembler::e64, Assembler::m1);
3016 __ vle64_v(hash_subkey, subkeyH);
3017 __ vrev8_v(hash_subkey, hash_subkey);
3018 __ vle64_v(partial_hash, state);
3019 __ vrev8_v(partial_hash, partial_hash);
3020
3021 __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
3022 Label L_ghash_loop;
3023 __ bind(L_ghash_loop);
3024 __ vle32_v(cipher_text, data);
3025 __ addi(data, data, BLOCK_SIZE);
3026 __ vghsh_vv(partial_hash, hash_subkey, cipher_text);
3027 __ subi(blocks, blocks, 1);
3028 __ bnez(blocks, L_ghash_loop);
3029
3030 __ vsetivli(x0, 2, Assembler::e64, Assembler::m1);
3031 __ vrev8_v(partial_hash, partial_hash);
3032 __ vse64_v(partial_hash, state);
3033 }
3034
3035 /**
3036 * Arguments:
3037 *
3038 * Input:
3039 * c_rarg0 - current state address
3040 * c_rarg1 - H key address
3041 * c_rarg2 - data address
3042 * c_rarg3 - number of blocks
3043 *
3044 * Output:
3045 * Updated state at c_rarg0
3046 */
3047 address generate_ghash_processBlocks() {
3048 assert(UseGHASHIntrinsics, "need GHASH instructions (Zvkg extension) and Zvbb support");
3049
3050 __ align(CodeEntryAlignment);
3051 StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
3052 StubCodeMark mark(this, stub_id);
3053
3054 address start = __ pc();
3055 __ enter();
3056
3057 Register state = c_rarg0;
3058 Register subkeyH = c_rarg1;
3059 Register data = c_rarg2;
3060 Register blocks = c_rarg3;
3061
3062 VectorRegister vtmp1 = v1;
3063 VectorRegister vtmp2 = v2;
3064 VectorRegister vtmp3 = v3;
3065
3066 ghash_loop(state, subkeyH, data, blocks, vtmp1, vtmp2, vtmp3);
3067
3068 __ leave();
3069 __ ret();
3070
3071 return start;
3072 }
3073
3074 void gcm_counterMode_AESCrypt_blocks(int round, Register in, Register out, Register key, Register counter,
3075 Register input_len, VectorRegister *working_vregs, Register blocks,
3076 VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3) {
3077 __ srli(blocks, input_len, 4);
3078
3079 const unsigned int BLOCK_SIZE = 16;
3080 const unsigned int MASK_VALUE = 0b1000; // we need {1, 0, 0, 0} mask value here
3081 __ vsetivli(x0, 1, Assembler::e8, Assembler::m1);
3082 __ vmv_v_i(v0, MASK_VALUE);
3083
3084 __ vsetivli(x0, 4, Assembler::e32, Assembler::m1);
3085 // load keys to working_vregs according to round
3086 aes_load_keys(key, working_vregs, round);
3087
3088 __ vle32_v(vtmp1, counter);
3089 Label L_aes_ctr_loop;
3090 __ bind(L_aes_ctr_loop);
3091 __ vmv_v_v(vtmp2, vtmp1);
3092 // encrypt counter according to round
3093 aes_encrypt(vtmp2, working_vregs, round);
3094 __ vle32_v(vtmp3, in);
3095 __ vxor_vv(vtmp2, vtmp2, vtmp3);
3096 __ vse32_v(vtmp2, out);
3097 __ addi(out, out, BLOCK_SIZE);
3098 __ addi(in, in, BLOCK_SIZE);
3099 __ sub(blocks, blocks, 1);
3100 __ vrev8_v(vtmp1, vtmp1, Assembler::VectorMask::v0_t);
3101 __ vadd_vi(vtmp1, vtmp1, 0x1, Assembler::VectorMask::v0_t);
3102 __ vrev8_v(vtmp1, vtmp1, Assembler::VectorMask::v0_t);
3103 __ bnez(blocks, L_aes_ctr_loop);
3104
3105 __ vse32_v(vtmp1, counter);
3106 }
3107
3108 void gcm_ghash_blocks(Register state, Register subkeyH, Register ct, Register input_len, Register blocks,
3109 VectorRegister vtmp1, VectorRegister vtmp2, VectorRegister vtmp3) {
3110 __ srli(blocks, input_len, 4);
3111
3112 ghash_loop(state, subkeyH, ct, blocks, vtmp1, vtmp2, vtmp3);
3113
3114 __ mv(x10, input_len);
3115 __ leave();
3116 __ ret();
3117 }
3118
3119
3120 // Vector AES Galois Counter Mode implementation. Parameters:
3121 //
3122 // in = c_rarg0
3123 // input_len = c_rarg1
3124 // ct = c_rarg2 - ciphertext that ghash will read (out for encrypt, in for decrypt)
3125 // out = c_rarg3
3126 // key = c_rarg4
3127 // state = c_rarg5 - GHASH.state
3128 // subkeyHtbl = c_rarg6 - powers of H
3129 // counter = c_rarg7 - 16 bytes of CTR
3130 // return - number of processed bytes
3131 address generate_galoisCounterMode_AESCrypt() {
3132 assert(UseGHASHIntrinsics, "need GHASH instructions (Zvkg extension) and Zvbb support");
3133 assert(UseAESCTRIntrinsics, "need AES instructions (Zvkned extension) and Zbb extension support");
3134
3135 __ align(CodeEntryAlignment);
3136 StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
3137 StubCodeMark mark(this, stub_id);
3138
3139 const Register in = c_rarg0;
3140 const Register input_len = c_rarg1;
3141 const Register ct = c_rarg2;
3142 const Register out = c_rarg3;
3143 const Register key = c_rarg4;
3144 const Register state = c_rarg5;
3145 const Register subkeyHtbl = c_rarg6;
3146 const Register counter = c_rarg7;
3147
3148 const Register keylen = x28;
3149 const Register blocks = x29;
3150
3151 VectorRegister working_vregs[] = {
3152 v1, v2, v3, v4, v5, v6, v7, v8,
3153 v9, v10, v11, v12, v13, v14, v15
3154 };
3155
3156 VectorRegister vtmp1 = v16;
3157 VectorRegister vtmp2 = v17;
3158 VectorRegister vtmp3 = v18;
3159
3160 const address start = __ pc();
3161 __ enter();
3162
3163 Label L_exit;
3164 // Requires input_len (512) bytes to efficiently use the intrinsic
3165 __ andi(input_len, input_len, -512);
3166 __ beqz(input_len, L_exit);
3167
3168 Label L_aes128, L_aes192;
3169 // Compute #rounds for AES based on the length of the key array
3170 __ lwu(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3171 __ mv(t0, 52); // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
3172 __ bltu(keylen, t0, L_aes128);
3173 __ beq(keylen, t0, L_aes192);
3174 // Else we fallthrough to the biggest case (256-bit key size)
3175
3176 // Note: the following function performs crypt with key += 15*16
3177 gcm_counterMode_AESCrypt_blocks(15, in, out, key, counter, input_len, working_vregs, blocks, vtmp1, vtmp2, vtmp3);
3178 gcm_ghash_blocks(state, subkeyHtbl, ct, input_len, blocks, vtmp1, vtmp2, vtmp3);
3179
3180 // Note: the following function performs crypt with key += 13*16
3181 __ bind(L_aes192);
3182 gcm_counterMode_AESCrypt_blocks(13, in, out, key, counter, input_len, working_vregs, blocks, vtmp1, vtmp2, vtmp3);
3183 gcm_ghash_blocks(state, subkeyHtbl, ct, input_len, blocks, vtmp1, vtmp2, vtmp3);
3184
3185 // Note: the following function performs crypt with key += 11*16
3186 __ bind(L_aes128);
3187 gcm_counterMode_AESCrypt_blocks(11, in, out, key, counter, input_len, working_vregs, blocks, vtmp1, vtmp2, vtmp3);
3188 gcm_ghash_blocks(state, subkeyHtbl, ct, input_len, blocks, vtmp1, vtmp2, vtmp3);
3189
3190 __ bind(L_exit);
3191 __ mv(x10, input_len);
3192 __ leave();
3193 __ ret();
3194
3195 return start;
3196 }
3197
3198 // code for comparing 8 characters of strings with Latin1 and Utf16 encoding
3199 void compare_string_8_x_LU(Register tmpL, Register tmpU,
3200 Register strL, Register strU, Label& DIFF) {
3201 const Register tmp = x30, tmpLval = x12;
3202
3203 int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
3204 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
3205
3206 #ifdef ASSERT
3207 if (AvoidUnalignedAccesses) {
3208 Label align_ok;
3209 __ andi(t0, strL, 0x7);
3210 __ beqz(t0, align_ok);
3211 __ stop("bad alignment");
3212 __ bind(align_ok);
3213 }
3214 #endif
3215 __ ld(tmpLval, Address(strL));
3216 __ addi(strL, strL, wordSize);
3217
3218 // compare first 4 characters
3219 __ load_long_misaligned(tmpU, Address(strU), tmp, (base_offset % 8) != 0 ? 4 : 8);
3220 __ addi(strU, strU, wordSize);
3221 __ inflate_lo32(tmpL, tmpLval);
3222 __ xorr(tmp, tmpU, tmpL);
3223 __ bnez(tmp, DIFF);
3224
3225 // compare second 4 characters
3226 __ load_long_misaligned(tmpU, Address(strU), tmp, (base_offset % 8) != 0 ? 4 : 8);
3227 __ addi(strU, strU, wordSize);
3228 __ inflate_hi32(tmpL, tmpLval);
3229 __ xorr(tmp, tmpU, tmpL);
3230 __ bnez(tmp, DIFF);
3231 }
3232
3233 // x10 = result
3234 // x11 = str1
3235 // x12 = cnt1
3236 // x13 = str2
3237 // x14 = cnt2
3238 // x28 = tmp1
3239 // x29 = tmp2
3240 // x30 = tmp3
3241 address generate_compare_long_string_different_encoding(StubId stub_id) {
3242 bool isLU;
3243 switch (stub_id) {
3244 case StubId::stubgen_compare_long_string_LU_id:
3245 isLU = true;
3246 break;
3247 case StubId::stubgen_compare_long_string_UL_id:
3248 isLU = false;
3249 break;
3250 default:
3251 ShouldNotReachHere();
3252 };
3253 __ align(CodeEntryAlignment);
3254 StubCodeMark mark(this, stub_id);
3255 address entry = __ pc();
3256 Label SMALL_LOOP, TAIL, LOAD_LAST, DONE, CALCULATE_DIFFERENCE;
3257 const Register result = x10, str1 = x11, str2 = x13, cnt2 = x14,
3258 tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x12;
3259
3260 int base_offset = arrayOopDesc::base_offset_in_bytes(T_BYTE);
3261 assert((base_offset % (UseCompactObjectHeaders ? 4 : 8)) == 0, "Must be");
3262
3263 Register strU = isLU ? str2 : str1,
3264 strL = isLU ? str1 : str2,
3265 tmpU = isLU ? tmp2 : tmp1, // where to keep U for comparison
3266 tmpL = isLU ? tmp1 : tmp2; // where to keep L for comparison
3267
3268 if (AvoidUnalignedAccesses && (base_offset % 8) != 0) {
3269 // Load 4 bytes from strL to make sure main loop is 8-byte aligned
3270 // cnt2 is >= 68 here, no need to check it for >= 0
3271 __ lwu(tmpL, Address(strL));
3272 __ addi(strL, strL, wordSize / 2);
3273 __ load_long_misaligned(tmpU, Address(strU), tmp4, (base_offset % 8) != 0 ? 4 : 8);
3274 __ addi(strU, strU, wordSize);
3275 __ inflate_lo32(tmp3, tmpL);
3276 __ mv(tmpL, tmp3);
3277 __ xorr(tmp3, tmpU, tmpL);
3278 __ bnez(tmp3, CALCULATE_DIFFERENCE);
3279 __ subi(cnt2, cnt2, wordSize / 2);
3280 }
3281
3282 // we are now 8-bytes aligned on strL when AvoidUnalignedAccesses is true
3283 __ subi(cnt2, cnt2, wordSize * 2);
3284 __ bltz(cnt2, TAIL);
3285 __ bind(SMALL_LOOP); // smaller loop
3286 __ subi(cnt2, cnt2, wordSize * 2);
3287 compare_string_8_x_LU(tmpL, tmpU, strL, strU, CALCULATE_DIFFERENCE);
3288 compare_string_8_x_LU(tmpL, tmpU, strL, strU, CALCULATE_DIFFERENCE);
3289 __ bgez(cnt2, SMALL_LOOP);
3290 __ addi(t0, cnt2, wordSize * 2);
3291 __ beqz(t0, DONE);
3292 __ bind(TAIL); // 1..15 characters left
3293 // Aligned access. Load bytes in portions - 4, 2, 1.
3294
3295 __ addi(t0, cnt2, wordSize);
3296 __ addi(cnt2, cnt2, wordSize * 2); // amount of characters left to process
3297 __ bltz(t0, LOAD_LAST);
3298 // remaining characters are greater than or equals to 8, we can do one compare_string_8_x_LU
3299 compare_string_8_x_LU(tmpL, tmpU, strL, strU, CALCULATE_DIFFERENCE);
3300 __ subi(cnt2, cnt2, wordSize);
3301 __ beqz(cnt2, DONE); // no character left
3302 __ bind(LOAD_LAST); // cnt2 = 1..7 characters left
3303
3304 __ subi(cnt2, cnt2, wordSize); // cnt2 is now an offset in strL which points to last 8 bytes
3305 __ slli(t0, cnt2, 1); // t0 is now an offset in strU which points to last 16 bytes
3306 __ add(strL, strL, cnt2); // Address of last 8 bytes in Latin1 string
3307 __ add(strU, strU, t0); // Address of last 16 bytes in UTF-16 string
3308 __ load_int_misaligned(tmpL, Address(strL), t0, false);
3309 __ load_long_misaligned(tmpU, Address(strU), t0, 2);
3310 __ inflate_lo32(tmp3, tmpL);
3311 __ mv(tmpL, tmp3);
3312 __ xorr(tmp3, tmpU, tmpL);
3313 __ bnez(tmp3, CALCULATE_DIFFERENCE);
3314
3315 __ addi(strL, strL, wordSize / 2); // Address of last 4 bytes in Latin1 string
3316 __ addi(strU, strU, wordSize); // Address of last 8 bytes in UTF-16 string
3317 __ load_int_misaligned(tmpL, Address(strL), t0, false);
3318 __ load_long_misaligned(tmpU, Address(strU), t0, 2);
3319 __ inflate_lo32(tmp3, tmpL);
3320 __ mv(tmpL, tmp3);
3321 __ xorr(tmp3, tmpU, tmpL);
3322 __ bnez(tmp3, CALCULATE_DIFFERENCE);
3323 __ j(DONE); // no character left
3324
3325 // Find the first different characters in the longwords and
3326 // compute their difference.
3327 __ bind(CALCULATE_DIFFERENCE);
3328 // count bits of trailing zero chars
3329 __ ctzc_bits(tmp4, tmp3);
3330 __ srl(tmp1, tmp1, tmp4);
3331 __ srl(tmp2, tmp2, tmp4);
3332 __ zext(tmp1, tmp1, 16);
3333 __ zext(tmp2, tmp2, 16);
3334 __ sub(result, tmp1, tmp2);
3335 __ bind(DONE);
3336 __ ret();
3337 return entry;
3338 }
3339
3340 address generate_method_entry_barrier() {
3341 __ align(CodeEntryAlignment);
3342 StubId stub_id = StubId::stubgen_method_entry_barrier_id;
3343 StubCodeMark mark(this, stub_id);
3344
3345 Label deoptimize_label;
3346
3347 address start = __ pc();
3348
3349 BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
3350
3351 if (bs_asm->nmethod_patching_type() == NMethodPatchingType::conc_instruction_and_data_patch) {
3352 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
3353 Address thread_epoch_addr(xthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()) + 4);
3354 __ la(t1, ExternalAddress(bs_asm->patching_epoch_addr()));
3355 __ lwu(t1, t1);
3356 __ sw(t1, thread_epoch_addr);
3357 // There are two ways this can work:
3358 // - The writer did system icache shootdown after the instruction stream update.
3359 // Hence do nothing.
3360 // - The writer trust us to make sure our icache is in sync before entering.
3361 // Hence use cmodx fence (fence.i, may change).
3362 if (UseCtxFencei) {
3363 __ cmodx_fence();
3364 }
3365 __ membar(__ LoadLoad);
3366 }
3367
3368 __ set_last_Java_frame(sp, fp, ra);
3369
3370 __ enter();
3371 __ addi(t1, sp, wordSize);
3372
3373 __ subi(sp, sp, 4 * wordSize);
3374
3375 __ push_call_clobbered_registers();
3376
3377 __ mv(c_rarg0, t1);
3378 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetNMethod::nmethod_stub_entry_barrier), 1);
3379
3380 __ reset_last_Java_frame(true);
3381
3382 __ mv(t0, x10);
3383
3384 __ pop_call_clobbered_registers();
3385
3386 __ bnez(t0, deoptimize_label);
3387
3388 __ leave();
3389 __ ret();
3390
3391 __ BIND(deoptimize_label);
3392
3393 __ ld(t0, Address(sp, 0));
3394 __ ld(fp, Address(sp, wordSize));
3395 __ ld(ra, Address(sp, wordSize * 2));
3396 __ ld(t1, Address(sp, wordSize * 3));
3397
3398 __ mv(sp, t0);
3399 __ jr(t1);
3400
3401 return start;
3402 }
3403
3404 // x10 = result
3405 // x11 = str1
3406 // x12 = cnt1
3407 // x13 = str2
3408 // x14 = cnt2
3409 // x28 = tmp1
3410 // x29 = tmp2
3411 // x30 = tmp3
3412 // x31 = tmp4
3413 address generate_compare_long_string_same_encoding(StubId stub_id) {
3414 bool isLL;
3415 switch (stub_id) {
3416 case StubId::stubgen_compare_long_string_LL_id:
3417 isLL = true;
3418 break;
3419 case StubId::stubgen_compare_long_string_UU_id:
3420 isLL = false;
3421 break;
3422 default:
3423 ShouldNotReachHere();
3424 };
3425 __ align(CodeEntryAlignment);
3426 StubCodeMark mark(this, stub_id);
3427 address entry = __ pc();
3428 Label SMALL_LOOP, CHECK_LAST, DIFF2, TAIL,
3429 LENGTH_DIFF, DIFF, LAST_CHECK_AND_LENGTH_DIFF;
3430 const Register result = x10, str1 = x11, cnt1 = x12, str2 = x13, cnt2 = x14,
3431 tmp1 = x28, tmp2 = x29, tmp3 = x30, tmp4 = x7, tmp5 = x31;
3432 RegSet spilled_regs = RegSet::of(tmp4, tmp5);
3433
3434 // cnt1/cnt2 contains amount of characters to compare. cnt1 can be re-used
3435 // update cnt2 counter with already loaded 8 bytes
3436 __ subi(cnt2, cnt2, wordSize / (isLL ? 1 : 2));
3437 // update pointers, because of previous read
3438 __ addi(str1, str1, wordSize);
3439 __ addi(str2, str2, wordSize);
3440 // less than 16 bytes left?
3441 __ subi(cnt2, cnt2, isLL ? 16 : 8);
3442 __ push_reg(spilled_regs, sp);
3443 __ bltz(cnt2, TAIL);
3444 __ bind(SMALL_LOOP);
3445 // compare 16 bytes of strings with same encoding
3446 __ ld(tmp5, Address(str1));
3447 __ addi(str1, str1, 8);
3448 __ xorr(tmp4, tmp1, tmp2);
3449 __ ld(cnt1, Address(str2));
3450 __ addi(str2, str2, 8);
3451 __ bnez(tmp4, DIFF);
3452 __ ld(tmp1, Address(str1));
3453 __ addi(str1, str1, 8);
3454 __ xorr(tmp4, tmp5, cnt1);
3455 __ ld(tmp2, Address(str2));
3456 __ addi(str2, str2, 8);
3457 __ bnez(tmp4, DIFF2);
3458
3459 __ subi(cnt2, cnt2, isLL ? 16 : 8);
3460 __ bgez(cnt2, SMALL_LOOP);
3461 __ bind(TAIL);
3462 __ addi(cnt2, cnt2, isLL ? 16 : 8);
3463 __ beqz(cnt2, LAST_CHECK_AND_LENGTH_DIFF);
3464 __ subi(cnt2, cnt2, isLL ? 8 : 4);
3465 __ blez(cnt2, CHECK_LAST);
3466 __ xorr(tmp4, tmp1, tmp2);
3467 __ bnez(tmp4, DIFF);
3468 __ ld(tmp1, Address(str1));
3469 __ addi(str1, str1, 8);
3470 __ ld(tmp2, Address(str2));
3471 __ addi(str2, str2, 8);
3472 __ subi(cnt2, cnt2, isLL ? 8 : 4);
3473 __ bind(CHECK_LAST);
3474 if (!isLL) {
3475 __ add(cnt2, cnt2, cnt2); // now in bytes
3476 }
3477 __ xorr(tmp4, tmp1, tmp2);
3478 __ bnez(tmp4, DIFF);
3479 __ add(str1, str1, cnt2);
3480 __ load_long_misaligned(tmp5, Address(str1), tmp3, isLL ? 1 : 2);
3481 __ add(str2, str2, cnt2);
3482 __ load_long_misaligned(cnt1, Address(str2), tmp3, isLL ? 1 : 2);
3483 __ xorr(tmp4, tmp5, cnt1);
3484 __ beqz(tmp4, LENGTH_DIFF);
3485 // Find the first different characters in the longwords and
3486 // compute their difference.
3487 __ bind(DIFF2);
3488 // count bits of trailing zero chars
3489 __ ctzc_bits(tmp3, tmp4, isLL);
3490 __ srl(tmp5, tmp5, tmp3);
3491 __ srl(cnt1, cnt1, tmp3);
3492 if (isLL) {
3493 __ zext(tmp5, tmp5, 8);
3494 __ zext(cnt1, cnt1, 8);
3495 } else {
3496 __ zext(tmp5, tmp5, 16);
3497 __ zext(cnt1, cnt1, 16);
3498 }
3499 __ sub(result, tmp5, cnt1);
3500 __ j(LENGTH_DIFF);
3501 __ bind(DIFF);
3502 // count bits of trailing zero chars
3503 __ ctzc_bits(tmp3, tmp4, isLL);
3504 __ srl(tmp1, tmp1, tmp3);
3505 __ srl(tmp2, tmp2, tmp3);
3506 if (isLL) {
3507 __ zext(tmp1, tmp1, 8);
3508 __ zext(tmp2, tmp2, 8);
3509 } else {
3510 __ zext(tmp1, tmp1, 16);
3511 __ zext(tmp2, tmp2, 16);
3512 }
3513 __ sub(result, tmp1, tmp2);
3514 __ j(LENGTH_DIFF);
3515 __ bind(LAST_CHECK_AND_LENGTH_DIFF);
3516 __ xorr(tmp4, tmp1, tmp2);
3517 __ bnez(tmp4, DIFF);
3518 __ bind(LENGTH_DIFF);
3519 __ pop_reg(spilled_regs, sp);
3520 __ ret();
3521 return entry;
3522 }
3523
3524 void generate_compare_long_strings() {
3525 StubRoutines::riscv::_compare_long_string_LL = generate_compare_long_string_same_encoding(StubId::stubgen_compare_long_string_LL_id);
3526 StubRoutines::riscv::_compare_long_string_UU = generate_compare_long_string_same_encoding(StubId::stubgen_compare_long_string_UU_id);
3527 StubRoutines::riscv::_compare_long_string_LU = generate_compare_long_string_different_encoding(StubId::stubgen_compare_long_string_LU_id);
3528 StubRoutines::riscv::_compare_long_string_UL = generate_compare_long_string_different_encoding(StubId::stubgen_compare_long_string_UL_id);
3529 }
3530
3531 // x10 result
3532 // x11 src
3533 // x12 src count
3534 // x13 pattern
3535 // x14 pattern count
3536 address generate_string_indexof_linear(StubId stub_id)
3537 {
3538 bool needle_isL;
3539 bool haystack_isL;
3540 switch (stub_id) {
3541 case StubId::stubgen_string_indexof_linear_ll_id:
3542 needle_isL = true;
3543 haystack_isL = true;
3544 break;
3545 case StubId::stubgen_string_indexof_linear_ul_id:
3546 needle_isL = true;
3547 haystack_isL = false;
3548 break;
3549 case StubId::stubgen_string_indexof_linear_uu_id:
3550 needle_isL = false;
3551 haystack_isL = false;
3552 break;
3553 default:
3554 ShouldNotReachHere();
3555 };
3556
3557 __ align(CodeEntryAlignment);
3558 StubCodeMark mark(this, stub_id);
3559 address entry = __ pc();
3560
3561 int needle_chr_size = needle_isL ? 1 : 2;
3562 int haystack_chr_size = haystack_isL ? 1 : 2;
3563 int needle_chr_shift = needle_isL ? 0 : 1;
3564 int haystack_chr_shift = haystack_isL ? 0 : 1;
3565 bool isL = needle_isL && haystack_isL;
3566 // parameters
3567 Register result = x10, haystack = x11, haystack_len = x12, needle = x13, needle_len = x14;
3568 // temporary registers
3569 Register mask1 = x20, match_mask = x21, first = x22, trailing_zeros = x23, mask2 = x24, tmp = x25;
3570 // redefinitions
3571 Register ch1 = x28, ch2 = x29;
3572 RegSet spilled_regs = RegSet::range(x20, x25) + RegSet::range(x28, x29);
3573
3574 __ push_reg(spilled_regs, sp);
3575
3576 Label L_LOOP, L_LOOP_PROCEED, L_SMALL, L_HAS_ZERO,
3577 L_HAS_ZERO_LOOP, L_CMP_LOOP, L_CMP_LOOP_NOMATCH, L_SMALL_PROCEED,
3578 L_SMALL_HAS_ZERO_LOOP, L_SMALL_CMP_LOOP_NOMATCH, L_SMALL_CMP_LOOP,
3579 L_POST_LOOP, L_CMP_LOOP_LAST_CMP, L_HAS_ZERO_LOOP_NOMATCH,
3580 L_SMALL_CMP_LOOP_LAST_CMP, L_SMALL_CMP_LOOP_LAST_CMP2,
3581 L_CMP_LOOP_LAST_CMP2, DONE, NOMATCH;
3582
3583 __ ld(ch1, Address(needle));
3584 __ ld(ch2, Address(haystack));
3585 // src.length - pattern.length
3586 __ sub(haystack_len, haystack_len, needle_len);
3587
3588 // first is needle[0]
3589 __ zext(first, ch1, needle_isL ? 8 : 16);
3590
3591 uint64_t mask0101 = UCONST64(0x0101010101010101);
3592 uint64_t mask0001 = UCONST64(0x0001000100010001);
3593 __ mv(mask1, haystack_isL ? mask0101 : mask0001);
3594 __ mul(first, first, mask1);
3595 uint64_t mask7f7f = UCONST64(0x7f7f7f7f7f7f7f7f);
3596 uint64_t mask7fff = UCONST64(0x7fff7fff7fff7fff);
3597 __ mv(mask2, haystack_isL ? mask7f7f : mask7fff);
3598 if (needle_isL != haystack_isL) {
3599 __ mv(tmp, ch1);
3600 }
3601 __ subi(haystack_len, haystack_len, wordSize / haystack_chr_size - 1);
3602 __ blez(haystack_len, L_SMALL);
3603
3604 if (needle_isL != haystack_isL) {
3605 __ inflate_lo32(ch1, tmp, match_mask, trailing_zeros);
3606 }
3607 // xorr, sub, orr, notr, andr
3608 // compare and set match_mask[i] with 0x80/0x8000 (Latin1/UTF16) if ch2[i] == first[i]
3609 // eg:
3610 // first: aa aa aa aa aa aa aa aa
3611 // ch2: aa aa li nx jd ka aa aa
3612 // match_mask: 80 80 00 00 00 00 80 80
3613 __ compute_match_mask(ch2, first, match_mask, mask1, mask2);
3614
3615 // search first char of needle, if success, goto L_HAS_ZERO;
3616 __ bnez(match_mask, L_HAS_ZERO);
3617 __ subi(haystack_len, haystack_len, wordSize / haystack_chr_size);
3618 __ addi(result, result, wordSize / haystack_chr_size);
3619 __ addi(haystack, haystack, wordSize);
3620 __ bltz(haystack_len, L_POST_LOOP);
3621
3622 __ bind(L_LOOP);
3623 __ ld(ch2, Address(haystack));
3624 __ compute_match_mask(ch2, first, match_mask, mask1, mask2);
3625 __ bnez(match_mask, L_HAS_ZERO);
3626
3627 __ bind(L_LOOP_PROCEED);
3628 __ subi(haystack_len, haystack_len, wordSize / haystack_chr_size);
3629 __ addi(haystack, haystack, wordSize);
3630 __ addi(result, result, wordSize / haystack_chr_size);
3631 __ bgez(haystack_len, L_LOOP);
3632
3633 __ bind(L_POST_LOOP);
3634 __ mv(ch2, -wordSize / haystack_chr_size);
3635 __ ble(haystack_len, ch2, NOMATCH); // no extra characters to check
3636 __ ld(ch2, Address(haystack));
3637 __ slli(haystack_len, haystack_len, LogBitsPerByte + haystack_chr_shift);
3638 __ neg(haystack_len, haystack_len);
3639 __ xorr(ch2, first, ch2);
3640 __ sub(match_mask, ch2, mask1);
3641 __ orr(ch2, ch2, mask2);
3642 __ mv(trailing_zeros, -1); // all bits set
3643 __ j(L_SMALL_PROCEED);
3644
3645 __ align(OptoLoopAlignment);
3646 __ bind(L_SMALL);
3647 __ slli(haystack_len, haystack_len, LogBitsPerByte + haystack_chr_shift);
3648 __ neg(haystack_len, haystack_len);
3649 if (needle_isL != haystack_isL) {
3650 __ inflate_lo32(ch1, tmp, match_mask, trailing_zeros);
3651 }
3652 __ xorr(ch2, first, ch2);
3653 __ sub(match_mask, ch2, mask1);
3654 __ orr(ch2, ch2, mask2);
3655 __ mv(trailing_zeros, -1); // all bits set
3656
3657 __ bind(L_SMALL_PROCEED);
3658 __ srl(trailing_zeros, trailing_zeros, haystack_len); // mask. zeroes on useless bits.
3659 __ notr(ch2, ch2);
3660 __ andr(match_mask, match_mask, ch2);
3661 __ andr(match_mask, match_mask, trailing_zeros); // clear useless bits and check
3662 __ beqz(match_mask, NOMATCH);
3663
3664 __ bind(L_SMALL_HAS_ZERO_LOOP);
3665 // count bits of trailing zero chars
3666 __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, ch2, tmp);
3667 __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15);
3668 __ mv(ch2, wordSize / haystack_chr_size);
3669 __ ble(needle_len, ch2, L_SMALL_CMP_LOOP_LAST_CMP2);
3670 __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL);
3671 __ mv(trailing_zeros, wordSize / haystack_chr_size);
3672 __ bne(ch1, ch2, L_SMALL_CMP_LOOP_NOMATCH);
3673
3674 __ bind(L_SMALL_CMP_LOOP);
3675 __ shadd(first, trailing_zeros, needle, first, needle_chr_shift);
3676 __ shadd(ch2, trailing_zeros, haystack, ch2, haystack_chr_shift);
3677 needle_isL ? __ lbu(first, Address(first)) : __ lhu(first, Address(first));
3678 haystack_isL ? __ lbu(ch2, Address(ch2)) : __ lhu(ch2, Address(ch2));
3679 __ addi(trailing_zeros, trailing_zeros, 1);
3680 __ bge(trailing_zeros, needle_len, L_SMALL_CMP_LOOP_LAST_CMP);
3681 __ beq(first, ch2, L_SMALL_CMP_LOOP);
3682
3683 __ bind(L_SMALL_CMP_LOOP_NOMATCH);
3684 __ beqz(match_mask, NOMATCH);
3685 // count bits of trailing zero chars
3686 __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, tmp, ch2);
3687 __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15);
3688 __ addi(result, result, 1);
3689 __ addi(haystack, haystack, haystack_chr_size);
3690 __ j(L_SMALL_HAS_ZERO_LOOP);
3691
3692 __ align(OptoLoopAlignment);
3693 __ bind(L_SMALL_CMP_LOOP_LAST_CMP);
3694 __ bne(first, ch2, L_SMALL_CMP_LOOP_NOMATCH);
3695 __ j(DONE);
3696
3697 __ align(OptoLoopAlignment);
3698 __ bind(L_SMALL_CMP_LOOP_LAST_CMP2);
3699 __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL);
3700 __ bne(ch1, ch2, L_SMALL_CMP_LOOP_NOMATCH);
3701 __ j(DONE);
3702
3703 __ align(OptoLoopAlignment);
3704 __ bind(L_HAS_ZERO);
3705 // count bits of trailing zero chars
3706 __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, tmp, ch2);
3707 __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15);
3708 __ slli(needle_len, needle_len, BitsPerByte * wordSize / 2);
3709 __ orr(haystack_len, haystack_len, needle_len); // restore needle_len(32bits)
3710 __ subi(result, result, 1); // array index from 0, so result -= 1
3711
3712 __ bind(L_HAS_ZERO_LOOP);
3713 __ mv(needle_len, wordSize / haystack_chr_size);
3714 __ srli(ch2, haystack_len, BitsPerByte * wordSize / 2);
3715 __ bge(needle_len, ch2, L_CMP_LOOP_LAST_CMP2);
3716 // load next 8 bytes from haystack, and increase result index
3717 __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL);
3718 __ addi(result, result, 1);
3719 __ mv(trailing_zeros, wordSize / haystack_chr_size);
3720 __ bne(ch1, ch2, L_CMP_LOOP_NOMATCH);
3721
3722 // compare one char
3723 __ bind(L_CMP_LOOP);
3724 __ shadd(needle_len, trailing_zeros, needle, needle_len, needle_chr_shift);
3725 needle_isL ? __ lbu(needle_len, Address(needle_len)) : __ lhu(needle_len, Address(needle_len));
3726 __ shadd(ch2, trailing_zeros, haystack, ch2, haystack_chr_shift);
3727 haystack_isL ? __ lbu(ch2, Address(ch2)) : __ lhu(ch2, Address(ch2));
3728 __ addi(trailing_zeros, trailing_zeros, 1); // next char index
3729 __ srli(tmp, haystack_len, BitsPerByte * wordSize / 2);
3730 __ bge(trailing_zeros, tmp, L_CMP_LOOP_LAST_CMP);
3731 __ beq(needle_len, ch2, L_CMP_LOOP);
3732
3733 __ bind(L_CMP_LOOP_NOMATCH);
3734 __ beqz(match_mask, L_HAS_ZERO_LOOP_NOMATCH);
3735 // count bits of trailing zero chars
3736 __ ctzc_bits(trailing_zeros, match_mask, haystack_isL, needle_len, ch2);
3737 __ addi(trailing_zeros, trailing_zeros, haystack_isL ? 7 : 15);
3738 __ addi(haystack, haystack, haystack_chr_size);
3739 __ j(L_HAS_ZERO_LOOP);
3740
3741 __ align(OptoLoopAlignment);
3742 __ bind(L_CMP_LOOP_LAST_CMP);
3743 __ bne(needle_len, ch2, L_CMP_LOOP_NOMATCH);
3744 __ j(DONE);
3745
3746 __ align(OptoLoopAlignment);
3747 __ bind(L_CMP_LOOP_LAST_CMP2);
3748 __ compute_index(haystack, trailing_zeros, match_mask, result, ch2, tmp, haystack_isL);
3749 __ addi(result, result, 1);
3750 __ bne(ch1, ch2, L_CMP_LOOP_NOMATCH);
3751 __ j(DONE);
3752
3753 __ align(OptoLoopAlignment);
3754 __ bind(L_HAS_ZERO_LOOP_NOMATCH);
3755 // 1) Restore "result" index. Index was wordSize/str2_chr_size * N until
3756 // L_HAS_ZERO block. Byte octet was analyzed in L_HAS_ZERO_LOOP,
3757 // so, result was increased at max by wordSize/str2_chr_size - 1, so,
3758 // respective high bit wasn't changed. L_LOOP_PROCEED will increase
3759 // result by analyzed characters value, so, we can just reset lower bits
3760 // in result here. Clear 2 lower bits for UU/UL and 3 bits for LL
3761 // 2) restore needle_len and haystack_len values from "compressed" haystack_len
3762 // 3) advance haystack value to represent next haystack octet. result & 7/3 is
3763 // index of last analyzed substring inside current octet. So, haystack in at
3764 // respective start address. We need to advance it to next octet
3765 __ andi(match_mask, result, wordSize / haystack_chr_size - 1);
3766 __ srli(needle_len, haystack_len, BitsPerByte * wordSize / 2);
3767 __ andi(result, result, haystack_isL ? -8 : -4);
3768 __ slli(tmp, match_mask, haystack_chr_shift);
3769 __ sub(haystack, haystack, tmp);
3770 __ sext(haystack_len, haystack_len, 32);
3771 __ j(L_LOOP_PROCEED);
3772
3773 __ align(OptoLoopAlignment);
3774 __ bind(NOMATCH);
3775 __ mv(result, -1);
3776
3777 __ bind(DONE);
3778 __ pop_reg(spilled_regs, sp);
3779 __ ret();
3780 return entry;
3781 }
3782
3783 void generate_string_indexof_stubs()
3784 {
3785 StubRoutines::riscv::_string_indexof_linear_ll = generate_string_indexof_linear(StubId::stubgen_string_indexof_linear_ll_id);
3786 StubRoutines::riscv::_string_indexof_linear_uu = generate_string_indexof_linear(StubId::stubgen_string_indexof_linear_uu_id);
3787 StubRoutines::riscv::_string_indexof_linear_ul = generate_string_indexof_linear(StubId::stubgen_string_indexof_linear_ul_id);
3788 }
3789
3790 #ifdef COMPILER2
3791 void generate_lookup_secondary_supers_table_stub() {
3792 StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_id;
3793 StubCodeMark mark(this, stub_id);
3794
3795 const Register
3796 r_super_klass = x10,
3797 r_array_base = x11,
3798 r_array_length = x12,
3799 r_array_index = x13,
3800 r_sub_klass = x14,
3801 result = x15,
3802 r_bitmap = x16;
3803
3804 for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
3805 StubRoutines::_lookup_secondary_supers_table_stubs[slot] = __ pc();
3806 Label L_success;
3807 __ enter();
3808 __ lookup_secondary_supers_table_const(r_sub_klass, r_super_klass, result,
3809 r_array_base, r_array_length, r_array_index,
3810 r_bitmap, slot, /*stub_is_near*/true);
3811 __ leave();
3812 __ ret();
3813 }
3814 }
3815
3816 // Slow path implementation for UseSecondarySupersTable.
3817 address generate_lookup_secondary_supers_table_slow_path_stub() {
3818 StubId stub_id = StubId::stubgen_lookup_secondary_supers_table_slow_path_id;
3819 StubCodeMark mark(this, stub_id);
3820
3821 address start = __ pc();
3822 const Register
3823 r_super_klass = x10, // argument
3824 r_array_base = x11, // argument
3825 temp1 = x12, // tmp
3826 r_array_index = x13, // argument
3827 result = x15, // argument
3828 r_bitmap = x16; // argument
3829
3830
3831 __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, result, temp1);
3832 __ ret();
3833
3834 return start;
3835 }
3836
3837 address generate_mulAdd()
3838 {
3839 __ align(CodeEntryAlignment);
3840 StubId stub_id = StubId::stubgen_mulAdd_id;
3841 StubCodeMark mark(this, stub_id);
3842
3843 address entry = __ pc();
3844
3845 const Register out = x10;
3846 const Register in = x11;
3847 const Register offset = x12;
3848 const Register len = x13;
3849 const Register k = x14;
3850 const Register tmp = x28;
3851
3852 BLOCK_COMMENT("Entry:");
3853 __ enter();
3854 __ mul_add(out, in, offset, len, k, tmp);
3855 __ leave();
3856 __ ret();
3857
3858 return entry;
3859 }
3860
3861 /**
3862 * Arguments:
3863 *
3864 * Input:
3865 * c_rarg0 - x address
3866 * c_rarg1 - x length
3867 * c_rarg2 - y address
3868 * c_rarg3 - y length
3869 * c_rarg4 - z address
3870 */
3871 address generate_multiplyToLen()
3872 {
3873 __ align(CodeEntryAlignment);
3874 StubId stub_id = StubId::stubgen_multiplyToLen_id;
3875 StubCodeMark mark(this, stub_id);
3876 address entry = __ pc();
3877
3878 const Register x = x10;
3879 const Register xlen = x11;
3880 const Register y = x12;
3881 const Register ylen = x13;
3882 const Register z = x14;
3883
3884 const Register tmp0 = x15;
3885 const Register tmp1 = x16;
3886 const Register tmp2 = x17;
3887 const Register tmp3 = x7;
3888 const Register tmp4 = x28;
3889 const Register tmp5 = x29;
3890 const Register tmp6 = x30;
3891 const Register tmp7 = x31;
3892
3893 BLOCK_COMMENT("Entry:");
3894 __ enter(); // required for proper stackwalking of RuntimeStub frame
3895 __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
3896 __ leave(); // required for proper stackwalking of RuntimeStub frame
3897 __ ret();
3898
3899 return entry;
3900 }
3901
3902 address generate_squareToLen()
3903 {
3904 __ align(CodeEntryAlignment);
3905 StubId stub_id = StubId::stubgen_squareToLen_id;
3906 StubCodeMark mark(this, stub_id);
3907 address entry = __ pc();
3908
3909 const Register x = x10;
3910 const Register xlen = x11;
3911 const Register z = x12;
3912 const Register y = x14; // == x
3913 const Register ylen = x15; // == xlen
3914
3915 const Register tmp0 = x13; // zlen, unused
3916 const Register tmp1 = x16;
3917 const Register tmp2 = x17;
3918 const Register tmp3 = x7;
3919 const Register tmp4 = x28;
3920 const Register tmp5 = x29;
3921 const Register tmp6 = x30;
3922 const Register tmp7 = x31;
3923
3924 BLOCK_COMMENT("Entry:");
3925 __ enter();
3926 __ mv(y, x);
3927 __ mv(ylen, xlen);
3928 __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
3929 __ leave();
3930 __ ret();
3931
3932 return entry;
3933 }
3934
3935 // Arguments:
3936 //
3937 // Input:
3938 // c_rarg0 - newArr address
3939 // c_rarg1 - oldArr address
3940 // c_rarg2 - newIdx
3941 // c_rarg3 - shiftCount
3942 // c_rarg4 - numIter
3943 //
3944 address generate_bigIntegerLeftShift() {
3945 __ align(CodeEntryAlignment);
3946 StubId stub_id = StubId::stubgen_bigIntegerLeftShiftWorker_id;
3947 StubCodeMark mark(this, stub_id);
3948 address entry = __ pc();
3949
3950 Label loop, exit;
3951
3952 Register newArr = c_rarg0;
3953 Register oldArr = c_rarg1;
3954 Register newIdx = c_rarg2;
3955 Register shiftCount = c_rarg3;
3956 Register numIter = c_rarg4;
3957
3958 Register shiftRevCount = c_rarg5;
3959 Register oldArrNext = t1;
3960
3961 __ beqz(numIter, exit);
3962 __ shadd(newArr, newIdx, newArr, t0, 2);
3963
3964 __ mv(shiftRevCount, 32);
3965 __ sub(shiftRevCount, shiftRevCount, shiftCount);
3966
3967 __ bind(loop);
3968 __ addi(oldArrNext, oldArr, 4);
3969 __ vsetvli(t0, numIter, Assembler::e32, Assembler::m4);
3970 __ vle32_v(v0, oldArr);
3971 __ vle32_v(v4, oldArrNext);
3972 __ vsll_vx(v0, v0, shiftCount);
3973 __ vsrl_vx(v4, v4, shiftRevCount);
3974 __ vor_vv(v0, v0, v4);
3975 __ vse32_v(v0, newArr);
3976 __ sub(numIter, numIter, t0);
3977 __ shadd(oldArr, t0, oldArr, t1, 2);
3978 __ shadd(newArr, t0, newArr, t1, 2);
3979 __ bnez(numIter, loop);
3980
3981 __ bind(exit);
3982 __ ret();
3983
3984 return entry;
3985 }
3986
3987 // Arguments:
3988 //
3989 // Input:
3990 // c_rarg0 - newArr address
3991 // c_rarg1 - oldArr address
3992 // c_rarg2 - newIdx
3993 // c_rarg3 - shiftCount
3994 // c_rarg4 - numIter
3995 //
3996 address generate_bigIntegerRightShift() {
3997 __ align(CodeEntryAlignment);
3998 StubId stub_id = StubId::stubgen_bigIntegerRightShiftWorker_id;
3999 StubCodeMark mark(this, stub_id);
4000 address entry = __ pc();
4001
4002 Label loop, exit;
4003
4004 Register newArr = c_rarg0;
4005 Register oldArr = c_rarg1;
4006 Register newIdx = c_rarg2;
4007 Register shiftCount = c_rarg3;
4008 Register numIter = c_rarg4;
4009 Register idx = numIter;
4010
4011 Register shiftRevCount = c_rarg5;
4012 Register oldArrNext = c_rarg6;
4013 Register newArrCur = t0;
4014 Register oldArrCur = t1;
4015
4016 __ beqz(idx, exit);
4017 __ shadd(newArr, newIdx, newArr, t0, 2);
4018
4019 __ mv(shiftRevCount, 32);
4020 __ sub(shiftRevCount, shiftRevCount, shiftCount);
4021
4022 __ bind(loop);
4023 __ vsetvli(t0, idx, Assembler::e32, Assembler::m4);
4024 __ sub(idx, idx, t0);
4025 __ shadd(oldArrNext, idx, oldArr, t1, 2);
4026 __ shadd(newArrCur, idx, newArr, t1, 2);
4027 __ addi(oldArrCur, oldArrNext, 4);
4028 __ vle32_v(v0, oldArrCur);
4029 __ vle32_v(v4, oldArrNext);
4030 __ vsrl_vx(v0, v0, shiftCount);
4031 __ vsll_vx(v4, v4, shiftRevCount);
4032 __ vor_vv(v0, v0, v4);
4033 __ vse32_v(v0, newArrCur);
4034 __ bnez(idx, loop);
4035
4036 __ bind(exit);
4037 __ ret();
4038
4039 return entry;
4040 }
4041 #endif
4042
4043 #ifdef COMPILER2
4044 class MontgomeryMultiplyGenerator : public MacroAssembler {
4045
4046 Register Pa_base, Pb_base, Pn_base, Pm_base, inv, Rlen, Ra, Rb, Rm, Rn,
4047 Pa, Pb, Pn, Pm, Rhi_ab, Rlo_ab, Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2, Ri, Rj;
4048
4049 RegSet _toSave;
4050 bool _squaring;
4051
4052 public:
4053 MontgomeryMultiplyGenerator (Assembler *as, bool squaring)
4054 : MacroAssembler(as->code()), _squaring(squaring) {
4055
4056 // Register allocation
4057
4058 RegSetIterator<Register> regs = RegSet::range(x10, x26).begin();
4059 Pa_base = *regs; // Argument registers
4060 if (squaring) {
4061 Pb_base = Pa_base;
4062 } else {
4063 Pb_base = *++regs;
4064 }
4065 Pn_base = *++regs;
4066 Rlen= *++regs;
4067 inv = *++regs;
4068 Pm_base = *++regs;
4069
4070 // Working registers:
4071 Ra = *++regs; // The current digit of a, b, n, and m.
4072 Rb = *++regs;
4073 Rm = *++regs;
4074 Rn = *++regs;
4075
4076 Pa = *++regs; // Pointers to the current/next digit of a, b, n, and m.
4077 Pb = *++regs;
4078 Pm = *++regs;
4079 Pn = *++regs;
4080
4081 tmp0 = *++regs; // Three registers which form a
4082 tmp1 = *++regs; // triple-precision accumuator.
4083 tmp2 = *++regs;
4084
4085 Ri = x6; // Inner and outer loop indexes.
4086 Rj = x7;
4087
4088 Rhi_ab = x28; // Product registers: low and high parts
4089 Rlo_ab = x29; // of a*b and m*n.
4090 Rhi_mn = x30;
4091 Rlo_mn = x31;
4092
4093 // x18 and up are callee-saved.
4094 _toSave = RegSet::range(x18, *regs) + Pm_base;
4095 }
4096
4097 private:
4098 void save_regs() {
4099 push_reg(_toSave, sp);
4100 }
4101
4102 void restore_regs() {
4103 pop_reg(_toSave, sp);
4104 }
4105
4106 template <typename T>
4107 void unroll_2(Register count, T block) {
4108 Label loop, end, odd;
4109 beqz(count, end);
4110 test_bit(t0, count, 0);
4111 bnez(t0, odd);
4112 align(16);
4113 bind(loop);
4114 (this->*block)();
4115 bind(odd);
4116 (this->*block)();
4117 subi(count, count, 2);
4118 bgtz(count, loop);
4119 bind(end);
4120 }
4121
4122 template <typename T>
4123 void unroll_2(Register count, T block, Register d, Register s, Register tmp) {
4124 Label loop, end, odd;
4125 beqz(count, end);
4126 test_bit(tmp, count, 0);
4127 bnez(tmp, odd);
4128 align(16);
4129 bind(loop);
4130 (this->*block)(d, s, tmp);
4131 bind(odd);
4132 (this->*block)(d, s, tmp);
4133 subi(count, count, 2);
4134 bgtz(count, loop);
4135 bind(end);
4136 }
4137
4138 void pre1(RegisterOrConstant i) {
4139 block_comment("pre1");
4140 // Pa = Pa_base;
4141 // Pb = Pb_base + i;
4142 // Pm = Pm_base;
4143 // Pn = Pn_base + i;
4144 // Ra = *Pa;
4145 // Rb = *Pb;
4146 // Rm = *Pm;
4147 // Rn = *Pn;
4148 if (i.is_register()) {
4149 slli(t0, i.as_register(), LogBytesPerWord);
4150 } else {
4151 mv(t0, i.as_constant());
4152 slli(t0, t0, LogBytesPerWord);
4153 }
4154
4155 mv(Pa, Pa_base);
4156 add(Pb, Pb_base, t0);
4157 mv(Pm, Pm_base);
4158 add(Pn, Pn_base, t0);
4159
4160 ld(Ra, Address(Pa));
4161 ld(Rb, Address(Pb));
4162 ld(Rm, Address(Pm));
4163 ld(Rn, Address(Pn));
4164
4165 // Zero the m*n result.
4166 mv(Rhi_mn, zr);
4167 mv(Rlo_mn, zr);
4168 }
4169
4170 // The core multiply-accumulate step of a Montgomery
4171 // multiplication. The idea is to schedule operations as a
4172 // pipeline so that instructions with long latencies (loads and
4173 // multiplies) have time to complete before their results are
4174 // used. This most benefits in-order implementations of the
4175 // architecture but out-of-order ones also benefit.
4176 void step() {
4177 block_comment("step");
4178 // MACC(Ra, Rb, tmp0, tmp1, tmp2);
4179 // Ra = *++Pa;
4180 // Rb = *--Pb;
4181 mulhu(Rhi_ab, Ra, Rb);
4182 mul(Rlo_ab, Ra, Rb);
4183 addi(Pa, Pa, wordSize);
4184 ld(Ra, Address(Pa));
4185 subi(Pb, Pb, wordSize);
4186 ld(Rb, Address(Pb));
4187 acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n from the
4188 // previous iteration.
4189 // MACC(Rm, Rn, tmp0, tmp1, tmp2);
4190 // Rm = *++Pm;
4191 // Rn = *--Pn;
4192 mulhu(Rhi_mn, Rm, Rn);
4193 mul(Rlo_mn, Rm, Rn);
4194 addi(Pm, Pm, wordSize);
4195 ld(Rm, Address(Pm));
4196 subi(Pn, Pn, wordSize);
4197 ld(Rn, Address(Pn));
4198 acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2);
4199 }
4200
4201 void post1() {
4202 block_comment("post1");
4203
4204 // MACC(Ra, Rb, tmp0, tmp1, tmp2);
4205 // Ra = *++Pa;
4206 // Rb = *--Pb;
4207 mulhu(Rhi_ab, Ra, Rb);
4208 mul(Rlo_ab, Ra, Rb);
4209 acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n
4210 acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2);
4211
4212 // *Pm = Rm = tmp0 * inv;
4213 mul(Rm, tmp0, inv);
4214 sd(Rm, Address(Pm));
4215
4216 // MACC(Rm, Rn, tmp0, tmp1, tmp2);
4217 // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0;
4218 mulhu(Rhi_mn, Rm, Rn);
4219
4220 #ifndef PRODUCT
4221 // assert(m[i] * n[0] + tmp0 == 0, "broken Montgomery multiply");
4222 {
4223 mul(Rlo_mn, Rm, Rn);
4224 add(Rlo_mn, tmp0, Rlo_mn);
4225 Label ok;
4226 beqz(Rlo_mn, ok);
4227 stop("broken Montgomery multiply");
4228 bind(ok);
4229 }
4230 #endif
4231 // We have very carefully set things up so that
4232 // m[i]*n[0] + tmp0 == 0 (mod b), so we don't have to calculate
4233 // the lower half of Rm * Rn because we know the result already:
4234 // it must be -tmp0. tmp0 + (-tmp0) must generate a carry iff
4235 // tmp0 != 0. So, rather than do a mul and an cad we just set
4236 // the carry flag iff tmp0 is nonzero.
4237 //
4238 // mul(Rlo_mn, Rm, Rn);
4239 // cad(zr, tmp0, Rlo_mn);
4240 subi(t0, tmp0, 1);
4241 sltu(t0, t0, tmp0); // Set carry iff tmp0 is nonzero
4242 cadc(tmp0, tmp1, Rhi_mn, t0);
4243 adc(tmp1, tmp2, zr, t0);
4244 mv(tmp2, zr);
4245 }
4246
4247 void pre2(Register i, Register len) {
4248 block_comment("pre2");
4249 // Pa = Pa_base + i-len;
4250 // Pb = Pb_base + len;
4251 // Pm = Pm_base + i-len;
4252 // Pn = Pn_base + len;
4253
4254 sub(Rj, i, len);
4255 // Rj == i-len
4256
4257 // Ra as temp register
4258 slli(Ra, Rj, LogBytesPerWord);
4259 add(Pa, Pa_base, Ra);
4260 add(Pm, Pm_base, Ra);
4261 slli(Ra, len, LogBytesPerWord);
4262 add(Pb, Pb_base, Ra);
4263 add(Pn, Pn_base, Ra);
4264
4265 // Ra = *++Pa;
4266 // Rb = *--Pb;
4267 // Rm = *++Pm;
4268 // Rn = *--Pn;
4269 addi(Pa, Pa, wordSize);
4270 ld(Ra, Address(Pa));
4271 subi(Pb, Pb, wordSize);
4272 ld(Rb, Address(Pb));
4273 addi(Pm, Pm, wordSize);
4274 ld(Rm, Address(Pm));
4275 subi(Pn, Pn, wordSize);
4276 ld(Rn, Address(Pn));
4277
4278 mv(Rhi_mn, zr);
4279 mv(Rlo_mn, zr);
4280 }
4281
4282 void post2(Register i, Register len) {
4283 block_comment("post2");
4284 sub(Rj, i, len);
4285
4286 cad(tmp0, tmp0, Rlo_mn, t0); // The pending m*n, low part
4287
4288 // As soon as we know the least significant digit of our result,
4289 // store it.
4290 // Pm_base[i-len] = tmp0;
4291 // Rj as temp register
4292 slli(Rj, Rj, LogBytesPerWord);
4293 add(Rj, Pm_base, Rj);
4294 sd(tmp0, Address(Rj));
4295
4296 // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0;
4297 cadc(tmp0, tmp1, Rhi_mn, t0); // The pending m*n, high part
4298 adc(tmp1, tmp2, zr, t0);
4299 mv(tmp2, zr);
4300 }
4301
4302 // A carry in tmp0 after Montgomery multiplication means that we
4303 // should subtract multiples of n from our result in m. We'll
4304 // keep doing that until there is no carry.
4305 void normalize(Register len) {
4306 block_comment("normalize");
4307 // while (tmp0)
4308 // tmp0 = sub(Pm_base, Pn_base, tmp0, len);
4309 Label loop, post, again;
4310 Register cnt = tmp1, i = tmp2; // Re-use registers; we're done with them now
4311 beqz(tmp0, post); {
4312 bind(again); {
4313 mv(i, zr);
4314 mv(cnt, len);
4315 slli(Rn, i, LogBytesPerWord);
4316 add(Rm, Pm_base, Rn);
4317 ld(Rm, Address(Rm));
4318 add(Rn, Pn_base, Rn);
4319 ld(Rn, Address(Rn));
4320 mv(t0, 1); // set carry flag, i.e. no borrow
4321 align(16);
4322 bind(loop); {
4323 notr(Rn, Rn);
4324 add(Rm, Rm, t0);
4325 add(Rm, Rm, Rn);
4326 sltu(t0, Rm, Rn);
4327 slli(Rn, i, LogBytesPerWord); // Rn as temp register
4328 add(Rn, Pm_base, Rn);
4329 sd(Rm, Address(Rn));
4330 addi(i, i, 1);
4331 slli(Rn, i, LogBytesPerWord);
4332 add(Rm, Pm_base, Rn);
4333 ld(Rm, Address(Rm));
4334 add(Rn, Pn_base, Rn);
4335 ld(Rn, Address(Rn));
4336 subi(cnt, cnt, 1);
4337 } bnez(cnt, loop);
4338 subi(tmp0, tmp0, 1);
4339 add(tmp0, tmp0, t0);
4340 } bnez(tmp0, again);
4341 } bind(post);
4342 }
4343
4344 // Move memory at s to d, reversing words.
4345 // Increments d to end of copied memory
4346 // Destroys tmp1, tmp2
4347 // Preserves len
4348 // Leaves s pointing to the address which was in d at start
4349 void reverse(Register d, Register s, Register len, Register tmp1, Register tmp2) {
4350 assert(tmp1->encoding() < x28->encoding(), "register corruption");
4351 assert(tmp2->encoding() < x28->encoding(), "register corruption");
4352
4353 shadd(s, len, s, tmp1, LogBytesPerWord);
4354 mv(tmp1, len);
4355 unroll_2(tmp1, &MontgomeryMultiplyGenerator::reverse1, d, s, tmp2);
4356 slli(tmp1, len, LogBytesPerWord);
4357 sub(s, d, tmp1);
4358 }
4359 // [63...0] -> [31...0][63...32]
4360 void reverse1(Register d, Register s, Register tmp) {
4361 subi(s, s, wordSize);
4362 ld(tmp, Address(s));
4363 ror(tmp, tmp, 32, t0);
4364 sd(tmp, Address(d));
4365 addi(d, d, wordSize);
4366 }
4367
4368 void step_squaring() {
4369 // An extra ACC
4370 step();
4371 acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2);
4372 }
4373
4374 void last_squaring(Register i) {
4375 Label dont;
4376 // if ((i & 1) == 0) {
4377 test_bit(t0, i, 0);
4378 bnez(t0, dont); {
4379 // MACC(Ra, Rb, tmp0, tmp1, tmp2);
4380 // Ra = *++Pa;
4381 // Rb = *--Pb;
4382 mulhu(Rhi_ab, Ra, Rb);
4383 mul(Rlo_ab, Ra, Rb);
4384 acc(Rhi_ab, Rlo_ab, tmp0, tmp1, tmp2);
4385 } bind(dont);
4386 }
4387
4388 void extra_step_squaring() {
4389 acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n
4390
4391 // MACC(Rm, Rn, tmp0, tmp1, tmp2);
4392 // Rm = *++Pm;
4393 // Rn = *--Pn;
4394 mulhu(Rhi_mn, Rm, Rn);
4395 mul(Rlo_mn, Rm, Rn);
4396 addi(Pm, Pm, wordSize);
4397 ld(Rm, Address(Pm));
4398 subi(Pn, Pn, wordSize);
4399 ld(Rn, Address(Pn));
4400 }
4401
4402 void post1_squaring() {
4403 acc(Rhi_mn, Rlo_mn, tmp0, tmp1, tmp2); // The pending m*n
4404
4405 // *Pm = Rm = tmp0 * inv;
4406 mul(Rm, tmp0, inv);
4407 sd(Rm, Address(Pm));
4408
4409 // MACC(Rm, Rn, tmp0, tmp1, tmp2);
4410 // tmp0 = tmp1; tmp1 = tmp2; tmp2 = 0;
4411 mulhu(Rhi_mn, Rm, Rn);
4412
4413 #ifndef PRODUCT
4414 // assert(m[i] * n[0] + tmp0 == 0, "broken Montgomery multiply");
4415 {
4416 mul(Rlo_mn, Rm, Rn);
4417 add(Rlo_mn, tmp0, Rlo_mn);
4418 Label ok;
4419 beqz(Rlo_mn, ok); {
4420 stop("broken Montgomery multiply");
4421 } bind(ok);
4422 }
4423 #endif
4424 // We have very carefully set things up so that
4425 // m[i]*n[0] + tmp0 == 0 (mod b), so we don't have to calculate
4426 // the lower half of Rm * Rn because we know the result already:
4427 // it must be -tmp0. tmp0 + (-tmp0) must generate a carry iff
4428 // tmp0 != 0. So, rather than do a mul and a cad we just set
4429 // the carry flag iff tmp0 is nonzero.
4430 //
4431 // mul(Rlo_mn, Rm, Rn);
4432 // cad(zr, tmp, Rlo_mn);
4433 subi(t0, tmp0, 1);
4434 sltu(t0, t0, tmp0); // Set carry iff tmp0 is nonzero
4435 cadc(tmp0, tmp1, Rhi_mn, t0);
4436 adc(tmp1, tmp2, zr, t0);
4437 mv(tmp2, zr);
4438 }
4439
4440 // use t0 as carry
4441 void acc(Register Rhi, Register Rlo,
4442 Register tmp0, Register tmp1, Register tmp2) {
4443 cad(tmp0, tmp0, Rlo, t0);
4444 cadc(tmp1, tmp1, Rhi, t0);
4445 adc(tmp2, tmp2, zr, t0);
4446 }
4447
4448 public:
4449 /**
4450 * Fast Montgomery multiplication. The derivation of the
4451 * algorithm is in A Cryptographic Library for the Motorola
4452 * DSP56000, Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237.
4453 *
4454 * Arguments:
4455 *
4456 * Inputs for multiplication:
4457 * c_rarg0 - int array elements a
4458 * c_rarg1 - int array elements b
4459 * c_rarg2 - int array elements n (the modulus)
4460 * c_rarg3 - int length
4461 * c_rarg4 - int inv
4462 * c_rarg5 - int array elements m (the result)
4463 *
4464 * Inputs for squaring:
4465 * c_rarg0 - int array elements a
4466 * c_rarg1 - int array elements n (the modulus)
4467 * c_rarg2 - int length
4468 * c_rarg3 - int inv
4469 * c_rarg4 - int array elements m (the result)
4470 *
4471 */
4472 address generate_multiply() {
4473 Label argh, nothing;
4474 bind(argh);
4475 stop("MontgomeryMultiply total_allocation must be <= 8192");
4476
4477 align(CodeEntryAlignment);
4478 address entry = pc();
4479
4480 beqz(Rlen, nothing);
4481
4482 enter();
4483
4484 // Make room.
4485 mv(Ra, 512);
4486 bgt(Rlen, Ra, argh);
4487 slli(Ra, Rlen, exact_log2(4 * sizeof(jint)));
4488 sub(Ra, sp, Ra);
4489 andi(sp, Ra, -2 * wordSize);
4490
4491 srliw(Rlen, Rlen, 1); // length in longwords = len/2
4492
4493 {
4494 // Copy input args, reversing as we go. We use Ra as a
4495 // temporary variable.
4496 reverse(Ra, Pa_base, Rlen, Ri, Rj);
4497 if (!_squaring)
4498 reverse(Ra, Pb_base, Rlen, Ri, Rj);
4499 reverse(Ra, Pn_base, Rlen, Ri, Rj);
4500 }
4501
4502 // Push all call-saved registers and also Pm_base which we'll need
4503 // at the end.
4504 save_regs();
4505
4506 #ifndef PRODUCT
4507 // assert(inv * n[0] == -1UL, "broken inverse in Montgomery multiply");
4508 {
4509 ld(Rn, Address(Pn_base));
4510 mul(Rlo_mn, Rn, inv);
4511 mv(t0, -1);
4512 Label ok;
4513 beq(Rlo_mn, t0, ok);
4514 stop("broken inverse in Montgomery multiply");
4515 bind(ok);
4516 }
4517 #endif
4518
4519 mv(Pm_base, Ra);
4520
4521 mv(tmp0, zr);
4522 mv(tmp1, zr);
4523 mv(tmp2, zr);
4524
4525 block_comment("for (int i = 0; i < len; i++) {");
4526 mv(Ri, zr); {
4527 Label loop, end;
4528 bge(Ri, Rlen, end);
4529
4530 bind(loop);
4531 pre1(Ri);
4532
4533 block_comment(" for (j = i; j; j--) {"); {
4534 mv(Rj, Ri);
4535 unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
4536 } block_comment(" } // j");
4537
4538 post1();
4539 addiw(Ri, Ri, 1);
4540 blt(Ri, Rlen, loop);
4541 bind(end);
4542 block_comment("} // i");
4543 }
4544
4545 block_comment("for (int i = len; i < 2*len; i++) {");
4546 mv(Ri, Rlen); {
4547 Label loop, end;
4548 slli(t0, Rlen, 1);
4549 bge(Ri, t0, end);
4550
4551 bind(loop);
4552 pre2(Ri, Rlen);
4553
4554 block_comment(" for (j = len*2-i-1; j; j--) {"); {
4555 slliw(Rj, Rlen, 1);
4556 subw(Rj, Rj, Ri);
4557 subiw(Rj, Rj, 1);
4558 unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
4559 } block_comment(" } // j");
4560
4561 post2(Ri, Rlen);
4562 addiw(Ri, Ri, 1);
4563 slli(t0, Rlen, 1);
4564 blt(Ri, t0, loop);
4565 bind(end);
4566 }
4567 block_comment("} // i");
4568
4569 normalize(Rlen);
4570
4571 mv(Ra, Pm_base); // Save Pm_base in Ra
4572 restore_regs(); // Restore caller's Pm_base
4573
4574 // Copy our result into caller's Pm_base
4575 reverse(Pm_base, Ra, Rlen, Ri, Rj);
4576
4577 leave();
4578 bind(nothing);
4579 ret();
4580
4581 return entry;
4582 }
4583
4584 /**
4585 *
4586 * Arguments:
4587 *
4588 * Inputs:
4589 * c_rarg0 - int array elements a
4590 * c_rarg1 - int array elements n (the modulus)
4591 * c_rarg2 - int length
4592 * c_rarg3 - int inv
4593 * c_rarg4 - int array elements m (the result)
4594 *
4595 */
4596 address generate_square() {
4597 Label argh;
4598 bind(argh);
4599 stop("MontgomeryMultiply total_allocation must be <= 8192");
4600
4601 align(CodeEntryAlignment);
4602 address entry = pc();
4603
4604 enter();
4605
4606 // Make room.
4607 mv(Ra, 512);
4608 bgt(Rlen, Ra, argh);
4609 slli(Ra, Rlen, exact_log2(4 * sizeof(jint)));
4610 sub(Ra, sp, Ra);
4611 andi(sp, Ra, -2 * wordSize);
4612
4613 srliw(Rlen, Rlen, 1); // length in longwords = len/2
4614
4615 {
4616 // Copy input args, reversing as we go. We use Ra as a
4617 // temporary variable.
4618 reverse(Ra, Pa_base, Rlen, Ri, Rj);
4619 reverse(Ra, Pn_base, Rlen, Ri, Rj);
4620 }
4621
4622 // Push all call-saved registers and also Pm_base which we'll need
4623 // at the end.
4624 save_regs();
4625
4626 mv(Pm_base, Ra);
4627
4628 mv(tmp0, zr);
4629 mv(tmp1, zr);
4630 mv(tmp2, zr);
4631
4632 block_comment("for (int i = 0; i < len; i++) {");
4633 mv(Ri, zr); {
4634 Label loop, end;
4635 bind(loop);
4636 bge(Ri, Rlen, end);
4637
4638 pre1(Ri);
4639
4640 block_comment("for (j = (i+1)/2; j; j--) {"); {
4641 addi(Rj, Ri, 1);
4642 srliw(Rj, Rj, 1);
4643 unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
4644 } block_comment(" } // j");
4645
4646 last_squaring(Ri);
4647
4648 block_comment(" for (j = i/2; j; j--) {"); {
4649 srliw(Rj, Ri, 1);
4650 unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
4651 } block_comment(" } // j");
4652
4653 post1_squaring();
4654 addi(Ri, Ri, 1);
4655 blt(Ri, Rlen, loop);
4656
4657 bind(end);
4658 block_comment("} // i");
4659 }
4660
4661 block_comment("for (int i = len; i < 2*len; i++) {");
4662 mv(Ri, Rlen); {
4663 Label loop, end;
4664 bind(loop);
4665 slli(t0, Rlen, 1);
4666 bge(Ri, t0, end);
4667
4668 pre2(Ri, Rlen);
4669
4670 block_comment(" for (j = (2*len-i-1)/2; j; j--) {"); {
4671 slli(Rj, Rlen, 1);
4672 sub(Rj, Rj, Ri);
4673 subi(Rj, Rj, 1);
4674 srliw(Rj, Rj, 1);
4675 unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
4676 } block_comment(" } // j");
4677
4678 last_squaring(Ri);
4679
4680 block_comment(" for (j = (2*len-i)/2; j; j--) {"); {
4681 slli(Rj, Rlen, 1);
4682 sub(Rj, Rj, Ri);
4683 srliw(Rj, Rj, 1);
4684 unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
4685 } block_comment(" } // j");
4686
4687 post2(Ri, Rlen);
4688 addi(Ri, Ri, 1);
4689 slli(t0, Rlen, 1);
4690 blt(Ri, t0, loop);
4691
4692 bind(end);
4693 block_comment("} // i");
4694 }
4695
4696 normalize(Rlen);
4697
4698 mv(Ra, Pm_base); // Save Pm_base in Ra
4699 restore_regs(); // Restore caller's Pm_base
4700
4701 // Copy our result into caller's Pm_base
4702 reverse(Pm_base, Ra, Rlen, Ri, Rj);
4703
4704 leave();
4705 ret();
4706
4707 return entry;
4708 }
4709 };
4710
4711 #endif // COMPILER2
4712
4713 address generate_cont_thaw(Continuation::thaw_kind kind) {
4714 bool return_barrier = Continuation::is_thaw_return_barrier(kind);
4715 bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
4716
4717 address start = __ pc();
4718
4719 if (return_barrier) {
4720 __ ld(sp, Address(xthread, JavaThread::cont_entry_offset()));
4721 }
4722
4723 #ifndef PRODUCT
4724 {
4725 Label OK;
4726 __ ld(t0, Address(xthread, JavaThread::cont_entry_offset()));
4727 __ beq(sp, t0, OK);
4728 __ stop("incorrect sp");
4729 __ bind(OK);
4730 }
4731 #endif
4732
4733 if (return_barrier) {
4734 // preserve possible return value from a method returning to the return barrier
4735 __ subi(sp, sp, 2 * wordSize);
4736 __ fsd(f10, Address(sp, 0 * wordSize));
4737 __ sd(x10, Address(sp, 1 * wordSize));
4738 }
4739
4740 __ mv(c_rarg1, (return_barrier ? 1 : 0));
4741 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), xthread, c_rarg1);
4742 __ mv(t1, x10); // x10 contains the size of the frames to thaw, 0 if overflow or no more frames
4743
4744 if (return_barrier) {
4745 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
4746 __ ld(x10, Address(sp, 1 * wordSize));
4747 __ fld(f10, Address(sp, 0 * wordSize));
4748 __ addi(sp, sp, 2 * wordSize);
4749 }
4750
4751 #ifndef PRODUCT
4752 {
4753 Label OK;
4754 __ ld(t0, Address(xthread, JavaThread::cont_entry_offset()));
4755 __ beq(sp, t0, OK);
4756 __ stop("incorrect sp");
4757 __ bind(OK);
4758 }
4759 #endif
4760
4761 Label thaw_success;
4762 // t1 contains the size of the frames to thaw, 0 if overflow or no more frames
4763 __ bnez(t1, thaw_success);
4764 __ j(RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
4765 __ bind(thaw_success);
4766
4767 // make room for the thawed frames
4768 __ sub(t0, sp, t1);
4769 __ andi(sp, t0, -16); // align
4770
4771 if (return_barrier) {
4772 // save original return value -- again
4773 __ subi(sp, sp, 2 * wordSize);
4774 __ fsd(f10, Address(sp, 0 * wordSize));
4775 __ sd(x10, Address(sp, 1 * wordSize));
4776 }
4777
4778 // If we want, we can templatize thaw by kind, and have three different entries
4779 __ mv(c_rarg1, kind);
4780
4781 __ call_VM_leaf(Continuation::thaw_entry(), xthread, c_rarg1);
4782 __ mv(t1, x10); // x10 is the sp of the yielding frame
4783
4784 if (return_barrier) {
4785 // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
4786 __ ld(x10, Address(sp, 1 * wordSize));
4787 __ fld(f10, Address(sp, 0 * wordSize));
4788 __ addi(sp, sp, 2 * wordSize);
4789 } else {
4790 __ mv(x10, zr); // return 0 (success) from doYield
4791 }
4792
4793 // we're now on the yield frame (which is in an address above us b/c sp has been pushed down)
4794 __ mv(fp, t1);
4795 __ subi(sp, t1, 2 * wordSize); // now pointing to fp spill
4796
4797 if (return_barrier_exception) {
4798 __ ld(c_rarg1, Address(fp, -1 * wordSize)); // return address
4799 __ verify_oop(x10);
4800 __ mv(x9, x10); // save return value contaning the exception oop in callee-saved x9
4801
4802 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), xthread, c_rarg1);
4803
4804 // see OptoRuntime::generate_exception_blob: x10 -- exception oop, x13 -- exception pc
4805
4806 __ mv(x11, x10); // the exception handler
4807 __ mv(x10, x9); // restore return value contaning the exception oop
4808 __ verify_oop(x10);
4809
4810 __ leave();
4811 __ mv(x13, ra);
4812 __ jr(x11); // the exception handler
4813 } else {
4814 // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame
4815 __ leave();
4816 __ ret();
4817 }
4818
4819 return start;
4820 }
4821
4822 address generate_cont_thaw() {
4823 if (!Continuations::enabled()) return nullptr;
4824
4825 StubId stub_id = StubId::stubgen_cont_thaw_id;
4826 StubCodeMark mark(this, stub_id);
4827 address start = __ pc();
4828 generate_cont_thaw(Continuation::thaw_top);
4829 return start;
4830 }
4831
4832 address generate_cont_returnBarrier() {
4833 if (!Continuations::enabled()) return nullptr;
4834
4835 // TODO: will probably need multiple return barriers depending on return type
4836 StubId stub_id = StubId::stubgen_cont_returnBarrier_id;
4837 StubCodeMark mark(this, stub_id);
4838 address start = __ pc();
4839
4840 generate_cont_thaw(Continuation::thaw_return_barrier);
4841
4842 return start;
4843 }
4844
4845 address generate_cont_returnBarrier_exception() {
4846 if (!Continuations::enabled()) return nullptr;
4847
4848 StubId stub_id = StubId::stubgen_cont_returnBarrierExc_id;
4849 StubCodeMark mark(this, stub_id);
4850 address start = __ pc();
4851
4852 generate_cont_thaw(Continuation::thaw_return_barrier_exception);
4853
4854 return start;
4855 }
4856
4857 address generate_cont_preempt_stub() {
4858 if (!Continuations::enabled()) return nullptr;
4859 StubId stub_id = StubId::stubgen_cont_preempt_id;
4860 StubCodeMark mark(this, stub_id);
4861 address start = __ pc();
4862
4863 __ reset_last_Java_frame(true);
4864
4865 // Set sp to enterSpecial frame, i.e. remove all frames copied into the heap.
4866 __ ld(sp, Address(xthread, JavaThread::cont_entry_offset()));
4867
4868 Label preemption_cancelled;
4869 __ lbu(t0, Address(xthread, JavaThread::preemption_cancelled_offset()));
4870 __ bnez(t0, preemption_cancelled);
4871
4872 // Remove enterSpecial frame from the stack and return to Continuation.run() to unmount.
4873 SharedRuntime::continuation_enter_cleanup(_masm);
4874 __ leave();
4875 __ ret();
4876
4877 // We acquired the monitor after freezing the frames so call thaw to continue execution.
4878 __ bind(preemption_cancelled);
4879 __ sb(zr, Address(xthread, JavaThread::preemption_cancelled_offset()));
4880 __ la(fp, Address(sp, checked_cast<int32_t>(ContinuationEntry::size() + 2 * wordSize)));
4881 __ la(t1, ExternalAddress(ContinuationEntry::thaw_call_pc_address()));
4882 __ ld(t1, Address(t1));
4883 __ jr(t1);
4884
4885 return start;
4886 }
4887
4888 #ifdef COMPILER2
4889
4890 #undef __
4891 #define __ this->
4892
4893 class Sha2Generator : public MacroAssembler {
4894 StubCodeGenerator* _cgen;
4895 public:
4896 Sha2Generator(MacroAssembler* masm, StubCodeGenerator* cgen) : MacroAssembler(masm->code()), _cgen(cgen) {}
4897 address generate_sha256_implCompress(StubId stub_id) {
4898 return generate_sha2_implCompress(Assembler::e32, stub_id);
4899 }
4900 address generate_sha512_implCompress(StubId stub_id) {
4901 return generate_sha2_implCompress(Assembler::e64, stub_id);
4902 }
4903 private:
4904
4905 void vleXX_v(Assembler::SEW vset_sew, VectorRegister vr, Register sr) {
4906 if (vset_sew == Assembler::e32) __ vle32_v(vr, sr);
4907 else __ vle64_v(vr, sr);
4908 }
4909
4910 void vseXX_v(Assembler::SEW vset_sew, VectorRegister vr, Register sr) {
4911 if (vset_sew == Assembler::e32) __ vse32_v(vr, sr);
4912 else __ vse64_v(vr, sr);
4913 }
4914
4915 // Overview of the logic in each "quad round".
4916 //
4917 // The code below repeats 16/20 times the logic implementing four rounds
4918 // of the SHA-256/512 core loop as documented by NIST. 16/20 "quad rounds"
4919 // to implementing the 64/80 single rounds.
4920 //
4921 // // Load four word (u32/64) constants (K[t+3], K[t+2], K[t+1], K[t+0])
4922 // // Output:
4923 // // vTmp1 = {K[t+3], K[t+2], K[t+1], K[t+0]}
4924 // vl1reXX.v vTmp1, ofs
4925 //
4926 // // Increment word constant address by stride (16/32 bytes, 4*4B/8B, 128b/256b)
4927 // addi ofs, ofs, 16/32
4928 //
4929 // // Add constants to message schedule words:
4930 // // Input
4931 // // vTmp1 = {K[t+3], K[t+2], K[t+1], K[t+0]}
4932 // // vW0 = {W[t+3], W[t+2], W[t+1], W[t+0]}; // Vt0 = W[3:0];
4933 // // Output
4934 // // vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]}
4935 // vadd.vv vTmp0, vTmp1, vW0
4936 //
4937 // // 2 rounds of working variables updates.
4938 // // vState1[t+4] <- vState1[t], vState0[t], vTmp0[t]
4939 // // Input:
4940 // // vState1 = {c[t],d[t],g[t],h[t]} " = vState1[t] "
4941 // // vState0 = {a[t],b[t],e[t],f[t]}
4942 // // vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]}
4943 // // Output:
4944 // // vState1 = {f[t+2],e[t+2],b[t+2],a[t+2]} " = vState0[t+2] "
4945 // // = {h[t+4],g[t+4],d[t+4],c[t+4]} " = vState1[t+4] "
4946 // vsha2cl.vv vState1, vState0, vTmp0
4947 //
4948 // // 2 rounds of working variables updates.
4949 // // vState0[t+4] <- vState0[t], vState0[t+2], vTmp0[t]
4950 // // Input
4951 // // vState0 = {a[t],b[t],e[t],f[t]} " = vState0[t] "
4952 // // = {h[t+2],g[t+2],d[t+2],c[t+2]} " = vState1[t+2] "
4953 // // vState1 = {f[t+2],e[t+2],b[t+2],a[t+2]} " = vState0[t+2] "
4954 // // vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]}
4955 // // Output:
4956 // // vState0 = {f[t+4],e[t+4],b[t+4],a[t+4]} " = vState0[t+4] "
4957 // vsha2ch.vv vState0, vState1, vTmp0
4958 //
4959 // // Combine 2QW into 1QW
4960 // //
4961 // // To generate the next 4 words, "new_vW0"/"vTmp0" from vW0-vW3, vsha2ms needs
4962 // // vW0[0..3], vW1[0], vW2[1..3], vW3[0, 2..3]
4963 // // and it can only take 3 vectors as inputs. Hence we need to combine
4964 // // vW1[0] and vW2[1..3] in a single vector.
4965 // //
4966 // // vmerge Vt4, Vt1, Vt2, V0
4967 // // Input
4968 // // V0 = mask // first word from vW2, 1..3 words from vW1
4969 // // vW2 = {Wt-8, Wt-7, Wt-6, Wt-5}
4970 // // vW1 = {Wt-12, Wt-11, Wt-10, Wt-9}
4971 // // Output
4972 // // Vt4 = {Wt-12, Wt-7, Wt-6, Wt-5}
4973 // vmerge.vvm vTmp0, vW2, vW1, v0
4974 //
4975 // // Generate next Four Message Schedule Words (hence allowing for 4 more rounds)
4976 // // Input
4977 // // vW0 = {W[t+ 3], W[t+ 2], W[t+ 1], W[t+ 0]} W[ 3: 0]
4978 // // vW3 = {W[t+15], W[t+14], W[t+13], W[t+12]} W[15:12]
4979 // // vTmp0 = {W[t+11], W[t+10], W[t+ 9], W[t+ 4]} W[11: 9,4]
4980 // // Output (next four message schedule words)
4981 // // vW0 = {W[t+19], W[t+18], W[t+17], W[t+16]} W[19:16]
4982 // vsha2ms.vv vW0, vTmp0, vW3
4983 //
4984 // BEFORE
4985 // vW0 - vW3 hold the message schedule words (initially the block words)
4986 // vW0 = W[ 3: 0] "oldest"
4987 // vW1 = W[ 7: 4]
4988 // vW2 = W[11: 8]
4989 // vW3 = W[15:12] "newest"
4990 //
4991 // vt6 - vt7 hold the working state variables
4992 // vState0 = {a[t],b[t],e[t],f[t]} // initially {H5,H4,H1,H0}
4993 // vState1 = {c[t],d[t],g[t],h[t]} // initially {H7,H6,H3,H2}
4994 //
4995 // AFTER
4996 // vW0 - vW3 hold the message schedule words (initially the block words)
4997 // vW1 = W[ 7: 4] "oldest"
4998 // vW2 = W[11: 8]
4999 // vW3 = W[15:12]
5000 // vW0 = W[19:16] "newest"
5001 //
5002 // vState0 and vState1 hold the working state variables
5003 // vState0 = {a[t+4],b[t+4],e[t+4],f[t+4]}
5004 // vState1 = {c[t+4],d[t+4],g[t+4],h[t+4]}
5005 //
5006 // The group of vectors vW0,vW1,vW2,vW3 is "rotated" by one in each quad-round,
5007 // hence the uses of those vectors rotate in each round, and we get back to the
5008 // initial configuration every 4 quad-rounds. We could avoid those changes at
5009 // the cost of moving those vectors at the end of each quad-rounds.
5010 void sha2_quad_round(Assembler::SEW vset_sew, VectorRegister rot1, VectorRegister rot2, VectorRegister rot3, VectorRegister rot4,
5011 Register scalarconst, VectorRegister vtemp, VectorRegister vtemp2, VectorRegister v_abef, VectorRegister v_cdgh,
5012 bool gen_words = true, bool step_const = true) {
5013 __ vleXX_v(vset_sew, vtemp, scalarconst);
5014 if (step_const) {
5015 __ addi(scalarconst, scalarconst, vset_sew == Assembler::e32 ? 16 : 32);
5016 }
5017 __ vadd_vv(vtemp2, vtemp, rot1);
5018 __ vsha2cl_vv(v_cdgh, v_abef, vtemp2);
5019 __ vsha2ch_vv(v_abef, v_cdgh, vtemp2);
5020 if (gen_words) {
5021 __ vmerge_vvm(vtemp2, rot3, rot2);
5022 __ vsha2ms_vv(rot1, vtemp2, rot4);
5023 }
5024 }
5025
5026 // Arguments:
5027 //
5028 // Inputs:
5029 // c_rarg0 - byte[] source+offset
5030 // c_rarg1 - int[] SHA.state
5031 // c_rarg2 - int offset
5032 // c_rarg3 - int limit
5033 //
5034 address generate_sha2_implCompress(Assembler::SEW vset_sew, StubId stub_id) {
5035 alignas(64) static const uint32_t round_consts_256[64] = {
5036 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
5037 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
5038 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
5039 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
5040 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
5041 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
5042 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
5043 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
5044 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
5045 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
5046 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
5047 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
5048 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
5049 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
5050 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
5051 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
5052 };
5053 alignas(64) static const uint64_t round_consts_512[80] = {
5054 0x428a2f98d728ae22l, 0x7137449123ef65cdl, 0xb5c0fbcfec4d3b2fl,
5055 0xe9b5dba58189dbbcl, 0x3956c25bf348b538l, 0x59f111f1b605d019l,
5056 0x923f82a4af194f9bl, 0xab1c5ed5da6d8118l, 0xd807aa98a3030242l,
5057 0x12835b0145706fbel, 0x243185be4ee4b28cl, 0x550c7dc3d5ffb4e2l,
5058 0x72be5d74f27b896fl, 0x80deb1fe3b1696b1l, 0x9bdc06a725c71235l,
5059 0xc19bf174cf692694l, 0xe49b69c19ef14ad2l, 0xefbe4786384f25e3l,
5060 0x0fc19dc68b8cd5b5l, 0x240ca1cc77ac9c65l, 0x2de92c6f592b0275l,
5061 0x4a7484aa6ea6e483l, 0x5cb0a9dcbd41fbd4l, 0x76f988da831153b5l,
5062 0x983e5152ee66dfabl, 0xa831c66d2db43210l, 0xb00327c898fb213fl,
5063 0xbf597fc7beef0ee4l, 0xc6e00bf33da88fc2l, 0xd5a79147930aa725l,
5064 0x06ca6351e003826fl, 0x142929670a0e6e70l, 0x27b70a8546d22ffcl,
5065 0x2e1b21385c26c926l, 0x4d2c6dfc5ac42aedl, 0x53380d139d95b3dfl,
5066 0x650a73548baf63del, 0x766a0abb3c77b2a8l, 0x81c2c92e47edaee6l,
5067 0x92722c851482353bl, 0xa2bfe8a14cf10364l, 0xa81a664bbc423001l,
5068 0xc24b8b70d0f89791l, 0xc76c51a30654be30l, 0xd192e819d6ef5218l,
5069 0xd69906245565a910l, 0xf40e35855771202al, 0x106aa07032bbd1b8l,
5070 0x19a4c116b8d2d0c8l, 0x1e376c085141ab53l, 0x2748774cdf8eeb99l,
5071 0x34b0bcb5e19b48a8l, 0x391c0cb3c5c95a63l, 0x4ed8aa4ae3418acbl,
5072 0x5b9cca4f7763e373l, 0x682e6ff3d6b2b8a3l, 0x748f82ee5defb2fcl,
5073 0x78a5636f43172f60l, 0x84c87814a1f0ab72l, 0x8cc702081a6439ecl,
5074 0x90befffa23631e28l, 0xa4506cebde82bde9l, 0xbef9a3f7b2c67915l,
5075 0xc67178f2e372532bl, 0xca273eceea26619cl, 0xd186b8c721c0c207l,
5076 0xeada7dd6cde0eb1el, 0xf57d4f7fee6ed178l, 0x06f067aa72176fbal,
5077 0x0a637dc5a2c898a6l, 0x113f9804bef90dael, 0x1b710b35131c471bl,
5078 0x28db77f523047d84l, 0x32caab7b40c72493l, 0x3c9ebe0a15c9bebcl,
5079 0x431d67c49c100d4cl, 0x4cc5d4becb3e42b6l, 0x597f299cfc657e2al,
5080 0x5fcb6fab3ad6faecl, 0x6c44198c4a475817l
5081 };
5082 const int const_add = vset_sew == Assembler::e32 ? 16 : 32;
5083
5084 bool multi_block;
5085 switch (stub_id) {
5086 case StubId::stubgen_sha256_implCompress_id:
5087 assert (vset_sew == Assembler::e32, "wrong macroassembler for stub");
5088 multi_block = false;
5089 break;
5090 case StubId::stubgen_sha256_implCompressMB_id:
5091 assert (vset_sew == Assembler::e32, "wrong macroassembler for stub");
5092 multi_block = true;
5093 break;
5094 case StubId::stubgen_sha512_implCompress_id:
5095 assert (vset_sew == Assembler::e64, "wrong macroassembler for stub");
5096 multi_block = false;
5097 break;
5098 case StubId::stubgen_sha512_implCompressMB_id:
5099 assert (vset_sew == Assembler::e64, "wrong macroassembler for stub");
5100 multi_block = true;
5101 break;
5102 default:
5103 ShouldNotReachHere();
5104 };
5105 __ align(CodeEntryAlignment);
5106 StubCodeMark mark(_cgen, stub_id);
5107 address start = __ pc();
5108
5109 Register buf = c_rarg0;
5110 Register state = c_rarg1;
5111 Register ofs = c_rarg2;
5112 Register limit = c_rarg3;
5113 Register consts = t2; // caller saved
5114 Register state_c = x28; // caller saved
5115 VectorRegister vindex = v2;
5116 VectorRegister vW0 = v4;
5117 VectorRegister vW1 = v6;
5118 VectorRegister vW2 = v8;
5119 VectorRegister vW3 = v10;
5120 VectorRegister vState0 = v12;
5121 VectorRegister vState1 = v14;
5122 VectorRegister vHash0 = v16;
5123 VectorRegister vHash1 = v18;
5124 VectorRegister vTmp0 = v20;
5125 VectorRegister vTmp1 = v22;
5126
5127 Label multi_block_loop;
5128
5129 __ enter();
5130
5131 address constant_table = vset_sew == Assembler::e32 ? (address)round_consts_256 : (address)round_consts_512;
5132 la(consts, ExternalAddress(constant_table));
5133
5134 // Register use in this function:
5135 //
5136 // VECTORS
5137 // vW0 - vW3 (512/1024-bits / 4*128/256 bits / 4*4*32/65 bits), hold the message
5138 // schedule words (Wt). They start with the message block
5139 // content (W0 to W15), then further words in the message
5140 // schedule generated via vsha2ms from previous Wt.
5141 // Initially:
5142 // vW0 = W[ 3:0] = { W3, W2, W1, W0}
5143 // vW1 = W[ 7:4] = { W7, W6, W5, W4}
5144 // vW2 = W[ 11:8] = {W11, W10, W9, W8}
5145 // vW3 = W[15:12] = {W15, W14, W13, W12}
5146 //
5147 // vState0 - vState1 hold the working state variables (a, b, ..., h)
5148 // vState0 = {f[t],e[t],b[t],a[t]}
5149 // vState1 = {h[t],g[t],d[t],c[t]}
5150 // Initially:
5151 // vState0 = {H5i-1, H4i-1, H1i-1 , H0i-1}
5152 // vState1 = {H7i-i, H6i-1, H3i-1 , H2i-1}
5153 //
5154 // v0 = masks for vrgather/vmerge. Single value during the 16 rounds.
5155 //
5156 // vTmp0 = temporary, Wt+Kt
5157 // vTmp1 = temporary, Kt
5158 //
5159 // vHash0/vHash1 = hold the initial values of the hash, byte-swapped.
5160 //
5161 // During most of the function the vector state is configured so that each
5162 // vector is interpreted as containing four 32/64 bits (e32/e64) elements (128/256 bits).
5163
5164 // vsha2ch/vsha2cl uses EGW of 4*SEW.
5165 // SHA256 SEW = e32, EGW = 128-bits
5166 // SHA512 SEW = e64, EGW = 256-bits
5167 //
5168 // VLEN is required to be at least 128.
5169 // For the case of VLEN=128 and SHA512 we need LMUL=2 to work with 4*e64 (EGW = 256)
5170 //
5171 // m1: LMUL=1/2
5172 // ta: tail agnostic (don't care about those lanes)
5173 // ma: mask agnostic (don't care about those lanes)
5174 // x0 is not written, we known the number of vector elements.
5175
5176 if (vset_sew == Assembler::e64 && MaxVectorSize == 16) { // SHA512 and VLEN = 128
5177 __ vsetivli(x0, 4, vset_sew, Assembler::m2, Assembler::ma, Assembler::ta);
5178 } else {
5179 __ vsetivli(x0, 4, vset_sew, Assembler::m1, Assembler::ma, Assembler::ta);
5180 }
5181
5182 int64_t indexes = vset_sew == Assembler::e32 ? 0x00041014ul : 0x00082028ul;
5183 __ li(t0, indexes);
5184 __ vmv_v_x(vindex, t0);
5185
5186 // Step-over a,b, so we are pointing to c.
5187 // const_add is equal to 4x state variable, div by 2 is thus 2, a,b
5188 __ addi(state_c, state, const_add/2);
5189
5190 // Use index-load to get {f,e,b,a},{h,g,d,c}
5191 __ vluxei8_v(vState0, state, vindex);
5192 __ vluxei8_v(vState1, state_c, vindex);
5193
5194 __ bind(multi_block_loop);
5195
5196 // Capture the initial H values in vHash0 and vHash1 to allow for computing
5197 // the resulting H', since H' = H+{a',b',c',...,h'}.
5198 __ vmv_v_v(vHash0, vState0);
5199 __ vmv_v_v(vHash1, vState1);
5200
5201 // Load the 512/1024-bits of the message block in vW0-vW3 and perform
5202 // an endian swap on each 4/8 bytes element.
5203 //
5204 // If Zvkb is not implemented one can use vrgather
5205 // with an index sequence to byte-swap.
5206 // sequence = [3 2 1 0 7 6 5 4 11 10 9 8 15 14 13 12]
5207 // <https://oeis.org/A004444> gives us "N ^ 3" as a nice formula to generate
5208 // this sequence. 'vid' gives us the N.
5209 __ vleXX_v(vset_sew, vW0, buf);
5210 __ vrev8_v(vW0, vW0);
5211 __ addi(buf, buf, const_add);
5212 __ vleXX_v(vset_sew, vW1, buf);
5213 __ vrev8_v(vW1, vW1);
5214 __ addi(buf, buf, const_add);
5215 __ vleXX_v(vset_sew, vW2, buf);
5216 __ vrev8_v(vW2, vW2);
5217 __ addi(buf, buf, const_add);
5218 __ vleXX_v(vset_sew, vW3, buf);
5219 __ vrev8_v(vW3, vW3);
5220 __ addi(buf, buf, const_add);
5221
5222 // Set v0 up for the vmerge that replaces the first word (idx==0)
5223 __ vid_v(v0);
5224 __ vmseq_vi(v0, v0, 0x0); // v0.mask[i] = (i == 0 ? 1 : 0)
5225
5226 VectorRegister rotation_regs[] = {vW0, vW1, vW2, vW3};
5227 int rot_pos = 0;
5228 // Quad-round #0 (+0, vW0->vW1->vW2->vW3) ... #11 (+3, vW3->vW0->vW1->vW2)
5229 const int qr_end = vset_sew == Assembler::e32 ? 12 : 16;
5230 for (int i = 0; i < qr_end; i++) {
5231 sha2_quad_round(vset_sew,
5232 rotation_regs[(rot_pos + 0) & 0x3],
5233 rotation_regs[(rot_pos + 1) & 0x3],
5234 rotation_regs[(rot_pos + 2) & 0x3],
5235 rotation_regs[(rot_pos + 3) & 0x3],
5236 consts,
5237 vTmp1, vTmp0, vState0, vState1);
5238 ++rot_pos;
5239 }
5240 // Quad-round #12 (+0, vW0->vW1->vW2->vW3) ... #15 (+3, vW3->vW0->vW1->vW2)
5241 // Note that we stop generating new message schedule words (Wt, vW0-13)
5242 // as we already generated all the words we end up consuming (i.e., W[63:60]).
5243 const int qr_c_end = qr_end + 4;
5244 for (int i = qr_end; i < qr_c_end; i++) {
5245 sha2_quad_round(vset_sew,
5246 rotation_regs[(rot_pos + 0) & 0x3],
5247 rotation_regs[(rot_pos + 1) & 0x3],
5248 rotation_regs[(rot_pos + 2) & 0x3],
5249 rotation_regs[(rot_pos + 3) & 0x3],
5250 consts,
5251 vTmp1, vTmp0, vState0, vState1, false, i < (qr_c_end-1));
5252 ++rot_pos;
5253 }
5254
5255 //--------------------------------------------------------------------------------
5256 // Compute the updated hash value H'
5257 // H' = H + {h',g',...,b',a'}
5258 // = {h,g,...,b,a} + {h',g',...,b',a'}
5259 // = {h+h',g+g',...,b+b',a+a'}
5260
5261 // H' = H+{a',b',c',...,h'}
5262 __ vadd_vv(vState0, vHash0, vState0);
5263 __ vadd_vv(vState1, vHash1, vState1);
5264
5265 if (multi_block) {
5266 int total_adds = vset_sew == Assembler::e32 ? 240 : 608;
5267 __ subi(consts, consts, total_adds);
5268 __ addi(ofs, ofs, vset_sew == Assembler::e32 ? 64 : 128);
5269 __ ble(ofs, limit, multi_block_loop);
5270 __ mv(c_rarg0, ofs); // return ofs
5271 }
5272
5273 // Store H[0..8] = {a,b,c,d,e,f,g,h} from
5274 // vState0 = {f,e,b,a}
5275 // vState1 = {h,g,d,c}
5276 __ vsuxei8_v(vState0, state, vindex);
5277 __ vsuxei8_v(vState1, state_c, vindex);
5278
5279 __ leave();
5280 __ ret();
5281
5282 return start;
5283 }
5284 };
5285
5286 #undef __
5287 #define __ _masm->
5288
5289 // Set of L registers that correspond to a contiguous memory area.
5290 // Each 64-bit register typically corresponds to 2 32-bit integers.
5291 template <uint L>
5292 class RegCache {
5293 private:
5294 MacroAssembler *_masm;
5295 Register _regs[L];
5296
5297 public:
5298 RegCache(MacroAssembler *masm, RegSet rs): _masm(masm) {
5299 assert(rs.size() == L, "%u registers are used to cache %u 4-byte data", rs.size(), 2 * L);
5300 auto it = rs.begin();
5301 for (auto &r: _regs) {
5302 r = *it;
5303 ++it;
5304 }
5305 }
5306
5307 // generate load for the i'th register
5308 void gen_load(uint i, Register base) {
5309 assert(i < L, "invalid i: %u", i);
5310 __ ld(_regs[i], Address(base, 8 * i));
5311 }
5312
5313 // add i'th 32-bit integer to dest
5314 void add_u32(const Register dest, uint i, const Register rtmp = t0) {
5315 assert(i < 2 * L, "invalid i: %u", i);
5316
5317 if (is_even(i)) {
5318 // Use the bottom 32 bits. No need to mask off the top 32 bits
5319 // as addw will do the right thing.
5320 __ addw(dest, dest, _regs[i / 2]);
5321 } else {
5322 // Use the top 32 bits by right-shifting them.
5323 __ srli(rtmp, _regs[i / 2], 32);
5324 __ addw(dest, dest, rtmp);
5325 }
5326 }
5327 };
5328
5329 typedef RegCache<8> BufRegCache;
5330
5331 // a += value + x + ac;
5332 // a = Integer.rotateLeft(a, s) + b;
5333 void m5_FF_GG_HH_II_epilogue(BufRegCache& reg_cache,
5334 Register a, Register b, Register c, Register d,
5335 int k, int s, int t,
5336 Register value) {
5337 // a += ac
5338 __ addw(a, a, t, t1);
5339
5340 // a += x;
5341 reg_cache.add_u32(a, k);
5342 // a += value;
5343 __ addw(a, a, value);
5344
5345 // a = Integer.rotateLeft(a, s) + b;
5346 __ rolw(a, a, s);
5347 __ addw(a, a, b);
5348 }
5349
5350 // a += ((b & c) | ((~b) & d)) + x + ac;
5351 // a = Integer.rotateLeft(a, s) + b;
5352 void md5_FF(BufRegCache& reg_cache,
5353 Register a, Register b, Register c, Register d,
5354 int k, int s, int t,
5355 Register rtmp1, Register rtmp2) {
5356 // rtmp1 = b & c
5357 __ andr(rtmp1, b, c);
5358
5359 // rtmp2 = (~b) & d
5360 __ andn(rtmp2, d, b);
5361
5362 // rtmp1 = (b & c) | ((~b) & d)
5363 __ orr(rtmp1, rtmp1, rtmp2);
5364
5365 m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1);
5366 }
5367
5368 // a += ((b & d) | (c & (~d))) + x + ac;
5369 // a = Integer.rotateLeft(a, s) + b;
5370 void md5_GG(BufRegCache& reg_cache,
5371 Register a, Register b, Register c, Register d,
5372 int k, int s, int t,
5373 Register rtmp1, Register rtmp2) {
5374 // rtmp1 = b & d
5375 __ andr(rtmp1, b, d);
5376
5377 // rtmp2 = c & (~d)
5378 __ andn(rtmp2, c, d);
5379
5380 // rtmp1 = (b & d) | (c & (~d))
5381 __ orr(rtmp1, rtmp1, rtmp2);
5382
5383 m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1);
5384 }
5385
5386 // a += ((b ^ c) ^ d) + x + ac;
5387 // a = Integer.rotateLeft(a, s) + b;
5388 void md5_HH(BufRegCache& reg_cache,
5389 Register a, Register b, Register c, Register d,
5390 int k, int s, int t,
5391 Register rtmp1, Register rtmp2) {
5392 // rtmp1 = (b ^ c) ^ d
5393 __ xorr(rtmp2, b, c);
5394 __ xorr(rtmp1, rtmp2, d);
5395
5396 m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1);
5397 }
5398
5399 // a += (c ^ (b | (~d))) + x + ac;
5400 // a = Integer.rotateLeft(a, s) + b;
5401 void md5_II(BufRegCache& reg_cache,
5402 Register a, Register b, Register c, Register d,
5403 int k, int s, int t,
5404 Register rtmp1, Register rtmp2) {
5405 // rtmp1 = c ^ (b | (~d))
5406 __ orn(rtmp2, b, d);
5407 __ xorr(rtmp1, c, rtmp2);
5408
5409 m5_FF_GG_HH_II_epilogue(reg_cache, a, b, c, d, k, s, t, rtmp1);
5410 }
5411
5412 // Arguments:
5413 //
5414 // Inputs:
5415 // c_rarg0 - byte[] source+offset
5416 // c_rarg1 - int[] SHA.state
5417 // c_rarg2 - int offset (multi_block == True)
5418 // c_rarg3 - int limit (multi_block == True)
5419 //
5420 // Registers:
5421 // x0 zero (zero)
5422 // x1 ra (return address)
5423 // x2 sp (stack pointer)
5424 // x3 gp (global pointer)
5425 // x4 tp (thread pointer)
5426 // x5 t0 (tmp register)
5427 // x6 t1 (tmp register)
5428 // x7 t2 state0
5429 // x8 f0/s0 (frame pointer)
5430 // x9 s1
5431 // x10 a0 rtmp1 / c_rarg0
5432 // x11 a1 rtmp2 / c_rarg1
5433 // x12 a2 a / c_rarg2
5434 // x13 a3 b / c_rarg3
5435 // x14 a4 c
5436 // x15 a5 d
5437 // x16 a6 buf
5438 // x17 a7 state
5439 // x18 s2 ofs [saved-reg] (multi_block == True)
5440 // x19 s3 limit [saved-reg] (multi_block == True)
5441 // x20 s4 state1 [saved-reg]
5442 // x21 s5 state2 [saved-reg]
5443 // x22 s6 state3 [saved-reg]
5444 // x23 s7
5445 // x24 s8 buf0 [saved-reg]
5446 // x25 s9 buf1 [saved-reg]
5447 // x26 s10 buf2 [saved-reg]
5448 // x27 s11 buf3 [saved-reg]
5449 // x28 t3 buf4
5450 // x29 t4 buf5
5451 // x30 t5 buf6
5452 // x31 t6 buf7
5453 address generate_md5_implCompress(StubId stub_id) {
5454 __ align(CodeEntryAlignment);
5455 bool multi_block;
5456 switch (stub_id) {
5457 case StubId::stubgen_md5_implCompress_id:
5458 multi_block = false;
5459 break;
5460 case StubId::stubgen_md5_implCompressMB_id:
5461 multi_block = true;
5462 break;
5463 default:
5464 ShouldNotReachHere();
5465 };
5466 StubCodeMark mark(this, stub_id);
5467 address start = __ pc();
5468
5469 // rotation constants
5470 const int S11 = 7;
5471 const int S12 = 12;
5472 const int S13 = 17;
5473 const int S14 = 22;
5474 const int S21 = 5;
5475 const int S22 = 9;
5476 const int S23 = 14;
5477 const int S24 = 20;
5478 const int S31 = 4;
5479 const int S32 = 11;
5480 const int S33 = 16;
5481 const int S34 = 23;
5482 const int S41 = 6;
5483 const int S42 = 10;
5484 const int S43 = 15;
5485 const int S44 = 21;
5486
5487 const int64_t mask32 = 0xffffffff;
5488
5489 Register buf_arg = c_rarg0; // a0
5490 Register state_arg = c_rarg1; // a1
5491 Register ofs_arg = c_rarg2; // a2
5492 Register limit_arg = c_rarg3; // a3
5493
5494 // we'll copy the args to these registers to free up a0-a3
5495 // to use for other values manipulated by instructions
5496 // that can be compressed
5497 Register buf = x16; // a6
5498 Register state = x17; // a7
5499 Register ofs = x18; // s2
5500 Register limit = x19; // s3
5501
5502 // using x12->15 to allow compressed instructions
5503 Register a = x12; // a2
5504 Register b = x13; // a3
5505 Register c = x14; // a4
5506 Register d = x15; // a5
5507
5508 Register state0 = x7; // t2
5509 Register state1 = x20; // s4
5510 Register state2 = x21; // s5
5511 Register state3 = x22; // s6
5512
5513 // using x10->x11 to allow compressed instructions
5514 Register rtmp1 = x10; // a0
5515 Register rtmp2 = x11; // a1
5516
5517 RegSet reg_cache_saved_regs = RegSet::of(x24, x25, x26, x27); // s8, s9, s10, s11
5518 RegSet reg_cache_regs;
5519 reg_cache_regs += reg_cache_saved_regs;
5520 reg_cache_regs += RegSet::of(t3, t4, t5, t6);
5521 BufRegCache reg_cache(_masm, reg_cache_regs);
5522
5523 RegSet saved_regs;
5524 if (multi_block) {
5525 saved_regs += RegSet::of(ofs, limit);
5526 }
5527 saved_regs += RegSet::of(state1, state2, state3);
5528 saved_regs += reg_cache_saved_regs;
5529
5530 __ push_reg(saved_regs, sp);
5531
5532 __ mv(buf, buf_arg);
5533 __ mv(state, state_arg);
5534 if (multi_block) {
5535 __ mv(ofs, ofs_arg);
5536 __ mv(limit, limit_arg);
5537 }
5538
5539 // to minimize the number of memory operations:
5540 // read the 4 state 4-byte values in pairs, with a single ld,
5541 // and split them into 2 registers.
5542 //
5543 // And, as the core algorithm of md5 works on 32-bits words, so
5544 // in the following code, it does not care about the content of
5545 // higher 32-bits in state[x]. Based on this observation,
5546 // we can apply further optimization, which is to just ignore the
5547 // higher 32-bits in state0/state2, rather than set the higher
5548 // 32-bits of state0/state2 to zero explicitly with extra instructions.
5549 __ ld(state0, Address(state));
5550 __ srli(state1, state0, 32);
5551 __ ld(state2, Address(state, 8));
5552 __ srli(state3, state2, 32);
5553
5554 Label md5_loop;
5555 __ BIND(md5_loop);
5556
5557 __ mv(a, state0);
5558 __ mv(b, state1);
5559 __ mv(c, state2);
5560 __ mv(d, state3);
5561
5562 // Round 1
5563 reg_cache.gen_load(0, buf);
5564 md5_FF(reg_cache, a, b, c, d, 0, S11, 0xd76aa478, rtmp1, rtmp2);
5565 md5_FF(reg_cache, d, a, b, c, 1, S12, 0xe8c7b756, rtmp1, rtmp2);
5566 reg_cache.gen_load(1, buf);
5567 md5_FF(reg_cache, c, d, a, b, 2, S13, 0x242070db, rtmp1, rtmp2);
5568 md5_FF(reg_cache, b, c, d, a, 3, S14, 0xc1bdceee, rtmp1, rtmp2);
5569 reg_cache.gen_load(2, buf);
5570 md5_FF(reg_cache, a, b, c, d, 4, S11, 0xf57c0faf, rtmp1, rtmp2);
5571 md5_FF(reg_cache, d, a, b, c, 5, S12, 0x4787c62a, rtmp1, rtmp2);
5572 reg_cache.gen_load(3, buf);
5573 md5_FF(reg_cache, c, d, a, b, 6, S13, 0xa8304613, rtmp1, rtmp2);
5574 md5_FF(reg_cache, b, c, d, a, 7, S14, 0xfd469501, rtmp1, rtmp2);
5575 reg_cache.gen_load(4, buf);
5576 md5_FF(reg_cache, a, b, c, d, 8, S11, 0x698098d8, rtmp1, rtmp2);
5577 md5_FF(reg_cache, d, a, b, c, 9, S12, 0x8b44f7af, rtmp1, rtmp2);
5578 reg_cache.gen_load(5, buf);
5579 md5_FF(reg_cache, c, d, a, b, 10, S13, 0xffff5bb1, rtmp1, rtmp2);
5580 md5_FF(reg_cache, b, c, d, a, 11, S14, 0x895cd7be, rtmp1, rtmp2);
5581 reg_cache.gen_load(6, buf);
5582 md5_FF(reg_cache, a, b, c, d, 12, S11, 0x6b901122, rtmp1, rtmp2);
5583 md5_FF(reg_cache, d, a, b, c, 13, S12, 0xfd987193, rtmp1, rtmp2);
5584 reg_cache.gen_load(7, buf);
5585 md5_FF(reg_cache, c, d, a, b, 14, S13, 0xa679438e, rtmp1, rtmp2);
5586 md5_FF(reg_cache, b, c, d, a, 15, S14, 0x49b40821, rtmp1, rtmp2);
5587
5588 // Round 2
5589 md5_GG(reg_cache, a, b, c, d, 1, S21, 0xf61e2562, rtmp1, rtmp2);
5590 md5_GG(reg_cache, d, a, b, c, 6, S22, 0xc040b340, rtmp1, rtmp2);
5591 md5_GG(reg_cache, c, d, a, b, 11, S23, 0x265e5a51, rtmp1, rtmp2);
5592 md5_GG(reg_cache, b, c, d, a, 0, S24, 0xe9b6c7aa, rtmp1, rtmp2);
5593 md5_GG(reg_cache, a, b, c, d, 5, S21, 0xd62f105d, rtmp1, rtmp2);
5594 md5_GG(reg_cache, d, a, b, c, 10, S22, 0x02441453, rtmp1, rtmp2);
5595 md5_GG(reg_cache, c, d, a, b, 15, S23, 0xd8a1e681, rtmp1, rtmp2);
5596 md5_GG(reg_cache, b, c, d, a, 4, S24, 0xe7d3fbc8, rtmp1, rtmp2);
5597 md5_GG(reg_cache, a, b, c, d, 9, S21, 0x21e1cde6, rtmp1, rtmp2);
5598 md5_GG(reg_cache, d, a, b, c, 14, S22, 0xc33707d6, rtmp1, rtmp2);
5599 md5_GG(reg_cache, c, d, a, b, 3, S23, 0xf4d50d87, rtmp1, rtmp2);
5600 md5_GG(reg_cache, b, c, d, a, 8, S24, 0x455a14ed, rtmp1, rtmp2);
5601 md5_GG(reg_cache, a, b, c, d, 13, S21, 0xa9e3e905, rtmp1, rtmp2);
5602 md5_GG(reg_cache, d, a, b, c, 2, S22, 0xfcefa3f8, rtmp1, rtmp2);
5603 md5_GG(reg_cache, c, d, a, b, 7, S23, 0x676f02d9, rtmp1, rtmp2);
5604 md5_GG(reg_cache, b, c, d, a, 12, S24, 0x8d2a4c8a, rtmp1, rtmp2);
5605
5606 // Round 3
5607 md5_HH(reg_cache, a, b, c, d, 5, S31, 0xfffa3942, rtmp1, rtmp2);
5608 md5_HH(reg_cache, d, a, b, c, 8, S32, 0x8771f681, rtmp1, rtmp2);
5609 md5_HH(reg_cache, c, d, a, b, 11, S33, 0x6d9d6122, rtmp1, rtmp2);
5610 md5_HH(reg_cache, b, c, d, a, 14, S34, 0xfde5380c, rtmp1, rtmp2);
5611 md5_HH(reg_cache, a, b, c, d, 1, S31, 0xa4beea44, rtmp1, rtmp2);
5612 md5_HH(reg_cache, d, a, b, c, 4, S32, 0x4bdecfa9, rtmp1, rtmp2);
5613 md5_HH(reg_cache, c, d, a, b, 7, S33, 0xf6bb4b60, rtmp1, rtmp2);
5614 md5_HH(reg_cache, b, c, d, a, 10, S34, 0xbebfbc70, rtmp1, rtmp2);
5615 md5_HH(reg_cache, a, b, c, d, 13, S31, 0x289b7ec6, rtmp1, rtmp2);
5616 md5_HH(reg_cache, d, a, b, c, 0, S32, 0xeaa127fa, rtmp1, rtmp2);
5617 md5_HH(reg_cache, c, d, a, b, 3, S33, 0xd4ef3085, rtmp1, rtmp2);
5618 md5_HH(reg_cache, b, c, d, a, 6, S34, 0x04881d05, rtmp1, rtmp2);
5619 md5_HH(reg_cache, a, b, c, d, 9, S31, 0xd9d4d039, rtmp1, rtmp2);
5620 md5_HH(reg_cache, d, a, b, c, 12, S32, 0xe6db99e5, rtmp1, rtmp2);
5621 md5_HH(reg_cache, c, d, a, b, 15, S33, 0x1fa27cf8, rtmp1, rtmp2);
5622 md5_HH(reg_cache, b, c, d, a, 2, S34, 0xc4ac5665, rtmp1, rtmp2);
5623
5624 // Round 4
5625 md5_II(reg_cache, a, b, c, d, 0, S41, 0xf4292244, rtmp1, rtmp2);
5626 md5_II(reg_cache, d, a, b, c, 7, S42, 0x432aff97, rtmp1, rtmp2);
5627 md5_II(reg_cache, c, d, a, b, 14, S43, 0xab9423a7, rtmp1, rtmp2);
5628 md5_II(reg_cache, b, c, d, a, 5, S44, 0xfc93a039, rtmp1, rtmp2);
5629 md5_II(reg_cache, a, b, c, d, 12, S41, 0x655b59c3, rtmp1, rtmp2);
5630 md5_II(reg_cache, d, a, b, c, 3, S42, 0x8f0ccc92, rtmp1, rtmp2);
5631 md5_II(reg_cache, c, d, a, b, 10, S43, 0xffeff47d, rtmp1, rtmp2);
5632 md5_II(reg_cache, b, c, d, a, 1, S44, 0x85845dd1, rtmp1, rtmp2);
5633 md5_II(reg_cache, a, b, c, d, 8, S41, 0x6fa87e4f, rtmp1, rtmp2);
5634 md5_II(reg_cache, d, a, b, c, 15, S42, 0xfe2ce6e0, rtmp1, rtmp2);
5635 md5_II(reg_cache, c, d, a, b, 6, S43, 0xa3014314, rtmp1, rtmp2);
5636 md5_II(reg_cache, b, c, d, a, 13, S44, 0x4e0811a1, rtmp1, rtmp2);
5637 md5_II(reg_cache, a, b, c, d, 4, S41, 0xf7537e82, rtmp1, rtmp2);
5638 md5_II(reg_cache, d, a, b, c, 11, S42, 0xbd3af235, rtmp1, rtmp2);
5639 md5_II(reg_cache, c, d, a, b, 2, S43, 0x2ad7d2bb, rtmp1, rtmp2);
5640 md5_II(reg_cache, b, c, d, a, 9, S44, 0xeb86d391, rtmp1, rtmp2);
5641
5642 __ addw(state0, state0, a);
5643 __ addw(state1, state1, b);
5644 __ addw(state2, state2, c);
5645 __ addw(state3, state3, d);
5646
5647 if (multi_block) {
5648 __ addi(buf, buf, 64);
5649 __ addi(ofs, ofs, 64);
5650 // if (ofs <= limit) goto m5_loop
5651 __ bge(limit, ofs, md5_loop);
5652 __ mv(c_rarg0, ofs); // return ofs
5653 }
5654
5655 // to minimize the number of memory operations:
5656 // write back the 4 state 4-byte values in pairs, with a single sd
5657 __ mv(t0, mask32);
5658 __ andr(state0, state0, t0);
5659 __ slli(state1, state1, 32);
5660 __ orr(state0, state0, state1);
5661 __ sd(state0, Address(state));
5662 __ andr(state2, state2, t0);
5663 __ slli(state3, state3, 32);
5664 __ orr(state2, state2, state3);
5665 __ sd(state2, Address(state, 8));
5666
5667 __ pop_reg(saved_regs, sp);
5668 __ ret();
5669
5670 return (address) start;
5671 }
5672
5673 /**
5674 * Perform the quarter round calculations on values contained within four vector registers.
5675 *
5676 * @param aVec the SIMD register containing only the "a" values
5677 * @param bVec the SIMD register containing only the "b" values
5678 * @param cVec the SIMD register containing only the "c" values
5679 * @param dVec the SIMD register containing only the "d" values
5680 * @param tmp_vr temporary vector register holds intermedia values.
5681 */
5682 void chacha20_quarter_round(VectorRegister aVec, VectorRegister bVec,
5683 VectorRegister cVec, VectorRegister dVec, VectorRegister tmp_vr) {
5684 // a += b, d ^= a, d <<<= 16
5685 __ vadd_vv(aVec, aVec, bVec);
5686 __ vxor_vv(dVec, dVec, aVec);
5687 __ vrole32_vi(dVec, 16, tmp_vr);
5688
5689 // c += d, b ^= c, b <<<= 12
5690 __ vadd_vv(cVec, cVec, dVec);
5691 __ vxor_vv(bVec, bVec, cVec);
5692 __ vrole32_vi(bVec, 12, tmp_vr);
5693
5694 // a += b, d ^= a, d <<<= 8
5695 __ vadd_vv(aVec, aVec, bVec);
5696 __ vxor_vv(dVec, dVec, aVec);
5697 __ vrole32_vi(dVec, 8, tmp_vr);
5698
5699 // c += d, b ^= c, b <<<= 7
5700 __ vadd_vv(cVec, cVec, dVec);
5701 __ vxor_vv(bVec, bVec, cVec);
5702 __ vrole32_vi(bVec, 7, tmp_vr);
5703 }
5704
5705 /**
5706 * int com.sun.crypto.provider.ChaCha20Cipher.implChaCha20Block(int[] initState, byte[] result)
5707 *
5708 * Input arguments:
5709 * c_rarg0 - state, the starting state
5710 * c_rarg1 - key_stream, the array that will hold the result of the ChaCha20 block function
5711 *
5712 * Implementation Note:
5713 * Parallelization is achieved by loading individual state elements into vectors for N blocks.
5714 * N depends on single vector register length.
5715 */
5716 address generate_chacha20Block() {
5717 Label L_Rounds;
5718
5719 __ align(CodeEntryAlignment);
5720 StubId stub_id = StubId::stubgen_chacha20Block_id;
5721 StubCodeMark mark(this, stub_id);
5722 address start = __ pc();
5723 __ enter();
5724
5725 const int states_len = 16;
5726 const int step = 4;
5727 const Register state = c_rarg0;
5728 const Register key_stream = c_rarg1;
5729 const Register tmp_addr = t0;
5730 const Register length = t1;
5731
5732 // Organize vector registers in an array that facilitates
5733 // putting repetitive opcodes into loop structures below.
5734 const VectorRegister work_vrs[16] = {
5735 v0, v1, v2, v3, v4, v5, v6, v7,
5736 v8, v9, v10, v11, v12, v13, v14, v15
5737 };
5738 const VectorRegister tmp_vr = v16;
5739 const VectorRegister counter_vr = v17;
5740
5741 {
5742 // Put 16 here, as com.sun.crypto.providerChaCha20Cipher.KS_MAX_LEN is 1024
5743 // in java level.
5744 __ vsetivli(length, 16, Assembler::e32, Assembler::m1);
5745 }
5746
5747 // Load from source state.
5748 // Every element in source state is duplicated to all elements in the corresponding vector.
5749 __ mv(tmp_addr, state);
5750 for (int i = 0; i < states_len; i += 1) {
5751 __ vlse32_v(work_vrs[i], tmp_addr, zr);
5752 __ addi(tmp_addr, tmp_addr, step);
5753 }
5754 // Adjust counter for every individual block.
5755 __ vid_v(counter_vr);
5756 __ vadd_vv(work_vrs[12], work_vrs[12], counter_vr);
5757
5758 // Perform 10 iterations of the 8 quarter round set
5759 {
5760 const Register loop = t2; // share t2 with other non-overlapping usages.
5761 __ mv(loop, 10);
5762 __ BIND(L_Rounds);
5763
5764 chacha20_quarter_round(work_vrs[0], work_vrs[4], work_vrs[8], work_vrs[12], tmp_vr);
5765 chacha20_quarter_round(work_vrs[1], work_vrs[5], work_vrs[9], work_vrs[13], tmp_vr);
5766 chacha20_quarter_round(work_vrs[2], work_vrs[6], work_vrs[10], work_vrs[14], tmp_vr);
5767 chacha20_quarter_round(work_vrs[3], work_vrs[7], work_vrs[11], work_vrs[15], tmp_vr);
5768
5769 chacha20_quarter_round(work_vrs[0], work_vrs[5], work_vrs[10], work_vrs[15], tmp_vr);
5770 chacha20_quarter_round(work_vrs[1], work_vrs[6], work_vrs[11], work_vrs[12], tmp_vr);
5771 chacha20_quarter_round(work_vrs[2], work_vrs[7], work_vrs[8], work_vrs[13], tmp_vr);
5772 chacha20_quarter_round(work_vrs[3], work_vrs[4], work_vrs[9], work_vrs[14], tmp_vr);
5773
5774 __ subi(loop, loop, 1);
5775 __ bnez(loop, L_Rounds);
5776 }
5777
5778 // Add the original state into the end working state.
5779 // We do this by first duplicating every element in source state array to the corresponding
5780 // vector, then adding it to the post-loop working state.
5781 __ mv(tmp_addr, state);
5782 for (int i = 0; i < states_len; i += 1) {
5783 __ vlse32_v(tmp_vr, tmp_addr, zr);
5784 __ addi(tmp_addr, tmp_addr, step);
5785 __ vadd_vv(work_vrs[i], work_vrs[i], tmp_vr);
5786 }
5787 // Add the counter overlay onto work_vrs[12] at the end.
5788 __ vadd_vv(work_vrs[12], work_vrs[12], counter_vr);
5789
5790 // Store result to key stream.
5791 {
5792 const Register stride = t2; // share t2 with other non-overlapping usages.
5793 // Every block occupies 64 bytes, so we use 64 as stride of the vector store.
5794 __ mv(stride, 64);
5795 for (int i = 0; i < states_len; i += 1) {
5796 __ vsse32_v(work_vrs[i], key_stream, stride);
5797 __ addi(key_stream, key_stream, step);
5798 }
5799 }
5800
5801 // Return length of output key_stream
5802 __ slli(c_rarg0, length, 6);
5803
5804 __ leave();
5805 __ ret();
5806
5807 return (address) start;
5808 }
5809
5810
5811 // ------------------------ SHA-1 intrinsic ------------------------
5812
5813 // K't =
5814 // 5a827999, 0 <= t <= 19
5815 // 6ed9eba1, 20 <= t <= 39
5816 // 8f1bbcdc, 40 <= t <= 59
5817 // ca62c1d6, 60 <= t <= 79
5818 void sha1_prepare_k(Register cur_k, int round) {
5819 assert(round >= 0 && round < 80, "must be");
5820
5821 static const int64_t ks[] = {0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6};
5822 if ((round % 20) == 0) {
5823 __ mv(cur_k, ks[round/20]);
5824 }
5825 }
5826
5827 // W't =
5828 // M't, 0 <= t <= 15
5829 // ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79
5830 void sha1_prepare_w(Register cur_w, Register ws[], Register buf, int round) {
5831 assert(round >= 0 && round < 80, "must be");
5832
5833 if (round < 16) {
5834 // in the first 16 rounds, in ws[], every register contains 2 W't, e.g.
5835 // in ws[0], high part contains W't-0, low part contains W't-1,
5836 // in ws[1], high part contains W't-2, low part contains W't-3,
5837 // ...
5838 // in ws[7], high part contains W't-14, low part contains W't-15.
5839
5840 if ((round % 2) == 0) {
5841 __ ld(ws[round/2], Address(buf, (round/2) * 8));
5842 // reverse bytes, as SHA-1 is defined in big-endian.
5843 __ revb(ws[round/2], ws[round/2]);
5844 __ srli(cur_w, ws[round/2], 32);
5845 } else {
5846 __ mv(cur_w, ws[round/2]);
5847 }
5848
5849 return;
5850 }
5851
5852 if ((round % 2) == 0) {
5853 int idx = 16;
5854 // W't = ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79
5855 __ srli(t1, ws[(idx-8)/2], 32);
5856 __ xorr(t0, ws[(idx-3)/2], t1);
5857
5858 __ srli(t1, ws[(idx-14)/2], 32);
5859 __ srli(cur_w, ws[(idx-16)/2], 32);
5860 __ xorr(cur_w, cur_w, t1);
5861
5862 __ xorr(cur_w, cur_w, t0);
5863 __ rolw(cur_w, cur_w, 1, t0);
5864
5865 // copy the cur_w value to ws[8].
5866 // now, valid w't values are at:
5867 // w0: ws[0]'s lower 32 bits
5868 // w1 ~ w14: ws[1] ~ ws[7]
5869 // w15: ws[8]'s higher 32 bits
5870 __ slli(ws[idx/2], cur_w, 32);
5871
5872 return;
5873 }
5874
5875 int idx = 17;
5876 // W't = ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79
5877 __ srli(t1, ws[(idx-3)/2], 32);
5878 __ xorr(t0, t1, ws[(idx-8)/2]);
5879
5880 __ xorr(cur_w, ws[(idx-16)/2], ws[(idx-14)/2]);
5881
5882 __ xorr(cur_w, cur_w, t0);
5883 __ rolw(cur_w, cur_w, 1, t0);
5884
5885 // copy the cur_w value to ws[8]
5886 __ zext(cur_w, cur_w, 32);
5887 __ orr(ws[idx/2], ws[idx/2], cur_w);
5888
5889 // shift the w't registers, so they start from ws[0] again.
5890 // now, valid w't values are at:
5891 // w0 ~ w15: ws[0] ~ ws[7]
5892 Register ws_0 = ws[0];
5893 for (int i = 0; i < 16/2; i++) {
5894 ws[i] = ws[i+1];
5895 }
5896 ws[8] = ws_0;
5897 }
5898
5899 // f't(x, y, z) =
5900 // Ch(x, y, z) = (x & y) ^ (~x & z) , 0 <= t <= 19
5901 // Parity(x, y, z) = x ^ y ^ z , 20 <= t <= 39
5902 // Maj(x, y, z) = (x & y) ^ (x & z) ^ (y & z) , 40 <= t <= 59
5903 // Parity(x, y, z) = x ^ y ^ z , 60 <= t <= 79
5904 void sha1_f(Register dst, Register x, Register y, Register z, int round) {
5905 assert(round >= 0 && round < 80, "must be");
5906 assert_different_registers(dst, x, y, z, t0, t1);
5907
5908 if (round < 20) {
5909 // (x & y) ^ (~x & z)
5910 __ andr(t0, x, y);
5911 __ andn(dst, z, x);
5912 __ xorr(dst, dst, t0);
5913 } else if (round >= 40 && round < 60) {
5914 // (x & y) ^ (x & z) ^ (y & z)
5915 __ andr(t0, x, y);
5916 __ andr(t1, x, z);
5917 __ andr(dst, y, z);
5918 __ xorr(dst, dst, t0);
5919 __ xorr(dst, dst, t1);
5920 } else {
5921 // x ^ y ^ z
5922 __ xorr(dst, x, y);
5923 __ xorr(dst, dst, z);
5924 }
5925 }
5926
5927 // T = ROTL'5(a) + f't(b, c, d) + e + K't + W't
5928 // e = d
5929 // d = c
5930 // c = ROTL'30(b)
5931 // b = a
5932 // a = T
5933 void sha1_process_round(Register a, Register b, Register c, Register d, Register e,
5934 Register cur_k, Register cur_w, Register tmp, int round) {
5935 assert(round >= 0 && round < 80, "must be");
5936 assert_different_registers(a, b, c, d, e, cur_w, cur_k, tmp, t0);
5937
5938 // T = ROTL'5(a) + f't(b, c, d) + e + K't + W't
5939
5940 // cur_w will be recalculated at the beginning of each round,
5941 // so, we can reuse it as a temp register here.
5942 Register tmp2 = cur_w;
5943
5944 // reuse e as a temporary register, as we will mv new value into it later
5945 Register tmp3 = e;
5946 __ add(tmp2, cur_k, tmp2);
5947 __ add(tmp3, tmp3, tmp2);
5948 __ rolw(tmp2, a, 5, t0);
5949
5950 sha1_f(tmp, b, c, d, round);
5951
5952 __ add(tmp2, tmp2, tmp);
5953 __ add(tmp2, tmp2, tmp3);
5954
5955 // e = d
5956 // d = c
5957 // c = ROTL'30(b)
5958 // b = a
5959 // a = T
5960 __ mv(e, d);
5961 __ mv(d, c);
5962
5963 __ rolw(c, b, 30);
5964 __ mv(b, a);
5965 __ mv(a, tmp2);
5966 }
5967
5968 // H(i)0 = a + H(i-1)0
5969 // H(i)1 = b + H(i-1)1
5970 // H(i)2 = c + H(i-1)2
5971 // H(i)3 = d + H(i-1)3
5972 // H(i)4 = e + H(i-1)4
5973 void sha1_calculate_im_hash(Register a, Register b, Register c, Register d, Register e,
5974 Register prev_ab, Register prev_cd, Register prev_e) {
5975 assert_different_registers(a, b, c, d, e, prev_ab, prev_cd, prev_e);
5976
5977 __ add(a, a, prev_ab);
5978 __ srli(prev_ab, prev_ab, 32);
5979 __ add(b, b, prev_ab);
5980
5981 __ add(c, c, prev_cd);
5982 __ srli(prev_cd, prev_cd, 32);
5983 __ add(d, d, prev_cd);
5984
5985 __ add(e, e, prev_e);
5986 }
5987
5988 void sha1_preserve_prev_abcde(Register a, Register b, Register c, Register d, Register e,
5989 Register prev_ab, Register prev_cd, Register prev_e) {
5990 assert_different_registers(a, b, c, d, e, prev_ab, prev_cd, prev_e, t0);
5991
5992 __ slli(t0, b, 32);
5993 __ zext(prev_ab, a, 32);
5994 __ orr(prev_ab, prev_ab, t0);
5995
5996 __ slli(t0, d, 32);
5997 __ zext(prev_cd, c, 32);
5998 __ orr(prev_cd, prev_cd, t0);
5999
6000 __ mv(prev_e, e);
6001 }
6002
6003 // Intrinsic for:
6004 // void sun.security.provider.SHA.implCompress0(byte[] buf, int ofs)
6005 // void sun.security.provider.DigestBase.implCompressMultiBlock0(byte[] b, int ofs, int limit)
6006 //
6007 // Arguments:
6008 //
6009 // Inputs:
6010 // c_rarg0: byte[] src array + offset
6011 // c_rarg1: int[] SHA.state
6012 // - - - - - - below are only for implCompressMultiBlock0 - - - - - -
6013 // c_rarg2: int offset
6014 // c_rarg3: int limit
6015 //
6016 // Outputs:
6017 // - - - - - - below are only for implCompressMultiBlock0 - - - - - -
6018 // c_rarg0: int offset, when (multi_block == true)
6019 //
6020 address generate_sha1_implCompress(StubId stub_id) {
6021 bool multi_block;
6022 switch (stub_id) {
6023 case StubId::stubgen_sha1_implCompress_id:
6024 multi_block = false;
6025 break;
6026 case StubId::stubgen_sha1_implCompressMB_id:
6027 multi_block = true;
6028 break;
6029 default:
6030 ShouldNotReachHere();
6031 };
6032 __ align(CodeEntryAlignment);
6033 StubCodeMark mark(this, stub_id);
6034
6035 address start = __ pc();
6036 __ enter();
6037
6038 RegSet saved_regs = RegSet::range(x18, x27);
6039 if (multi_block) {
6040 // use x9 as src below.
6041 saved_regs += RegSet::of(x9);
6042 }
6043 __ push_reg(saved_regs, sp);
6044
6045 // c_rarg0 - c_rarg3: x10 - x13
6046 Register buf = c_rarg0;
6047 Register state = c_rarg1;
6048 Register offset = c_rarg2;
6049 Register limit = c_rarg3;
6050 // use src to contain the original start point of the array.
6051 Register src = x9;
6052
6053 if (multi_block) {
6054 __ sub(limit, limit, offset);
6055 __ add(limit, limit, buf);
6056 __ sub(src, buf, offset);
6057 }
6058
6059 // [args-reg]: x14 - x17
6060 // [temp-reg]: x28 - x31
6061 // [saved-reg]: x18 - x27
6062
6063 // h0/1/2/3/4
6064 const Register a = x14, b = x15, c = x16, d = x17, e = x28;
6065 // w0, w1, ... w15
6066 // put two adjecent w's in one register:
6067 // one at high word part, another at low word part
6068 // at different round (even or odd), w't value reside in different items in ws[].
6069 // w0 ~ w15, either reside in
6070 // ws[0] ~ ws[7], where
6071 // w0 at higher 32 bits of ws[0],
6072 // w1 at lower 32 bits of ws[0],
6073 // ...
6074 // w14 at higher 32 bits of ws[7],
6075 // w15 at lower 32 bits of ws[7].
6076 // or, reside in
6077 // w0: ws[0]'s lower 32 bits
6078 // w1 ~ w14: ws[1] ~ ws[7]
6079 // w15: ws[8]'s higher 32 bits
6080 Register ws[9] = {x29, x30, x31, x18,
6081 x19, x20, x21, x22,
6082 x23}; // auxiliary register for calculating w's value
6083 // current k't's value
6084 const Register cur_k = x24;
6085 // current w't's value
6086 const Register cur_w = x25;
6087 // values of a, b, c, d, e in the previous round
6088 const Register prev_ab = x26, prev_cd = x27;
6089 const Register prev_e = offset; // reuse offset/c_rarg2
6090
6091 // load 5 words state into a, b, c, d, e.
6092 //
6093 // To minimize the number of memory operations, we apply following
6094 // optimization: read the states (a/b/c/d) of 4-byte values in pairs,
6095 // with a single ld, and split them into 2 registers.
6096 //
6097 // And, as the core algorithm of SHA-1 works on 32-bits words, so
6098 // in the following code, it does not care about the content of
6099 // higher 32-bits in a/b/c/d/e. Based on this observation,
6100 // we can apply further optimization, which is to just ignore the
6101 // higher 32-bits in a/c/e, rather than set the higher
6102 // 32-bits of a/c/e to zero explicitly with extra instructions.
6103 __ ld(a, Address(state, 0));
6104 __ srli(b, a, 32);
6105 __ ld(c, Address(state, 8));
6106 __ srli(d, c, 32);
6107 __ lw(e, Address(state, 16));
6108
6109 Label L_sha1_loop;
6110 if (multi_block) {
6111 __ BIND(L_sha1_loop);
6112 }
6113
6114 sha1_preserve_prev_abcde(a, b, c, d, e, prev_ab, prev_cd, prev_e);
6115
6116 for (int round = 0; round < 80; round++) {
6117 // prepare K't value
6118 sha1_prepare_k(cur_k, round);
6119
6120 // prepare W't value
6121 sha1_prepare_w(cur_w, ws, buf, round);
6122
6123 // one round process
6124 sha1_process_round(a, b, c, d, e, cur_k, cur_w, t2, round);
6125 }
6126
6127 // compute the intermediate hash value
6128 sha1_calculate_im_hash(a, b, c, d, e, prev_ab, prev_cd, prev_e);
6129
6130 if (multi_block) {
6131 int64_t block_bytes = 16 * 4;
6132 __ addi(buf, buf, block_bytes);
6133
6134 __ bge(limit, buf, L_sha1_loop, /* is_far */ true);
6135 }
6136
6137 // store back the state.
6138 __ zext(a, a, 32);
6139 __ slli(b, b, 32);
6140 __ orr(a, a, b);
6141 __ sd(a, Address(state, 0));
6142 __ zext(c, c, 32);
6143 __ slli(d, d, 32);
6144 __ orr(c, c, d);
6145 __ sd(c, Address(state, 8));
6146 __ sw(e, Address(state, 16));
6147
6148 // return offset
6149 if (multi_block) {
6150 __ sub(c_rarg0, buf, src);
6151 }
6152
6153 __ pop_reg(saved_regs, sp);
6154
6155 __ leave();
6156 __ ret();
6157
6158 return (address) start;
6159 }
6160
6161 /**
6162 * vector registers:
6163 * input VectorRegister's: intputV1-V3, for m2 they could be v2, v4, v6, for m1 they could be v1, v2, v3
6164 * index VectorRegister's: idxV1-V4, for m2 they could be v8, v10, v12, v14, for m1 they could be v4, v5, v6, v7
6165 * output VectorRegister's: outputV1-V4, for m2 they could be v16, v18, v20, v22, for m1 they could be v8, v9, v10, v11
6166 *
6167 * NOTE: each field will occupy a vector register group
6168 */
6169 void base64_vector_encode_round(Register src, Register dst, Register codec,
6170 Register size, Register stepSrc, Register stepDst,
6171 VectorRegister inputV1, VectorRegister inputV2, VectorRegister inputV3,
6172 VectorRegister idxV1, VectorRegister idxV2, VectorRegister idxV3, VectorRegister idxV4,
6173 VectorRegister outputV1, VectorRegister outputV2, VectorRegister outputV3, VectorRegister outputV4,
6174 Assembler::LMUL lmul) {
6175 // set vector register type/len
6176 __ vsetvli(x0, size, Assembler::e8, lmul);
6177
6178 // segmented load src into v registers: mem(src) => vr(3)
6179 __ vlseg3e8_v(inputV1, src);
6180
6181 // src = src + register_group_len_bytes * 3
6182 __ add(src, src, stepSrc);
6183
6184 // encoding
6185 // 1. compute index into lookup table: vr(3) => vr(4)
6186 __ vsrl_vi(idxV1, inputV1, 2);
6187
6188 __ vsrl_vi(idxV2, inputV2, 2);
6189 __ vsll_vi(inputV1, inputV1, 6);
6190 __ vor_vv(idxV2, idxV2, inputV1);
6191 __ vsrl_vi(idxV2, idxV2, 2);
6192
6193 __ vsrl_vi(idxV3, inputV3, 4);
6194 __ vsll_vi(inputV2, inputV2, 4);
6195 __ vor_vv(idxV3, inputV2, idxV3);
6196 __ vsrl_vi(idxV3, idxV3, 2);
6197
6198 __ vsll_vi(idxV4, inputV3, 2);
6199 __ vsrl_vi(idxV4, idxV4, 2);
6200
6201 // 2. indexed load: vr(4) => vr(4)
6202 __ vluxei8_v(outputV1, codec, idxV1);
6203 __ vluxei8_v(outputV2, codec, idxV2);
6204 __ vluxei8_v(outputV3, codec, idxV3);
6205 __ vluxei8_v(outputV4, codec, idxV4);
6206
6207 // segmented store encoded data in v registers back to dst: vr(4) => mem(dst)
6208 __ vsseg4e8_v(outputV1, dst);
6209
6210 // dst = dst + register_group_len_bytes * 4
6211 __ add(dst, dst, stepDst);
6212 }
6213
6214 /**
6215 * void j.u.Base64.Encoder.encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL)
6216 *
6217 * Input arguments:
6218 * c_rarg0 - src, source array
6219 * c_rarg1 - sp, src start offset
6220 * c_rarg2 - sl, src end offset
6221 * c_rarg3 - dst, dest array
6222 * c_rarg4 - dp, dst start offset
6223 * c_rarg5 - isURL, Base64 or URL character set
6224 */
6225 address generate_base64_encodeBlock() {
6226 alignas(64) static const char toBase64[64] = {
6227 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
6228 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
6229 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
6230 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
6231 '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
6232 };
6233
6234 alignas(64) static const char toBase64URL[64] = {
6235 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
6236 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
6237 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
6238 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
6239 '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'
6240 };
6241
6242 __ align(CodeEntryAlignment);
6243 StubId stub_id = StubId::stubgen_base64_encodeBlock_id;
6244 StubCodeMark mark(this, stub_id);
6245 address start = __ pc();
6246 __ enter();
6247
6248 Register src = c_rarg0;
6249 Register soff = c_rarg1;
6250 Register send = c_rarg2;
6251 Register dst = c_rarg3;
6252 Register doff = c_rarg4;
6253 Register isURL = c_rarg5;
6254
6255 Register codec = c_rarg6;
6256 Register length = c_rarg7; // total length of src data in bytes
6257
6258 Label ProcessData, Exit;
6259
6260 // length should be multiple of 3
6261 __ sub(length, send, soff);
6262 // real src/dst to process data
6263 __ add(src, src, soff);
6264 __ add(dst, dst, doff);
6265
6266 // load the codec base address
6267 __ la(codec, ExternalAddress((address) toBase64));
6268 __ beqz(isURL, ProcessData);
6269 __ la(codec, ExternalAddress((address) toBase64URL));
6270 __ BIND(ProcessData);
6271
6272 // vector version
6273 if (UseRVV) {
6274 Label ProcessM2, ProcessM1, ProcessScalar;
6275
6276 Register size = soff;
6277 Register stepSrcM1 = send;
6278 Register stepSrcM2 = doff;
6279 Register stepDst = isURL;
6280
6281 __ mv(size, MaxVectorSize * 2);
6282 __ mv(stepSrcM1, MaxVectorSize * 3);
6283 __ slli(stepSrcM2, stepSrcM1, 1);
6284 __ mv(stepDst, MaxVectorSize * 2 * 4);
6285
6286 __ blt(length, stepSrcM2, ProcessM1);
6287
6288 __ BIND(ProcessM2);
6289 base64_vector_encode_round(src, dst, codec,
6290 size, stepSrcM2, stepDst,
6291 v2, v4, v6, // inputs
6292 v8, v10, v12, v14, // indexes
6293 v16, v18, v20, v22, // outputs
6294 Assembler::m2);
6295
6296 __ sub(length, length, stepSrcM2);
6297 __ bge(length, stepSrcM2, ProcessM2);
6298
6299 __ BIND(ProcessM1);
6300 __ blt(length, stepSrcM1, ProcessScalar);
6301
6302 __ srli(size, size, 1);
6303 __ srli(stepDst, stepDst, 1);
6304 base64_vector_encode_round(src, dst, codec,
6305 size, stepSrcM1, stepDst,
6306 v1, v2, v3, // inputs
6307 v4, v5, v6, v7, // indexes
6308 v8, v9, v10, v11, // outputs
6309 Assembler::m1);
6310 __ sub(length, length, stepSrcM1);
6311
6312 __ BIND(ProcessScalar);
6313 }
6314
6315 // scalar version
6316 {
6317 Register byte1 = soff, byte0 = send, byte2 = doff;
6318 Register combined24Bits = isURL;
6319
6320 __ beqz(length, Exit);
6321
6322 Label ScalarLoop;
6323 __ BIND(ScalarLoop);
6324 {
6325 // plain: [byte0[7:0] : byte1[7:0] : byte2[7:0]] =>
6326 // encoded: [byte0[7:2] : byte0[1:0]+byte1[7:4] : byte1[3:0]+byte2[7:6] : byte2[5:0]]
6327
6328 // load 3 bytes src data
6329 __ lbu(byte0, Address(src, 0));
6330 __ lbu(byte1, Address(src, 1));
6331 __ lbu(byte2, Address(src, 2));
6332 __ addi(src, src, 3);
6333
6334 // construct 24 bits from 3 bytes
6335 __ slliw(byte0, byte0, 16);
6336 __ slliw(byte1, byte1, 8);
6337 __ orr(combined24Bits, byte0, byte1);
6338 __ orr(combined24Bits, combined24Bits, byte2);
6339
6340 // get codec index and encode(ie. load from codec by index)
6341 __ slliw(byte0, combined24Bits, 8);
6342 __ srliw(byte0, byte0, 26);
6343 __ add(byte0, codec, byte0);
6344 __ lbu(byte0, byte0);
6345
6346 __ slliw(byte1, combined24Bits, 14);
6347 __ srliw(byte1, byte1, 26);
6348 __ add(byte1, codec, byte1);
6349 __ lbu(byte1, byte1);
6350
6351 __ slliw(byte2, combined24Bits, 20);
6352 __ srliw(byte2, byte2, 26);
6353 __ add(byte2, codec, byte2);
6354 __ lbu(byte2, byte2);
6355
6356 __ andi(combined24Bits, combined24Bits, 0x3f);
6357 __ add(combined24Bits, codec, combined24Bits);
6358 __ lbu(combined24Bits, combined24Bits);
6359
6360 // store 4 bytes encoded data
6361 __ sb(byte0, Address(dst, 0));
6362 __ sb(byte1, Address(dst, 1));
6363 __ sb(byte2, Address(dst, 2));
6364 __ sb(combined24Bits, Address(dst, 3));
6365
6366 __ subi(length, length, 3);
6367 __ addi(dst, dst, 4);
6368 // loop back
6369 __ bnez(length, ScalarLoop);
6370 }
6371 }
6372
6373 __ BIND(Exit);
6374
6375 __ leave();
6376 __ ret();
6377
6378 return (address) start;
6379 }
6380
6381 /**
6382 * vector registers:
6383 * input VectorRegister's: intputV1-V4, for m2 they could be v2, v4, v6, for m1 they could be v2, v4, v6, v8
6384 * index VectorRegister's: idxV1-V3, for m2 they could be v8, v10, v12, v14, for m1 they could be v10, v12, v14, v16
6385 * output VectorRegister's: outputV1-V4, for m2 they could be v16, v18, v20, v22, for m1 they could be v18, v20, v22
6386 *
6387 * NOTE: each field will occupy a single vector register group
6388 */
6389 void base64_vector_decode_round(Register src, Register dst, Register codec,
6390 Register size, Register stepSrc, Register stepDst, Register failedIdx,
6391 VectorRegister inputV1, VectorRegister inputV2, VectorRegister inputV3, VectorRegister inputV4,
6392 VectorRegister idxV1, VectorRegister idxV2, VectorRegister idxV3, VectorRegister idxV4,
6393 VectorRegister outputV1, VectorRegister outputV2, VectorRegister outputV3,
6394 Assembler::LMUL lmul) {
6395 // set vector register type/len
6396 __ vsetvli(x0, size, Assembler::e8, lmul, Assembler::ma, Assembler::ta);
6397
6398 // segmented load src into v registers: mem(src) => vr(4)
6399 __ vlseg4e8_v(inputV1, src);
6400
6401 // src = src + register_group_len_bytes * 4
6402 __ add(src, src, stepSrc);
6403
6404 // decoding
6405 // 1. indexed load: vr(4) => vr(4)
6406 __ vluxei8_v(idxV1, codec, inputV1);
6407 __ vluxei8_v(idxV2, codec, inputV2);
6408 __ vluxei8_v(idxV3, codec, inputV3);
6409 __ vluxei8_v(idxV4, codec, inputV4);
6410
6411 // 2. check wrong data
6412 __ vor_vv(outputV1, idxV1, idxV2);
6413 __ vor_vv(outputV2, idxV3, idxV4);
6414 __ vor_vv(outputV1, outputV1, outputV2);
6415 __ vmseq_vi(v0, outputV1, -1);
6416 __ vfirst_m(failedIdx, v0);
6417 Label NoFailure, FailureAtIdx0;
6418 // valid value can only be -1 when < 0
6419 __ bltz(failedIdx, NoFailure);
6420 // when the first data (at index 0) fails, no need to process data anymore
6421 __ beqz(failedIdx, FailureAtIdx0);
6422 __ vsetvli(x0, failedIdx, Assembler::e8, lmul, Assembler::mu, Assembler::tu);
6423 __ slli(stepDst, failedIdx, 1);
6424 __ add(stepDst, failedIdx, stepDst);
6425 __ BIND(NoFailure);
6426
6427 // 3. compute the decoded data: vr(4) => vr(3)
6428 __ vsll_vi(idxV1, idxV1, 2);
6429 __ vsrl_vi(outputV1, idxV2, 4);
6430 __ vor_vv(outputV1, outputV1, idxV1);
6431
6432 __ vsll_vi(idxV2, idxV2, 4);
6433 __ vsrl_vi(outputV2, idxV3, 2);
6434 __ vor_vv(outputV2, outputV2, idxV2);
6435
6436 __ vsll_vi(idxV3, idxV3, 6);
6437 __ vor_vv(outputV3, idxV4, idxV3);
6438
6439 // segmented store encoded data in v registers back to dst: vr(3) => mem(dst)
6440 __ vsseg3e8_v(outputV1, dst);
6441
6442 // dst = dst + register_group_len_bytes * 3
6443 __ add(dst, dst, stepDst);
6444 __ BIND(FailureAtIdx0);
6445 }
6446
6447 /**
6448 * int j.u.Base64.Decoder.decodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL, boolean isMIME)
6449 *
6450 * Input arguments:
6451 * c_rarg0 - src, source array
6452 * c_rarg1 - sp, src start offset
6453 * c_rarg2 - sl, src end offset
6454 * c_rarg3 - dst, dest array
6455 * c_rarg4 - dp, dst start offset
6456 * c_rarg5 - isURL, Base64 or URL character set
6457 * c_rarg6 - isMIME, Decoding MIME block
6458 */
6459 address generate_base64_decodeBlock() {
6460
6461 static const uint8_t fromBase64[256] = {
6462 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6463 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6464 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u, 255u, 63u,
6465 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
6466 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
6467 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 255u,
6468 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
6469 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
6470 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6471 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6472 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6473 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6474 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6475 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6476 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6477 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6478 };
6479
6480 static const uint8_t fromBase64URL[256] = {
6481 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6482 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6483 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 62u, 255u, 255u,
6484 52u, 53u, 54u, 55u, 56u, 57u, 58u, 59u, 60u, 61u, 255u, 255u, 255u, 255u, 255u, 255u,
6485 255u, 0u, 1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u,
6486 15u, 16u, 17u, 18u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 255u, 255u, 255u, 255u, 63u,
6487 255u, 26u, 27u, 28u, 29u, 30u, 31u, 32u, 33u, 34u, 35u, 36u, 37u, 38u, 39u, 40u,
6488 41u, 42u, 43u, 44u, 45u, 46u, 47u, 48u, 49u, 50u, 51u, 255u, 255u, 255u, 255u, 255u,
6489 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6490 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6491 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6492 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6493 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6494 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6495 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6496 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6497 };
6498
6499 __ align(CodeEntryAlignment);
6500 StubId stub_id = StubId::stubgen_base64_decodeBlock_id;
6501 StubCodeMark mark(this, stub_id);
6502 address start = __ pc();
6503 __ enter();
6504
6505 Register src = c_rarg0;
6506 Register soff = c_rarg1;
6507 Register send = c_rarg2;
6508 Register dst = c_rarg3;
6509 Register doff = c_rarg4;
6510 Register isURL = c_rarg5;
6511 Register isMIME = c_rarg6;
6512
6513 Register codec = c_rarg7;
6514 Register dstBackup = t6;
6515 Register length = t3; // total length of src data in bytes
6516
6517 Label ProcessData, Exit;
6518 Label ProcessScalar, ScalarLoop;
6519
6520 // passed in length (send - soff) is guaranteed to be > 4,
6521 // and in this intrinsic we only process data of length in multiple of 4,
6522 // it's not guaranteed to be multiple of 4 by java level, so do it explicitly
6523 __ sub(length, send, soff);
6524 __ andi(length, length, -4);
6525 // real src/dst to process data
6526 __ add(src, src, soff);
6527 __ add(dst, dst, doff);
6528 // backup of dst, used to calculate the return value at exit
6529 __ mv(dstBackup, dst);
6530
6531 // load the codec base address
6532 __ la(codec, ExternalAddress((address) fromBase64));
6533 __ beqz(isURL, ProcessData);
6534 __ la(codec, ExternalAddress((address) fromBase64URL));
6535 __ BIND(ProcessData);
6536
6537 // vector version
6538 if (UseRVV) {
6539 // for MIME case, it has a default length limit of 76 which could be
6540 // different(smaller) from (send - soff), so in MIME case, we go through
6541 // the scalar code path directly.
6542 __ bnez(isMIME, ScalarLoop);
6543
6544 Label ProcessM1, ProcessM2;
6545
6546 Register failedIdx = soff;
6547 Register stepSrcM1 = send;
6548 Register stepSrcM2 = doff;
6549 Register stepDst = isURL;
6550 Register size = t4;
6551
6552 __ mv(size, MaxVectorSize * 2);
6553 __ mv(stepSrcM1, MaxVectorSize * 4);
6554 __ slli(stepSrcM2, stepSrcM1, 1);
6555 __ mv(stepDst, MaxVectorSize * 2 * 3);
6556
6557 __ blt(length, stepSrcM2, ProcessM1);
6558
6559
6560 // Assembler::m2
6561 __ BIND(ProcessM2);
6562 base64_vector_decode_round(src, dst, codec,
6563 size, stepSrcM2, stepDst, failedIdx,
6564 v2, v4, v6, v8, // inputs
6565 v10, v12, v14, v16, // indexes
6566 v18, v20, v22, // outputs
6567 Assembler::m2);
6568 __ sub(length, length, stepSrcM2);
6569
6570 // error check
6571 // valid value of failedIdx can only be -1 when < 0
6572 __ bgez(failedIdx, Exit);
6573
6574 __ bge(length, stepSrcM2, ProcessM2);
6575
6576
6577 // Assembler::m1
6578 __ BIND(ProcessM1);
6579 __ blt(length, stepSrcM1, ProcessScalar);
6580
6581 __ srli(size, size, 1);
6582 __ srli(stepDst, stepDst, 1);
6583 base64_vector_decode_round(src, dst, codec,
6584 size, stepSrcM1, stepDst, failedIdx,
6585 v1, v2, v3, v4, // inputs
6586 v5, v6, v7, v8, // indexes
6587 v9, v10, v11, // outputs
6588 Assembler::m1);
6589 __ sub(length, length, stepSrcM1);
6590
6591 // error check
6592 // valid value of failedIdx can only be -1 when < 0
6593 __ bgez(failedIdx, Exit);
6594
6595 __ BIND(ProcessScalar);
6596 __ beqz(length, Exit);
6597 }
6598
6599 // scalar version
6600 {
6601 Register byte0 = soff, byte1 = send, byte2 = doff, byte3 = isURL;
6602 Register combined32Bits = t4;
6603
6604 // encoded: [byte0[5:0] : byte1[5:0] : byte2[5:0]] : byte3[5:0]] =>
6605 // plain: [byte0[5:0]+byte1[5:4] : byte1[3:0]+byte2[5:2] : byte2[1:0]+byte3[5:0]]
6606 __ BIND(ScalarLoop);
6607
6608 // load 4 bytes encoded src data
6609 __ lbu(byte0, Address(src, 0));
6610 __ lbu(byte1, Address(src, 1));
6611 __ lbu(byte2, Address(src, 2));
6612 __ lbu(byte3, Address(src, 3));
6613 __ addi(src, src, 4);
6614
6615 // get codec index and decode (ie. load from codec by index)
6616 __ add(byte0, codec, byte0);
6617 __ add(byte1, codec, byte1);
6618 __ lb(byte0, Address(byte0, 0));
6619 __ lb(byte1, Address(byte1, 0));
6620 __ add(byte2, codec, byte2);
6621 __ add(byte3, codec, byte3);
6622 __ lb(byte2, Address(byte2, 0));
6623 __ lb(byte3, Address(byte3, 0));
6624 __ slliw(byte0, byte0, 18);
6625 __ slliw(byte1, byte1, 12);
6626 __ orr(byte0, byte0, byte1);
6627 __ orr(byte0, byte0, byte3);
6628 __ slliw(byte2, byte2, 6);
6629 // For performance consideration, `combined32Bits` is constructed for 2 purposes at the same time,
6630 // 1. error check below
6631 // 2. decode below
6632 __ orr(combined32Bits, byte0, byte2);
6633
6634 // error check
6635 __ bltz(combined32Bits, Exit);
6636
6637 // store 3 bytes decoded data
6638 __ sraiw(byte0, combined32Bits, 16);
6639 __ sraiw(byte1, combined32Bits, 8);
6640 __ sb(byte0, Address(dst, 0));
6641 __ sb(byte1, Address(dst, 1));
6642 __ sb(combined32Bits, Address(dst, 2));
6643
6644 __ subi(length, length, 4);
6645 __ addi(dst, dst, 3);
6646 // loop back
6647 __ bnez(length, ScalarLoop);
6648 }
6649
6650 __ BIND(Exit);
6651 __ sub(c_rarg0, dst, dstBackup);
6652
6653 __ leave();
6654 __ ret();
6655
6656 return (address) start;
6657 }
6658
6659 void adler32_process_bytes(Register buff, Register s1, Register s2, VectorRegister vtable,
6660 VectorRegister vzero, VectorRegister vbytes, VectorRegister vs1acc, VectorRegister vs2acc,
6661 Register temp0, Register temp1, Register temp2, Register temp3,
6662 VectorRegister vtemp1, VectorRegister vtemp2, int step, Assembler::LMUL lmul) {
6663
6664 assert((lmul == Assembler::m4 && step == 64) ||
6665 (lmul == Assembler::m2 && step == 32) ||
6666 (lmul == Assembler::m1 && step == 16),
6667 "LMUL should be aligned with step: m4 and 64, m2 and 32 or m1 and 16");
6668 // Below is function for calculating Adler32 checksum with 64-, 32- or 16-byte step. LMUL=m4, m2 or m1 is used.
6669 // The results are in v12, v13, ..., v22, v23. Example below is for 64-byte step case.
6670 // We use b1, b2, ..., b64 to denote the 64 bytes loaded in each iteration.
6671 // In non-vectorized code, we update s1 and s2 as:
6672 // s1 <- s1 + b1
6673 // s2 <- s2 + s1
6674 // s1 <- s1 + b2
6675 // s2 <- s2 + b1
6676 // ...
6677 // s1 <- s1 + b64
6678 // s2 <- s2 + s1
6679 // Putting above assignments together, we have:
6680 // s1_new = s1 + b1 + b2 + ... + b64
6681 // s2_new = s2 + (s1 + b1) + (s1 + b1 + b2) + ... + (s1 + b1 + b2 + ... + b64) =
6682 // = s2 + s1 * 64 + (b1 * 64 + b2 * 63 + ... + b64 * 1) =
6683 // = s2 + s1 * 64 + (b1, b2, ... b64) dot (64, 63, ... 1)
6684
6685 __ mv(temp3, step);
6686 // Load data
6687 __ vsetvli(temp0, temp3, Assembler::e8, lmul);
6688 __ vle8_v(vbytes, buff);
6689 __ addi(buff, buff, step);
6690
6691 // Upper bound reduction sum for s1_new:
6692 // 0xFF * 64 = 0x3FC0, so:
6693 // 1. Need to do vector-widening reduction sum
6694 // 2. It is safe to perform sign-extension during vmv.x.s with 16-bits elements
6695 __ vwredsumu_vs(vs1acc, vbytes, vzero);
6696 // Multiplication for s2_new
6697 __ vwmulu_vv(vs2acc, vtable, vbytes);
6698
6699 // s2 = s2 + s1 * log2(step)
6700 __ slli(temp1, s1, exact_log2(step));
6701 __ add(s2, s2, temp1);
6702
6703 // Summing up calculated results for s2_new
6704 if (MaxVectorSize > 16) {
6705 __ vsetvli(temp0, temp3, Assembler::e16, lmul);
6706 } else {
6707 // Half of vector-widening multiplication result is in successor of vs2acc
6708 // group for vlen == 16, in which case we need to double vector register
6709 // group width in order to reduction sum all of them
6710 Assembler::LMUL lmulx2 = (lmul == Assembler::m1) ? Assembler::m2 :
6711 (lmul == Assembler::m2) ? Assembler::m4 : Assembler::m8;
6712 __ vsetvli(temp0, temp3, Assembler::e16, lmulx2);
6713 }
6714 // Upper bound for reduction sum:
6715 // 0xFF * (64 + 63 + ... + 2 + 1) = 0x817E0 max for whole register group, so:
6716 // 1. Need to do vector-widening reduction sum
6717 // 2. It is safe to perform sign-extension during vmv.x.s with 32-bits elements
6718 __ vwredsumu_vs(vtemp1, vs2acc, vzero);
6719
6720 // Extracting results for:
6721 // s1_new
6722 __ vmv_x_s(temp0, vs1acc);
6723 __ add(s1, s1, temp0);
6724 // s2_new
6725 __ vsetvli(temp0, temp3, Assembler::e32, Assembler::m1);
6726 __ vmv_x_s(temp1, vtemp1);
6727 __ add(s2, s2, temp1);
6728 }
6729
6730 /***
6731 * int java.util.zip.Adler32.updateBytes(int adler, byte[] b, int off, int len)
6732 *
6733 * Arguments:
6734 *
6735 * Inputs:
6736 * c_rarg0 - int adler
6737 * c_rarg1 - byte* buff (b + off)
6738 * c_rarg2 - int len
6739 *
6740 * Output:
6741 * c_rarg0 - int adler result
6742 */
6743 address generate_updateBytesAdler32() {
6744 __ align(CodeEntryAlignment);
6745 StubId stub_id = StubId::stubgen_updateBytesAdler32_id;
6746 StubCodeMark mark(this, stub_id);
6747 address start = __ pc();
6748
6749 Label L_nmax, L_nmax_loop, L_nmax_loop_entry, L_by16, L_by16_loop,
6750 L_by16_loop_unroll, L_by1_loop, L_do_mod, L_combine, L_by1;
6751
6752 // Aliases
6753 Register adler = c_rarg0;
6754 Register s1 = c_rarg0;
6755 Register s2 = c_rarg3;
6756 Register buff = c_rarg1;
6757 Register len = c_rarg2;
6758 Register nmax = c_rarg4;
6759 Register base = c_rarg5;
6760 Register count = c_rarg6;
6761 Register temp0 = t3;
6762 Register temp1 = t4;
6763 Register temp2 = t5;
6764 Register temp3 = t6;
6765
6766 VectorRegister vzero = v31;
6767 VectorRegister vbytes = v8; // group: v8, v9, v10, v11
6768 VectorRegister vs1acc = v12; // group: v12, v13, v14, v15
6769 VectorRegister vs2acc = v16; // group: v16, v17, v18, v19, v20, v21, v22, v23
6770 VectorRegister vtable_64 = v24; // group: v24, v25, v26, v27
6771 VectorRegister vtable_32 = v4; // group: v4, v5
6772 VectorRegister vtable_16 = v30;
6773 VectorRegister vtemp1 = v28;
6774 VectorRegister vtemp2 = v29;
6775
6776 // Max number of bytes we can process before having to take the mod
6777 // 0x15B0 is 5552 in decimal, the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
6778 const uint64_t BASE = 0xfff1;
6779 const uint64_t NMAX = 0x15B0;
6780
6781 // Loops steps
6782 int step_64 = 64;
6783 int step_32 = 32;
6784 int step_16 = 16;
6785 int step_1 = 1;
6786
6787 __ enter(); // Required for proper stackwalking of RuntimeStub frame
6788 __ mv(temp1, 64);
6789 __ vsetvli(temp0, temp1, Assembler::e8, Assembler::m4);
6790
6791 // Generating accumulation coefficients for further calculations
6792 // vtable_64:
6793 __ vid_v(vtemp1);
6794 __ vrsub_vx(vtable_64, vtemp1, temp1);
6795 // vtable_64 group now contains { 0x40, 0x3f, 0x3e, ..., 0x3, 0x2, 0x1 }
6796
6797 // vtable_32:
6798 __ mv(temp1, 32);
6799 __ vsetvli(temp0, temp1, Assembler::e8, Assembler::m2);
6800 __ vid_v(vtemp1);
6801 __ vrsub_vx(vtable_32, vtemp1, temp1);
6802 // vtable_32 group now contains { 0x20, 0x1f, 0x1e, ..., 0x3, 0x2, 0x1 }
6803
6804 __ vsetivli(temp0, 16, Assembler::e8, Assembler::m1);
6805 // vtable_16:
6806 __ mv(temp1, 16);
6807 __ vid_v(vtemp1);
6808 __ vrsub_vx(vtable_16, vtemp1, temp1);
6809 // vtable_16 now contains { 0x10, 0xf, 0xe, ..., 0x3, 0x2, 0x1 }
6810
6811 __ vmv_v_i(vzero, 0);
6812
6813 __ mv(base, BASE);
6814 __ mv(nmax, NMAX);
6815
6816 // s1 is initialized to the lower 16 bits of adler
6817 // s2 is initialized to the upper 16 bits of adler
6818 __ srliw(s2, adler, 16); // s2 = ((adler >> 16) & 0xffff)
6819 __ zext(s1, adler, 16); // s1 = (adler & 0xffff)
6820
6821 // The pipelined loop needs at least 16 elements for 1 iteration
6822 // It does check this, but it is more effective to skip to the cleanup loop
6823 __ mv(temp0, step_16);
6824 __ bgeu(len, temp0, L_nmax);
6825 __ beqz(len, L_combine);
6826
6827 // Jumping to L_by1_loop
6828 __ subi(len, len, step_1);
6829 __ j(L_by1_loop);
6830
6831 __ bind(L_nmax);
6832 __ sub(len, len, nmax);
6833 __ subi(count, nmax, 16);
6834 __ bltz(len, L_by16);
6835
6836 // Align L_nmax loop by 64
6837 __ bind(L_nmax_loop_entry);
6838 __ subi(count, count, 32);
6839
6840 __ bind(L_nmax_loop);
6841 adler32_process_bytes(buff, s1, s2, vtable_64, vzero,
6842 vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3,
6843 vtemp1, vtemp2, step_64, Assembler::m4);
6844 __ subi(count, count, step_64);
6845 __ bgtz(count, L_nmax_loop);
6846
6847 // There are three iterations left to do
6848 adler32_process_bytes(buff, s1, s2, vtable_32, vzero,
6849 vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3,
6850 vtemp1, vtemp2, step_32, Assembler::m2);
6851 adler32_process_bytes(buff, s1, s2, vtable_16, vzero,
6852 vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3,
6853 vtemp1, vtemp2, step_16, Assembler::m1);
6854
6855 // s1 = s1 % BASE
6856 __ remuw(s1, s1, base);
6857 // s2 = s2 % BASE
6858 __ remuw(s2, s2, base);
6859
6860 __ sub(len, len, nmax);
6861 __ subi(count, nmax, 16);
6862 __ bgez(len, L_nmax_loop_entry);
6863
6864 __ bind(L_by16);
6865 __ add(len, len, count);
6866 __ bltz(len, L_by1);
6867 // Trying to unroll
6868 __ mv(temp3, step_64);
6869 __ blt(len, temp3, L_by16_loop);
6870
6871 __ bind(L_by16_loop_unroll);
6872 adler32_process_bytes(buff, s1, s2, vtable_64, vzero,
6873 vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3,
6874 vtemp1, vtemp2, step_64, Assembler::m4);
6875 __ subi(len, len, step_64);
6876 // By now the temp3 should still be 64
6877 __ bge(len, temp3, L_by16_loop_unroll);
6878
6879 __ bind(L_by16_loop);
6880 adler32_process_bytes(buff, s1, s2, vtable_16, vzero,
6881 vbytes, vs1acc, vs2acc, temp0, temp1, temp2, temp3,
6882 vtemp1, vtemp2, step_16, Assembler::m1);
6883 __ subi(len, len, step_16);
6884 __ bgez(len, L_by16_loop);
6885
6886 __ bind(L_by1);
6887 __ addi(len, len, 15);
6888 __ bltz(len, L_do_mod);
6889
6890 __ bind(L_by1_loop);
6891 __ lbu(temp0, Address(buff, 0));
6892 __ addi(buff, buff, step_1);
6893 __ add(s1, temp0, s1);
6894 __ add(s2, s2, s1);
6895 __ subi(len, len, step_1);
6896 __ bgez(len, L_by1_loop);
6897
6898 __ bind(L_do_mod);
6899 // s1 = s1 % BASE
6900 __ remuw(s1, s1, base);
6901 // s2 = s2 % BASE
6902 __ remuw(s2, s2, base);
6903
6904 // Combine lower bits and higher bits
6905 // adler = s1 | (s2 << 16)
6906 __ bind(L_combine);
6907 __ slli(s2, s2, 16);
6908 __ orr(s1, s1, s2);
6909
6910 __ leave(); // Required for proper stackwalking of RuntimeStub frame
6911 __ ret();
6912
6913 return start;
6914 }
6915
6916 #endif // COMPILER2
6917
6918 // x10 = input (float16)
6919 // f10 = result (float)
6920 // t1 = temporary register
6921 address generate_float16ToFloat() {
6922 __ align(CodeEntryAlignment);
6923 StubId stub_id = StubId::stubgen_hf2f_id;
6924 StubCodeMark mark(this, stub_id);
6925 address entry = __ pc();
6926 BLOCK_COMMENT("float16ToFloat:");
6927
6928 FloatRegister dst = f10;
6929 Register src = x10;
6930 Label NaN_SLOW;
6931
6932 assert(VM_Version::supports_float16_float_conversion(), "must");
6933
6934 // On riscv, NaN needs a special process as fcvt does not work in that case.
6935 // On riscv, Inf does not need a special process as fcvt can handle it correctly.
6936 // but we consider to get the slow path to process NaN and Inf at the same time,
6937 // as both of them are rare cases, and if we try to get the slow path to handle
6938 // only NaN case it would sacrifise the performance for normal cases,
6939 // i.e. non-NaN and non-Inf cases.
6940
6941 // check whether it's a NaN or +/- Inf.
6942 __ mv(t0, 0x7c00);
6943 __ andr(t1, src, t0);
6944 // jump to stub processing NaN and Inf cases.
6945 __ beq(t0, t1, NaN_SLOW);
6946
6947 // non-NaN or non-Inf cases, just use built-in instructions.
6948 __ fmv_h_x(dst, src);
6949 __ fcvt_s_h(dst, dst);
6950 __ ret();
6951
6952 __ bind(NaN_SLOW);
6953 // following instructions mainly focus on NaN, as riscv does not handle
6954 // NaN well with fcvt, but the code also works for Inf at the same time.
6955
6956 // construct a NaN in 32 bits from the NaN in 16 bits,
6957 // we need the payloads of non-canonical NaNs to be preserved.
6958 __ mv(t1, 0x7f800000);
6959 // sign-bit was already set via sign-extension if necessary.
6960 __ slli(t0, src, 13);
6961 __ orr(t1, t0, t1);
6962 __ fmv_w_x(dst, t1);
6963
6964 __ ret();
6965 return entry;
6966 }
6967
6968 // f10 = input (float)
6969 // x10 = result (float16)
6970 // f11 = temporary float register
6971 // t1 = temporary register
6972 address generate_floatToFloat16() {
6973 __ align(CodeEntryAlignment);
6974 StubId stub_id = StubId::stubgen_f2hf_id;
6975 StubCodeMark mark(this, stub_id);
6976 address entry = __ pc();
6977 BLOCK_COMMENT("floatToFloat16:");
6978
6979 Register dst = x10;
6980 FloatRegister src = f10, ftmp = f11;
6981 Label NaN_SLOW;
6982
6983 assert(VM_Version::supports_float16_float_conversion(), "must");
6984
6985 // On riscv, NaN needs a special process as fcvt does not work in that case.
6986
6987 // check whether it's a NaN.
6988 // replace fclass with feq as performance optimization.
6989 __ feq_s(t0, src, src);
6990 // jump to stub processing NaN cases.
6991 __ beqz(t0, NaN_SLOW);
6992
6993 // non-NaN cases, just use built-in instructions.
6994 __ fcvt_h_s(ftmp, src);
6995 __ fmv_x_h(dst, ftmp);
6996 __ ret();
6997
6998 __ bind(NaN_SLOW);
6999
7000 __ float_to_float16_NaN(dst, src, t0, t1);
7001
7002 __ ret();
7003 return entry;
7004 }
7005
7006 #ifdef COMPILER2
7007
7008 static const int64_t right_2_bits = right_n_bits(2);
7009 static const int64_t right_3_bits = right_n_bits(3);
7010
7011 // In sun.security.util.math.intpoly.IntegerPolynomial1305, integers
7012 // are represented as long[5], with BITS_PER_LIMB = 26.
7013 // Pack five 26-bit limbs into three 64-bit registers.
7014 void poly1305_pack_26(Register dest0, Register dest1, Register dest2, Register src, Register tmp1, Register tmp2) {
7015 assert_different_registers(dest0, dest1, dest2, src, tmp1, tmp2);
7016
7017 // The goal is to have 128-bit value in dest2:dest1:dest0
7018 __ ld(dest0, Address(src, 0)); // 26 bits in dest0
7019
7020 __ ld(tmp1, Address(src, sizeof(jlong)));
7021 __ slli(tmp1, tmp1, 26);
7022 __ add(dest0, dest0, tmp1); // 52 bits in dest0
7023
7024 __ ld(tmp2, Address(src, 2 * sizeof(jlong)));
7025 __ slli(tmp1, tmp2, 52);
7026 __ add(dest0, dest0, tmp1); // dest0 is full
7027
7028 __ srli(dest1, tmp2, 12); // 14-bit in dest1
7029
7030 __ ld(tmp1, Address(src, 3 * sizeof(jlong)));
7031 __ slli(tmp1, tmp1, 14);
7032 __ add(dest1, dest1, tmp1); // 40-bit in dest1
7033
7034 __ ld(tmp1, Address(src, 4 * sizeof(jlong)));
7035 __ slli(tmp2, tmp1, 40);
7036 __ add(dest1, dest1, tmp2); // dest1 is full
7037
7038 if (dest2->is_valid()) {
7039 __ srli(tmp1, tmp1, 24);
7040 __ mv(dest2, tmp1); // 2 bits in dest2
7041 } else {
7042 #ifdef ASSERT
7043 Label OK;
7044 __ srli(tmp1, tmp1, 24);
7045 __ beq(zr, tmp1, OK); // 2 bits
7046 __ stop("high bits of Poly1305 integer should be zero");
7047 __ should_not_reach_here();
7048 __ bind(OK);
7049 #endif
7050 }
7051 }
7052
7053 // As above, but return only a 128-bit integer, packed into two
7054 // 64-bit registers.
7055 void poly1305_pack_26(Register dest0, Register dest1, Register src, Register tmp1, Register tmp2) {
7056 poly1305_pack_26(dest0, dest1, noreg, src, tmp1, tmp2);
7057 }
7058
7059 // U_2:U_1:U_0: += (U_2 >> 2) * 5
7060 void poly1305_reduce(Register U_2, Register U_1, Register U_0, Register tmp1, Register tmp2) {
7061 assert_different_registers(U_2, U_1, U_0, tmp1, tmp2);
7062
7063 // First, U_2:U_1:U_0 += (U_2 >> 2)
7064 __ srli(tmp1, U_2, 2);
7065 __ cad(U_0, U_0, tmp1, tmp2); // Add tmp1 to U_0 with carry output to tmp2
7066 __ andi(U_2, U_2, right_2_bits); // Clear U_2 except for the lowest two bits
7067 __ cad(U_1, U_1, tmp2, tmp2); // Add carry to U_1 with carry output to tmp2
7068 __ add(U_2, U_2, tmp2);
7069
7070 // Second, U_2:U_1:U_0 += (U_2 >> 2) << 2
7071 __ slli(tmp1, tmp1, 2);
7072 __ cad(U_0, U_0, tmp1, tmp2); // Add tmp1 to U_0 with carry output to tmp2
7073 __ cad(U_1, U_1, tmp2, tmp2); // Add carry to U_1 with carry output to tmp2
7074 __ add(U_2, U_2, tmp2);
7075 }
7076
7077 // Poly1305, RFC 7539
7078 // void com.sun.crypto.provider.Poly1305.processMultipleBlocks(byte[] input, int offset, int length, long[] aLimbs, long[] rLimbs)
7079
7080 // Arguments:
7081 // c_rarg0: input_start -- where the input is stored
7082 // c_rarg1: length
7083 // c_rarg2: acc_start -- where the output will be stored
7084 // c_rarg3: r_start -- where the randomly generated 128-bit key is stored
7085
7086 // See https://loup-vaillant.fr/tutorials/poly1305-design for a
7087 // description of the tricks used to simplify and accelerate this
7088 // computation.
7089
7090 address generate_poly1305_processBlocks() {
7091 __ align(CodeEntryAlignment);
7092 StubId stub_id = StubId::stubgen_poly1305_processBlocks_id;
7093 StubCodeMark mark(this, stub_id);
7094 address start = __ pc();
7095 __ enter();
7096 Label here;
7097
7098 RegSet saved_regs = RegSet::range(x18, x21);
7099 RegSetIterator<Register> regs = (RegSet::range(x14, x31) - RegSet::range(x22, x27)).begin();
7100 __ push_reg(saved_regs, sp);
7101
7102 // Arguments
7103 const Register input_start = c_rarg0, length = c_rarg1, acc_start = c_rarg2, r_start = c_rarg3;
7104
7105 // R_n is the 128-bit randomly-generated key, packed into two
7106 // registers. The caller passes this key to us as long[5], with
7107 // BITS_PER_LIMB = 26.
7108 const Register R_0 = *regs, R_1 = *++regs;
7109 poly1305_pack_26(R_0, R_1, r_start, t1, t2);
7110
7111 // RR_n is (R_n >> 2) * 5
7112 const Register RR_0 = *++regs, RR_1 = *++regs;
7113 __ srli(t1, R_0, 2);
7114 __ shadd(RR_0, t1, t1, t2, 2);
7115 __ srli(t1, R_1, 2);
7116 __ shadd(RR_1, t1, t1, t2, 2);
7117
7118 // U_n is the current checksum
7119 const Register U_0 = *++regs, U_1 = *++regs, U_2 = *++regs;
7120 poly1305_pack_26(U_0, U_1, U_2, acc_start, t1, t2);
7121
7122 static constexpr int BLOCK_LENGTH = 16;
7123 Label DONE, LOOP;
7124
7125 __ mv(t1, BLOCK_LENGTH);
7126 __ blt(length, t1, DONE); {
7127 __ bind(LOOP);
7128
7129 // S_n is to be the sum of U_n and the next block of data
7130 const Register S_0 = *++regs, S_1 = *++regs, S_2 = *++regs;
7131 __ ld(S_0, Address(input_start, 0));
7132 __ ld(S_1, Address(input_start, wordSize));
7133
7134 __ cad(S_0, S_0, U_0, t1); // Add U_0 to S_0 with carry output to t1
7135 __ cadc(S_1, S_1, U_1, t1); // Add U_1 with carry to S_1 with carry output to t1
7136 __ add(S_2, U_2, t1);
7137
7138 __ addi(S_2, S_2, 1);
7139
7140 const Register U_0HI = *++regs, U_1HI = *++regs;
7141
7142 // NB: this logic depends on some of the special properties of
7143 // Poly1305 keys. In particular, because we know that the top
7144 // four bits of R_0 and R_1 are zero, we can add together
7145 // partial products without any risk of needing to propagate a
7146 // carry out.
7147 __ wide_mul(U_0, U_0HI, S_0, R_0);
7148 __ wide_madd(U_0, U_0HI, S_1, RR_1, t1, t2);
7149 __ wide_madd(U_0, U_0HI, S_2, RR_0, t1, t2);
7150
7151 __ wide_mul(U_1, U_1HI, S_0, R_1);
7152 __ wide_madd(U_1, U_1HI, S_1, R_0, t1, t2);
7153 __ wide_madd(U_1, U_1HI, S_2, RR_1, t1, t2);
7154
7155 __ andi(U_2, R_0, right_2_bits);
7156 __ mul(U_2, S_2, U_2);
7157
7158 // Partial reduction mod 2**130 - 5
7159 __ cad(U_1, U_1, U_0HI, t1); // Add U_0HI to U_1 with carry output to t1
7160 __ adc(U_2, U_2, U_1HI, t1);
7161 // Sum is now in U_2:U_1:U_0.
7162
7163 // U_2:U_1:U_0: += (U_2 >> 2) * 5
7164 poly1305_reduce(U_2, U_1, U_0, t1, t2);
7165
7166 __ subi(length, length, BLOCK_LENGTH);
7167 __ addi(input_start, input_start, BLOCK_LENGTH);
7168 __ mv(t1, BLOCK_LENGTH);
7169 __ bge(length, t1, LOOP);
7170 }
7171
7172 // Further reduce modulo 2^130 - 5
7173 poly1305_reduce(U_2, U_1, U_0, t1, t2);
7174
7175 // Unpack the sum into five 26-bit limbs and write to memory.
7176 // First 26 bits is the first limb
7177 __ slli(t1, U_0, 38); // Take lowest 26 bits
7178 __ srli(t1, t1, 38);
7179 __ sd(t1, Address(acc_start)); // First 26-bit limb
7180
7181 // 27-52 bits of U_0 is the second limb
7182 __ slli(t1, U_0, 12); // Take next 27-52 bits
7183 __ srli(t1, t1, 38);
7184 __ sd(t1, Address(acc_start, sizeof (jlong))); // Second 26-bit limb
7185
7186 // Getting 53-64 bits of U_0 and 1-14 bits of U_1 in one register
7187 __ srli(t1, U_0, 52);
7188 __ slli(t2, U_1, 50);
7189 __ srli(t2, t2, 38);
7190 __ add(t1, t1, t2);
7191 __ sd(t1, Address(acc_start, 2 * sizeof (jlong))); // Third 26-bit limb
7192
7193 // Storing 15-40 bits of U_1
7194 __ slli(t1, U_1, 24); // Already used up 14 bits
7195 __ srli(t1, t1, 38); // Clear all other bits from t1
7196 __ sd(t1, Address(acc_start, 3 * sizeof (jlong))); // Fourth 26-bit limb
7197
7198 // Storing 41-64 bits of U_1 and first three bits from U_2 in one register
7199 __ srli(t1, U_1, 40);
7200 __ andi(t2, U_2, right_3_bits);
7201 __ slli(t2, t2, 24);
7202 __ add(t1, t1, t2);
7203 __ sd(t1, Address(acc_start, 4 * sizeof (jlong))); // Fifth 26-bit limb
7204
7205 __ bind(DONE);
7206 __ pop_reg(saved_regs, sp);
7207 __ leave(); // Required for proper stackwalking
7208 __ ret();
7209
7210 return start;
7211 }
7212
7213 address generate_arrays_hashcode_powers_of_31() {
7214 assert(UseRVV, "sanity");
7215 const int lmul = 2;
7216 const int stride = MaxVectorSize / sizeof(jint) * lmul;
7217 __ align(CodeEntryAlignment);
7218 StubCodeMark mark(this, "StubRoutines", "arrays_hashcode_powers_of_31");
7219 address start = __ pc();
7220 for (int i = stride; i >= 0; i--) {
7221 jint power_of_31 = 1;
7222 for (int j = i; j > 0; j--) {
7223 power_of_31 = java_multiply(power_of_31, 31);
7224 }
7225 __ emit_int32(power_of_31);
7226 }
7227
7228 return start;
7229 }
7230
7231 #endif // COMPILER2
7232
7233 /**
7234 * Arguments:
7235 *
7236 * Inputs:
7237 * c_rarg0 - int crc
7238 * c_rarg1 - byte* buf
7239 * c_rarg2 - int length
7240 *
7241 * Output:
7242 * c_rarg0 - int crc result
7243 */
7244 address generate_updateBytesCRC32() {
7245 assert(UseCRC32Intrinsics, "what are we doing here?");
7246
7247 __ align(CodeEntryAlignment);
7248 StubId stub_id = StubId::stubgen_updateBytesCRC32_id;
7249 StubCodeMark mark(this, stub_id);
7250
7251 address start = __ pc();
7252
7253 // input parameters
7254 const Register crc = c_rarg0; // crc
7255 const Register buf = c_rarg1; // source java byte array address
7256 const Register len = c_rarg2; // length
7257
7258 BLOCK_COMMENT("Entry:");
7259 __ enter(); // required for proper stackwalking of RuntimeStub frame
7260
7261 __ kernel_crc32(crc, buf, len,
7262 c_rarg3, c_rarg4, c_rarg5, c_rarg6, // tmp's for tables
7263 c_rarg7, t2, t3, t4, t5, t6); // misc tmps
7264
7265 __ leave(); // required for proper stackwalking of RuntimeStub frame
7266 __ ret();
7267
7268 return start;
7269 }
7270
7271 // exception handler for upcall stubs
7272 address generate_upcall_stub_exception_handler() {
7273 StubId stub_id = StubId::stubgen_upcall_stub_exception_handler_id;
7274 StubCodeMark mark(this, stub_id);
7275 address start = __ pc();
7276
7277 // Native caller has no idea how to handle exceptions,
7278 // so we just crash here. Up to callee to catch exceptions.
7279 __ verify_oop(x10); // return a exception oop in a0
7280 __ rt_call(CAST_FROM_FN_PTR(address, UpcallLinker::handle_uncaught_exception));
7281 __ should_not_reach_here();
7282
7283 return start;
7284 }
7285
7286 // load Method* target of MethodHandle
7287 // j_rarg0 = jobject receiver
7288 // xmethod = Method* result
7289 address generate_upcall_stub_load_target() {
7290
7291 StubId stub_id = StubId::stubgen_upcall_stub_load_target_id;
7292 StubCodeMark mark(this, stub_id);
7293 address start = __ pc();
7294
7295 __ resolve_global_jobject(j_rarg0, t0, t1);
7296 // Load target method from receiver
7297 __ load_heap_oop(xmethod, Address(j_rarg0, java_lang_invoke_MethodHandle::form_offset()), t0, t1);
7298 __ load_heap_oop(xmethod, Address(xmethod, java_lang_invoke_LambdaForm::vmentry_offset()), t0, t1);
7299 __ load_heap_oop(xmethod, Address(xmethod, java_lang_invoke_MemberName::method_offset()), t0, t1);
7300 __ access_load_at(T_ADDRESS, IN_HEAP, xmethod,
7301 Address(xmethod, java_lang_invoke_ResolvedMethodName::vmtarget_offset()),
7302 noreg, noreg);
7303 __ sd(xmethod, Address(xthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
7304
7305 __ ret();
7306
7307 return start;
7308 }
7309
7310 #undef __
7311
7312 // Initialization
7313 void generate_preuniverse_stubs() {
7314 // preuniverse stubs are not needed for riscv
7315 }
7316
7317 void generate_initial_stubs() {
7318 // Generate initial stubs and initializes the entry points
7319
7320 // entry points that exist in all platforms Note: This is code
7321 // that could be shared among different platforms - however the
7322 // benefit seems to be smaller than the disadvantage of having a
7323 // much more complicated generator structure. See also comment in
7324 // stubRoutines.hpp.
7325
7326 StubRoutines::_forward_exception_entry = generate_forward_exception();
7327
7328 if (UnsafeMemoryAccess::_table == nullptr) {
7329 UnsafeMemoryAccess::create_table(8 + 4); // 8 for copyMemory; 4 for setMemory
7330 }
7331
7332 StubRoutines::_call_stub_entry =
7333 generate_call_stub(StubRoutines::_call_stub_return_address);
7334
7335 // is referenced by megamorphic call
7336 StubRoutines::_catch_exception_entry = generate_catch_exception();
7337
7338 if (UseCRC32Intrinsics) {
7339 StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
7340 }
7341
7342 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
7343 vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
7344 StubRoutines::_hf2f = generate_float16ToFloat();
7345 StubRoutines::_f2hf = generate_floatToFloat16();
7346 }
7347 }
7348
7349 void generate_continuation_stubs() {
7350 // Continuation stubs:
7351 StubRoutines::_cont_thaw = generate_cont_thaw();
7352 StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
7353 StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
7354 StubRoutines::_cont_preempt_stub = generate_cont_preempt_stub();
7355 }
7356
7357 void generate_final_stubs() {
7358 // support for verify_oop (must happen after universe_init)
7359 if (VerifyOops) {
7360 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
7361 }
7362
7363 // arraycopy stubs used by compilers
7364 generate_arraycopy_stubs();
7365
7366 StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
7367
7368 #ifdef COMPILER2
7369 if (UseSecondarySupersTable) {
7370 StubRoutines::_lookup_secondary_supers_table_slow_path_stub = generate_lookup_secondary_supers_table_slow_path_stub();
7371 if (!InlineSecondarySupersTest) {
7372 generate_lookup_secondary_supers_table_stub();
7373 }
7374 }
7375 #endif // COMPILER2
7376
7377 StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler();
7378 StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target();
7379
7380 StubRoutines::riscv::set_completed();
7381 }
7382
7383 void generate_compiler_stubs() {
7384 #ifdef COMPILER2
7385 if (UseMulAddIntrinsic) {
7386 StubRoutines::_mulAdd = generate_mulAdd();
7387 }
7388
7389 if (UseMultiplyToLenIntrinsic) {
7390 StubRoutines::_multiplyToLen = generate_multiplyToLen();
7391 }
7392
7393 if (UseSquareToLenIntrinsic) {
7394 StubRoutines::_squareToLen = generate_squareToLen();
7395 }
7396
7397 if (UseMontgomeryMultiplyIntrinsic) {
7398 StubId stub_id = StubId::stubgen_montgomeryMultiply_id;
7399 StubCodeMark mark(this, stub_id);
7400 MontgomeryMultiplyGenerator g(_masm, /*squaring*/false);
7401 StubRoutines::_montgomeryMultiply = g.generate_multiply();
7402 }
7403
7404 if (UseMontgomerySquareIntrinsic) {
7405 StubId stub_id = StubId::stubgen_montgomerySquare_id;
7406 StubCodeMark mark(this, stub_id);
7407 MontgomeryMultiplyGenerator g(_masm, /*squaring*/true);
7408 StubRoutines::_montgomerySquare = g.generate_square();
7409 }
7410
7411 if (UseAESIntrinsics) {
7412 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
7413 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
7414 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
7415 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt();
7416 }
7417
7418 if (UseAESCTRIntrinsics) {
7419 StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt();
7420 }
7421
7422 if (UseGHASHIntrinsics) {
7423 StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
7424 }
7425
7426 if (UseAESCTRIntrinsics && UseGHASHIntrinsics) {
7427 StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt();
7428 }
7429
7430 if (UsePoly1305Intrinsics) {
7431 StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks();
7432 }
7433
7434 if (UseRVV) {
7435 StubRoutines::_bigIntegerLeftShiftWorker = generate_bigIntegerLeftShift();
7436 StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift();
7437 }
7438
7439 if (UseVectorizedHashCodeIntrinsic && UseRVV) {
7440 StubRoutines::riscv::_arrays_hashcode_powers_of_31 = generate_arrays_hashcode_powers_of_31();
7441 }
7442
7443 if (UseSHA256Intrinsics) {
7444 Sha2Generator sha2(_masm, this);
7445 StubRoutines::_sha256_implCompress = sha2.generate_sha256_implCompress(StubId::stubgen_sha256_implCompress_id);
7446 StubRoutines::_sha256_implCompressMB = sha2.generate_sha256_implCompress(StubId::stubgen_sha256_implCompressMB_id);
7447 }
7448
7449 if (UseSHA512Intrinsics) {
7450 Sha2Generator sha2(_masm, this);
7451 StubRoutines::_sha512_implCompress = sha2.generate_sha512_implCompress(StubId::stubgen_sha512_implCompress_id);
7452 StubRoutines::_sha512_implCompressMB = sha2.generate_sha512_implCompress(StubId::stubgen_sha512_implCompressMB_id);
7453 }
7454
7455 if (UseMD5Intrinsics) {
7456 StubRoutines::_md5_implCompress = generate_md5_implCompress(StubId::stubgen_md5_implCompress_id);
7457 StubRoutines::_md5_implCompressMB = generate_md5_implCompress(StubId::stubgen_md5_implCompressMB_id);
7458 }
7459
7460 if (UseChaCha20Intrinsics) {
7461 StubRoutines::_chacha20Block = generate_chacha20Block();
7462 }
7463
7464 if (UseSHA1Intrinsics) {
7465 StubRoutines::_sha1_implCompress = generate_sha1_implCompress(StubId::stubgen_sha1_implCompress_id);
7466 StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(StubId::stubgen_sha1_implCompressMB_id);
7467 }
7468
7469 if (UseBASE64Intrinsics) {
7470 StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock();
7471 StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock();
7472 }
7473
7474 if (UseAdler32Intrinsics) {
7475 StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
7476 }
7477
7478 generate_compare_long_strings();
7479
7480 generate_string_indexof_stubs();
7481
7482 #endif // COMPILER2
7483 }
7484
7485 public:
7486 StubGenerator(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) : StubCodeGenerator(code, blob_id, stub_data) {
7487 switch(blob_id) {
7488 case BlobId::stubgen_preuniverse_id:
7489 generate_preuniverse_stubs();
7490 break;
7491 case BlobId::stubgen_initial_id:
7492 generate_initial_stubs();
7493 break;
7494 case BlobId::stubgen_continuation_id:
7495 generate_continuation_stubs();
7496 break;
7497 case BlobId::stubgen_compiler_id:
7498 generate_compiler_stubs();
7499 break;
7500 case BlobId::stubgen_final_id:
7501 generate_final_stubs();
7502 break;
7503 default:
7504 fatal("unexpected blob id: %s", StubInfo::name(blob_id));
7505 break;
7506 };
7507 }
7508 }; // end class declaration
7509
7510 void StubGenerator_generate(CodeBuffer* code, BlobId blob_id, AOTStubData* stub_data) {
7511 StubGenerator g(code, blob_id, stub_data);
7512 }