1 /*
   2  * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.hpp"
  28 #include "asm/macroAssembler.inline.hpp"
  29 #include "asm/register.hpp"
  30 #include "atomic_aarch64.hpp"
  31 #include "compiler/oopMap.hpp"
  32 #include "gc/shared/barrierSet.hpp"
  33 #include "gc/shared/barrierSetAssembler.hpp"
  34 #include "gc/shared/gc_globals.hpp"
  35 #include "gc/shared/tlab_globals.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "memory/universe.hpp"
  38 #include "nativeInst_aarch64.hpp"
  39 #include "oops/instanceOop.hpp"
  40 #include "oops/method.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "oops/oop.inline.hpp"
  43 #include "prims/methodHandles.hpp"
  44 #include "prims/upcallLinker.hpp"
  45 #include "runtime/atomic.hpp"
  46 #include "runtime/continuation.hpp"
  47 #include "runtime/continuationEntry.inline.hpp"
  48 #include "runtime/frame.inline.hpp"
  49 #include "runtime/handles.inline.hpp"
  50 #include "runtime/javaThread.hpp"
  51 #include "runtime/sharedRuntime.hpp"
  52 #include "runtime/stubCodeGenerator.hpp"
  53 #include "runtime/stubRoutines.hpp"
  54 #include "utilities/align.hpp"
  55 #include "utilities/checkedCast.hpp"
  56 #include "utilities/globalDefinitions.hpp"
  57 #include "utilities/powerOfTwo.hpp"
  58 #ifdef COMPILER2
  59 #include "opto/runtime.hpp"
  60 #endif
  61 #if INCLUDE_ZGC
  62 #include "gc/z/zThreadLocalData.hpp"
  63 #endif
  64 
  65 // Declaration and definition of StubGenerator (no .hpp file).
  66 // For a more detailed description of the stub routine structure
  67 // see the comment in stubRoutines.hpp
  68 
  69 #undef __
  70 #define __ _masm->
  71 
  72 #ifdef PRODUCT
  73 #define BLOCK_COMMENT(str) /* nothing */
  74 #else
  75 #define BLOCK_COMMENT(str) __ block_comment(str)
  76 #endif
  77 
  78 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  79 
  80 // Stub Code definitions
  81 
  82 class StubGenerator: public StubCodeGenerator {
  83  private:
  84 
  85 #ifdef PRODUCT
  86 #define inc_counter_np(counter) ((void)0)
  87 #else
  88   void inc_counter_np_(uint& counter) {
  89     __ incrementw(ExternalAddress((address)&counter));
  90   }
  91 #define inc_counter_np(counter) \
  92   BLOCK_COMMENT("inc_counter " #counter); \
  93   inc_counter_np_(counter);
  94 #endif
  95 
  96   // Call stubs are used to call Java from C
  97   //
  98   // Arguments:
  99   //    c_rarg0:   call wrapper address                   address
 100   //    c_rarg1:   result                                 address
 101   //    c_rarg2:   result type                            BasicType
 102   //    c_rarg3:   method                                 Method*
 103   //    c_rarg4:   (interpreter) entry point              address
 104   //    c_rarg5:   parameters                             intptr_t*
 105   //    c_rarg6:   parameter size (in words)              int
 106   //    c_rarg7:   thread                                 Thread*
 107   //
 108   // There is no return from the stub itself as any Java result
 109   // is written to result
 110   //
 111   // we save r30 (lr) as the return PC at the base of the frame and
 112   // link r29 (fp) below it as the frame pointer installing sp (r31)
 113   // into fp.
 114   //
 115   // we save r0-r7, which accounts for all the c arguments.
 116   //
 117   // TODO: strictly do we need to save them all? they are treated as
 118   // volatile by C so could we omit saving the ones we are going to
 119   // place in global registers (thread? method?) or those we only use
 120   // during setup of the Java call?
 121   //
 122   // we don't need to save r8 which C uses as an indirect result location
 123   // return register.
 124   //
 125   // we don't need to save r9-r15 which both C and Java treat as
 126   // volatile
 127   //
 128   // we don't need to save r16-18 because Java does not use them
 129   //
 130   // we save r19-r28 which Java uses as scratch registers and C
 131   // expects to be callee-save
 132   //
 133   // we save the bottom 64 bits of each value stored in v8-v15; it is
 134   // the responsibility of the caller to preserve larger values.
 135   //
 136   // so the stub frame looks like this when we enter Java code
 137   //
 138   //     [ return_from_Java     ] <--- sp
 139   //     [ argument word n      ]
 140   //      ...
 141   // -29 [ argument word 1      ]
 142   // -28 [ saved Floating-point Control Register ]
 143   // -26 [ saved v15            ] <--- sp_after_call
 144   // -25 [ saved v14            ]
 145   // -24 [ saved v13            ]
 146   // -23 [ saved v12            ]
 147   // -22 [ saved v11            ]
 148   // -21 [ saved v10            ]
 149   // -20 [ saved v9             ]
 150   // -19 [ saved v8             ]
 151   // -18 [ saved r28            ]
 152   // -17 [ saved r27            ]
 153   // -16 [ saved r26            ]
 154   // -15 [ saved r25            ]
 155   // -14 [ saved r24            ]
 156   // -13 [ saved r23            ]
 157   // -12 [ saved r22            ]
 158   // -11 [ saved r21            ]
 159   // -10 [ saved r20            ]
 160   //  -9 [ saved r19            ]
 161   //  -8 [ call wrapper    (r0) ]
 162   //  -7 [ result          (r1) ]
 163   //  -6 [ result type     (r2) ]
 164   //  -5 [ method          (r3) ]
 165   //  -4 [ entry point     (r4) ]
 166   //  -3 [ parameters      (r5) ]
 167   //  -2 [ parameter size  (r6) ]
 168   //  -1 [ thread (r7)          ]
 169   //   0 [ saved fp       (r29) ] <--- fp == saved sp (r31)
 170   //   1 [ saved lr       (r30) ]
 171 
 172   // Call stub stack layout word offsets from fp
 173   enum call_stub_layout {
 174     sp_after_call_off  = -28,
 175 
 176     fpcr_off           = sp_after_call_off,
 177     d15_off            = -26,
 178     d13_off            = -24,
 179     d11_off            = -22,
 180     d9_off             = -20,
 181 
 182     r28_off            = -18,
 183     r26_off            = -16,
 184     r24_off            = -14,
 185     r22_off            = -12,
 186     r20_off            = -10,
 187     call_wrapper_off   =  -8,
 188     result_off         =  -7,
 189     result_type_off    =  -6,
 190     method_off         =  -5,
 191     entry_point_off    =  -4,
 192     parameter_size_off =  -2,
 193     thread_off         =  -1,
 194     fp_f               =   0,
 195     retaddr_off        =   1,
 196   };
 197 
 198   address generate_call_stub(address& return_address) {
 199     assert((int)frame::entry_frame_after_call_words == -(int)sp_after_call_off + 1 &&
 200            (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
 201            "adjust this code");
 202 
 203     StubCodeMark mark(this, "StubRoutines", "call_stub");
 204     address start = __ pc();
 205 
 206     const Address sp_after_call (rfp, sp_after_call_off * wordSize);
 207 
 208     const Address fpcr_save     (rfp, fpcr_off           * wordSize);
 209     const Address call_wrapper  (rfp, call_wrapper_off   * wordSize);
 210     const Address result        (rfp, result_off         * wordSize);
 211     const Address result_type   (rfp, result_type_off    * wordSize);
 212     const Address method        (rfp, method_off         * wordSize);
 213     const Address entry_point   (rfp, entry_point_off    * wordSize);
 214     const Address parameter_size(rfp, parameter_size_off * wordSize);
 215 
 216     const Address thread        (rfp, thread_off         * wordSize);
 217 
 218     const Address d15_save      (rfp, d15_off * wordSize);
 219     const Address d13_save      (rfp, d13_off * wordSize);
 220     const Address d11_save      (rfp, d11_off * wordSize);
 221     const Address d9_save       (rfp, d9_off * wordSize);
 222 
 223     const Address r28_save      (rfp, r28_off * wordSize);
 224     const Address r26_save      (rfp, r26_off * wordSize);
 225     const Address r24_save      (rfp, r24_off * wordSize);
 226     const Address r22_save      (rfp, r22_off * wordSize);
 227     const Address r20_save      (rfp, r20_off * wordSize);
 228 
 229     // stub code
 230 
 231     address aarch64_entry = __ pc();
 232 
 233     // set up frame and move sp to end of save area
 234     __ enter();
 235     __ sub(sp, rfp, -sp_after_call_off * wordSize);
 236 
 237     // save register parameters and Java scratch/global registers
 238     // n.b. we save thread even though it gets installed in
 239     // rthread because we want to sanity check rthread later
 240     __ str(c_rarg7,  thread);
 241     __ strw(c_rarg6, parameter_size);
 242     __ stp(c_rarg4, c_rarg5,  entry_point);
 243     __ stp(c_rarg2, c_rarg3,  result_type);
 244     __ stp(c_rarg0, c_rarg1,  call_wrapper);
 245 
 246     __ stp(r20, r19,   r20_save);
 247     __ stp(r22, r21,   r22_save);
 248     __ stp(r24, r23,   r24_save);
 249     __ stp(r26, r25,   r26_save);
 250     __ stp(r28, r27,   r28_save);
 251 
 252     __ stpd(v9,  v8,   d9_save);
 253     __ stpd(v11, v10,  d11_save);
 254     __ stpd(v13, v12,  d13_save);
 255     __ stpd(v15, v14,  d15_save);
 256 
 257     __ get_fpcr(rscratch1);
 258     __ str(rscratch1, fpcr_save);
 259     // Set FPCR to the state we need. We do want Round to Nearest. We
 260     // don't want non-IEEE rounding modes or floating-point traps.
 261     __ bfi(rscratch1, zr, 22, 4); // Clear DN, FZ, and Rmode
 262     __ bfi(rscratch1, zr, 8, 5);  // Clear exception-control bits (8-12)
 263     __ set_fpcr(rscratch1);
 264 
 265     // install Java thread in global register now we have saved
 266     // whatever value it held
 267     __ mov(rthread, c_rarg7);
 268     // And method
 269     __ mov(rmethod, c_rarg3);
 270 
 271     // set up the heapbase register
 272     __ reinit_heapbase();
 273 
 274 #ifdef ASSERT
 275     // make sure we have no pending exceptions
 276     {
 277       Label L;
 278       __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
 279       __ cmp(rscratch1, (u1)NULL_WORD);
 280       __ br(Assembler::EQ, L);
 281       __ stop("StubRoutines::call_stub: entered with pending exception");
 282       __ BIND(L);
 283     }
 284 #endif
 285     // pass parameters if any
 286     __ mov(esp, sp);
 287     __ sub(rscratch1, sp, c_rarg6, ext::uxtw, LogBytesPerWord); // Move SP out of the way
 288     __ andr(sp, rscratch1, -2 * wordSize);
 289 
 290     BLOCK_COMMENT("pass parameters if any");
 291     Label parameters_done;
 292     // parameter count is still in c_rarg6
 293     // and parameter pointer identifying param 1 is in c_rarg5
 294     __ cbzw(c_rarg6, parameters_done);
 295 
 296     address loop = __ pc();
 297     __ ldr(rscratch1, Address(__ post(c_rarg5, wordSize)));
 298     __ subsw(c_rarg6, c_rarg6, 1);
 299     __ push(rscratch1);
 300     __ br(Assembler::GT, loop);
 301 
 302     __ BIND(parameters_done);
 303 
 304     // call Java entry -- passing methdoOop, and current sp
 305     //      rmethod: Method*
 306     //      r19_sender_sp: sender sp
 307     BLOCK_COMMENT("call Java function");
 308     __ mov(r19_sender_sp, sp);
 309     __ blr(c_rarg4);
 310 
 311     // we do this here because the notify will already have been done
 312     // if we get to the next instruction via an exception
 313     //
 314     // n.b. adding this instruction here affects the calculation of
 315     // whether or not a routine returns to the call stub (used when
 316     // doing stack walks) since the normal test is to check the return
 317     // pc against the address saved below. so we may need to allow for
 318     // this extra instruction in the check.
 319 
 320     // save current address for use by exception handling code
 321 
 322     return_address = __ pc();
 323 
 324     // store result depending on type (everything that is not
 325     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 326     // n.b. this assumes Java returns an integral result in r0
 327     // and a floating result in j_farg0
 328     __ ldr(j_rarg2, result);
 329     Label is_long, is_float, is_double, exit;
 330     __ ldr(j_rarg1, result_type);
 331     __ cmp(j_rarg1, (u1)T_OBJECT);
 332     __ br(Assembler::EQ, is_long);
 333     __ cmp(j_rarg1, (u1)T_LONG);
 334     __ br(Assembler::EQ, is_long);
 335     __ cmp(j_rarg1, (u1)T_FLOAT);
 336     __ br(Assembler::EQ, is_float);
 337     __ cmp(j_rarg1, (u1)T_DOUBLE);
 338     __ br(Assembler::EQ, is_double);
 339 
 340     // handle T_INT case
 341     __ strw(r0, Address(j_rarg2));
 342 
 343     __ BIND(exit);
 344 
 345     // pop parameters
 346     __ sub(esp, rfp, -sp_after_call_off * wordSize);
 347 
 348 #ifdef ASSERT
 349     // verify that threads correspond
 350     {
 351       Label L, S;
 352       __ ldr(rscratch1, thread);
 353       __ cmp(rthread, rscratch1);
 354       __ br(Assembler::NE, S);
 355       __ get_thread(rscratch1);
 356       __ cmp(rthread, rscratch1);
 357       __ br(Assembler::EQ, L);
 358       __ BIND(S);
 359       __ stop("StubRoutines::call_stub: threads must correspond");
 360       __ BIND(L);
 361     }
 362 #endif
 363 
 364     __ pop_cont_fastpath(rthread);
 365 
 366     // restore callee-save registers
 367     __ ldpd(v15, v14,  d15_save);
 368     __ ldpd(v13, v12,  d13_save);
 369     __ ldpd(v11, v10,  d11_save);
 370     __ ldpd(v9,  v8,   d9_save);
 371 
 372     __ ldp(r28, r27,   r28_save);
 373     __ ldp(r26, r25,   r26_save);
 374     __ ldp(r24, r23,   r24_save);
 375     __ ldp(r22, r21,   r22_save);
 376     __ ldp(r20, r19,   r20_save);
 377 
 378     // restore fpcr
 379     __ ldr(rscratch1,  fpcr_save);
 380     __ set_fpcr(rscratch1);
 381 
 382     __ ldp(c_rarg0, c_rarg1,  call_wrapper);
 383     __ ldrw(c_rarg2, result_type);
 384     __ ldr(c_rarg3,  method);
 385     __ ldp(c_rarg4, c_rarg5,  entry_point);
 386     __ ldp(c_rarg6, c_rarg7,  parameter_size);
 387 
 388     // leave frame and return to caller
 389     __ leave();
 390     __ ret(lr);
 391 
 392     // handle return types different from T_INT
 393 
 394     __ BIND(is_long);
 395     __ str(r0, Address(j_rarg2, 0));
 396     __ br(Assembler::AL, exit);
 397 
 398     __ BIND(is_float);
 399     __ strs(j_farg0, Address(j_rarg2, 0));
 400     __ br(Assembler::AL, exit);
 401 
 402     __ BIND(is_double);
 403     __ strd(j_farg0, Address(j_rarg2, 0));
 404     __ br(Assembler::AL, exit);
 405 
 406     return start;
 407   }
 408 
 409   // Return point for a Java call if there's an exception thrown in
 410   // Java code.  The exception is caught and transformed into a
 411   // pending exception stored in JavaThread that can be tested from
 412   // within the VM.
 413   //
 414   // Note: Usually the parameters are removed by the callee. In case
 415   // of an exception crossing an activation frame boundary, that is
 416   // not the case if the callee is compiled code => need to setup the
 417   // rsp.
 418   //
 419   // r0: exception oop
 420 
 421   address generate_catch_exception() {
 422     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 423     address start = __ pc();
 424 
 425     // same as in generate_call_stub():
 426     const Address sp_after_call(rfp, sp_after_call_off * wordSize);
 427     const Address thread        (rfp, thread_off         * wordSize);
 428 
 429 #ifdef ASSERT
 430     // verify that threads correspond
 431     {
 432       Label L, S;
 433       __ ldr(rscratch1, thread);
 434       __ cmp(rthread, rscratch1);
 435       __ br(Assembler::NE, S);
 436       __ get_thread(rscratch1);
 437       __ cmp(rthread, rscratch1);
 438       __ br(Assembler::EQ, L);
 439       __ bind(S);
 440       __ stop("StubRoutines::catch_exception: threads must correspond");
 441       __ bind(L);
 442     }
 443 #endif
 444 
 445     // set pending exception
 446     __ verify_oop(r0);
 447 
 448     __ str(r0, Address(rthread, Thread::pending_exception_offset()));
 449     __ mov(rscratch1, (address)__FILE__);
 450     __ str(rscratch1, Address(rthread, Thread::exception_file_offset()));
 451     __ movw(rscratch1, (int)__LINE__);
 452     __ strw(rscratch1, Address(rthread, Thread::exception_line_offset()));
 453 
 454     // complete return to VM
 455     assert(StubRoutines::_call_stub_return_address != nullptr,
 456            "_call_stub_return_address must have been generated before");
 457     __ b(StubRoutines::_call_stub_return_address);
 458 
 459     return start;
 460   }
 461 
 462   // Continuation point for runtime calls returning with a pending
 463   // exception.  The pending exception check happened in the runtime
 464   // or native call stub.  The pending exception in Thread is
 465   // converted into a Java-level exception.
 466   //
 467   // Contract with Java-level exception handlers:
 468   // r0: exception
 469   // r3: throwing pc
 470   //
 471   // NOTE: At entry of this stub, exception-pc must be in LR !!
 472 
 473   // NOTE: this is always used as a jump target within generated code
 474   // so it just needs to be generated code with no x86 prolog
 475 
 476   address generate_forward_exception() {
 477     StubCodeMark mark(this, "StubRoutines", "forward exception");
 478     address start = __ pc();
 479 
 480     // Upon entry, LR points to the return address returning into
 481     // Java (interpreted or compiled) code; i.e., the return address
 482     // becomes the throwing pc.
 483     //
 484     // Arguments pushed before the runtime call are still on the stack
 485     // but the exception handler will reset the stack pointer ->
 486     // ignore them.  A potential result in registers can be ignored as
 487     // well.
 488 
 489 #ifdef ASSERT
 490     // make sure this code is only executed if there is a pending exception
 491     {
 492       Label L;
 493       __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 494       __ cbnz(rscratch1, L);
 495       __ stop("StubRoutines::forward exception: no pending exception (1)");
 496       __ bind(L);
 497     }
 498 #endif
 499 
 500     // compute exception handler into r19
 501 
 502     // call the VM to find the handler address associated with the
 503     // caller address. pass thread in r0 and caller pc (ret address)
 504     // in r1. n.b. the caller pc is in lr, unlike x86 where it is on
 505     // the stack.
 506     __ mov(c_rarg1, lr);
 507     // lr will be trashed by the VM call so we move it to R19
 508     // (callee-saved) because we also need to pass it to the handler
 509     // returned by this call.
 510     __ mov(r19, lr);
 511     BLOCK_COMMENT("call exception_handler_for_return_address");
 512     __ call_VM_leaf(CAST_FROM_FN_PTR(address,
 513                          SharedRuntime::exception_handler_for_return_address),
 514                     rthread, c_rarg1);
 515     // Reinitialize the ptrue predicate register, in case the external runtime
 516     // call clobbers ptrue reg, as we may return to SVE compiled code.
 517     __ reinitialize_ptrue();
 518 
 519     // we should not really care that lr is no longer the callee
 520     // address. we saved the value the handler needs in r19 so we can
 521     // just copy it to r3. however, the C2 handler will push its own
 522     // frame and then calls into the VM and the VM code asserts that
 523     // the PC for the frame above the handler belongs to a compiled
 524     // Java method. So, we restore lr here to satisfy that assert.
 525     __ mov(lr, r19);
 526     // setup r0 & r3 & clear pending exception
 527     __ mov(r3, r19);
 528     __ mov(r19, r0);
 529     __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
 530     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 531 
 532 #ifdef ASSERT
 533     // make sure exception is set
 534     {
 535       Label L;
 536       __ cbnz(r0, L);
 537       __ stop("StubRoutines::forward exception: no pending exception (2)");
 538       __ bind(L);
 539     }
 540 #endif
 541 
 542     // continue at exception handler
 543     // r0: exception
 544     // r3: throwing pc
 545     // r19: exception handler
 546     __ verify_oop(r0);
 547     __ br(r19);
 548 
 549     return start;
 550   }
 551 
 552   // Non-destructive plausibility checks for oops
 553   //
 554   // Arguments:
 555   //    r0: oop to verify
 556   //    rscratch1: error message
 557   //
 558   // Stack after saving c_rarg3:
 559   //    [tos + 0]: saved c_rarg3
 560   //    [tos + 1]: saved c_rarg2
 561   //    [tos + 2]: saved lr
 562   //    [tos + 3]: saved rscratch2
 563   //    [tos + 4]: saved r0
 564   //    [tos + 5]: saved rscratch1
 565   address generate_verify_oop() {
 566 
 567     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 568     address start = __ pc();
 569 
 570     Label exit, error;
 571 
 572     // save c_rarg2 and c_rarg3
 573     __ stp(c_rarg3, c_rarg2, Address(__ pre(sp, -16)));
 574 
 575     // __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 576     __ lea(c_rarg2, ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 577     __ ldr(c_rarg3, Address(c_rarg2));
 578     __ add(c_rarg3, c_rarg3, 1);
 579     __ str(c_rarg3, Address(c_rarg2));
 580 
 581     // object is in r0
 582     // make sure object is 'reasonable'
 583     __ cbz(r0, exit); // if obj is null it is OK
 584 
 585     BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
 586     bs_asm->check_oop(_masm, r0, c_rarg2, c_rarg3, error);
 587 
 588     // return if everything seems ok
 589     __ bind(exit);
 590 
 591     __ ldp(c_rarg3, c_rarg2, Address(__ post(sp, 16)));
 592     __ ret(lr);
 593 
 594     // handle errors
 595     __ bind(error);
 596     __ ldp(c_rarg3, c_rarg2, Address(__ post(sp, 16)));
 597 
 598     __ push(RegSet::range(r0, r29), sp);
 599     // debug(char* msg, int64_t pc, int64_t regs[])
 600     __ mov(c_rarg0, rscratch1);      // pass address of error message
 601     __ mov(c_rarg1, lr);             // pass return address
 602     __ mov(c_rarg2, sp);             // pass address of regs on stack
 603 #ifndef PRODUCT
 604     assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 605 #endif
 606     BLOCK_COMMENT("call MacroAssembler::debug");
 607     __ mov(rscratch1, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
 608     __ blr(rscratch1);
 609     __ hlt(0);
 610 
 611     return start;
 612   }
 613 
 614   // Generate indices for iota vector.
 615   address generate_iota_indices(const char *stub_name) {
 616     __ align(CodeEntryAlignment);
 617     StubCodeMark mark(this, "StubRoutines", stub_name);
 618     address start = __ pc();
 619     // B
 620     __ emit_data64(0x0706050403020100, relocInfo::none);
 621     __ emit_data64(0x0F0E0D0C0B0A0908, relocInfo::none);
 622     // H
 623     __ emit_data64(0x0003000200010000, relocInfo::none);
 624     __ emit_data64(0x0007000600050004, relocInfo::none);
 625     // S
 626     __ emit_data64(0x0000000100000000, relocInfo::none);
 627     __ emit_data64(0x0000000300000002, relocInfo::none);
 628     // D
 629     __ emit_data64(0x0000000000000000, relocInfo::none);
 630     __ emit_data64(0x0000000000000001, relocInfo::none);
 631     // S - FP
 632     __ emit_data64(0x3F80000000000000, relocInfo::none); // 0.0f, 1.0f
 633     __ emit_data64(0x4040000040000000, relocInfo::none); // 2.0f, 3.0f
 634     // D - FP
 635     __ emit_data64(0x0000000000000000, relocInfo::none); // 0.0d
 636     __ emit_data64(0x3FF0000000000000, relocInfo::none); // 1.0d
 637     return start;
 638   }
 639 
 640   // The inner part of zero_words().  This is the bulk operation,
 641   // zeroing words in blocks, possibly using DC ZVA to do it.  The
 642   // caller is responsible for zeroing the last few words.
 643   //
 644   // Inputs:
 645   // r10: the HeapWord-aligned base address of an array to zero.
 646   // r11: the count in HeapWords, r11 > 0.
 647   //
 648   // Returns r10 and r11, adjusted for the caller to clear.
 649   // r10: the base address of the tail of words left to clear.
 650   // r11: the number of words in the tail.
 651   //      r11 < MacroAssembler::zero_words_block_size.
 652 
 653   address generate_zero_blocks() {
 654     Label done;
 655     Label base_aligned;
 656 
 657     Register base = r10, cnt = r11;
 658 
 659     __ align(CodeEntryAlignment);
 660     StubCodeMark mark(this, "StubRoutines", "zero_blocks");
 661     address start = __ pc();
 662 
 663     if (UseBlockZeroing) {
 664       int zva_length = VM_Version::zva_length();
 665 
 666       // Ensure ZVA length can be divided by 16. This is required by
 667       // the subsequent operations.
 668       assert (zva_length % 16 == 0, "Unexpected ZVA Length");
 669 
 670       __ tbz(base, 3, base_aligned);
 671       __ str(zr, Address(__ post(base, 8)));
 672       __ sub(cnt, cnt, 1);
 673       __ bind(base_aligned);
 674 
 675       // Ensure count >= zva_length * 2 so that it still deserves a zva after
 676       // alignment.
 677       Label small;
 678       int low_limit = MAX2(zva_length * 2, (int)BlockZeroingLowLimit);
 679       __ subs(rscratch1, cnt, low_limit >> 3);
 680       __ br(Assembler::LT, small);
 681       __ zero_dcache_blocks(base, cnt);
 682       __ bind(small);
 683     }
 684 
 685     {
 686       // Number of stp instructions we'll unroll
 687       const int unroll =
 688         MacroAssembler::zero_words_block_size / 2;
 689       // Clear the remaining blocks.
 690       Label loop;
 691       __ subs(cnt, cnt, unroll * 2);
 692       __ br(Assembler::LT, done);
 693       __ bind(loop);
 694       for (int i = 0; i < unroll; i++)
 695         __ stp(zr, zr, __ post(base, 16));
 696       __ subs(cnt, cnt, unroll * 2);
 697       __ br(Assembler::GE, loop);
 698       __ bind(done);
 699       __ add(cnt, cnt, unroll * 2);
 700     }
 701 
 702     __ ret(lr);
 703 
 704     return start;
 705   }
 706 
 707 
 708   typedef enum {
 709     copy_forwards = 1,
 710     copy_backwards = -1
 711   } copy_direction;
 712 
 713   // Helper object to reduce noise when telling the GC barriers how to perform loads and stores
 714   // for arraycopy stubs.
 715   class ArrayCopyBarrierSetHelper : StackObj {
 716     BarrierSetAssembler* _bs_asm;
 717     MacroAssembler* _masm;
 718     DecoratorSet _decorators;
 719     BasicType _type;
 720     Register _gct1;
 721     Register _gct2;
 722     Register _gct3;
 723     FloatRegister _gcvt1;
 724     FloatRegister _gcvt2;
 725     FloatRegister _gcvt3;
 726 
 727   public:
 728     ArrayCopyBarrierSetHelper(MacroAssembler* masm,
 729                               DecoratorSet decorators,
 730                               BasicType type,
 731                               Register gct1,
 732                               Register gct2,
 733                               Register gct3,
 734                               FloatRegister gcvt1,
 735                               FloatRegister gcvt2,
 736                               FloatRegister gcvt3)
 737       : _bs_asm(BarrierSet::barrier_set()->barrier_set_assembler()),
 738         _masm(masm),
 739         _decorators(decorators),
 740         _type(type),
 741         _gct1(gct1),
 742         _gct2(gct2),
 743         _gct3(gct3),
 744         _gcvt1(gcvt1),
 745         _gcvt2(gcvt2),
 746         _gcvt3(gcvt3) {
 747     }
 748 
 749     void copy_load_at_32(FloatRegister dst1, FloatRegister dst2, Address src) {
 750       _bs_asm->copy_load_at(_masm, _decorators, _type, 32,
 751                             dst1, dst2, src,
 752                             _gct1, _gct2, _gcvt1);
 753     }
 754 
 755     void copy_store_at_32(Address dst, FloatRegister src1, FloatRegister src2) {
 756       _bs_asm->copy_store_at(_masm, _decorators, _type, 32,
 757                              dst, src1, src2,
 758                              _gct1, _gct2, _gct3, _gcvt1, _gcvt2, _gcvt3);
 759     }
 760 
 761     void copy_load_at_16(Register dst1, Register dst2, Address src) {
 762       _bs_asm->copy_load_at(_masm, _decorators, _type, 16,
 763                             dst1, dst2, src,
 764                             _gct1);
 765     }
 766 
 767     void copy_store_at_16(Address dst, Register src1, Register src2) {
 768       _bs_asm->copy_store_at(_masm, _decorators, _type, 16,
 769                              dst, src1, src2,
 770                              _gct1, _gct2, _gct3);
 771     }
 772 
 773     void copy_load_at_8(Register dst, Address src) {
 774       _bs_asm->copy_load_at(_masm, _decorators, _type, 8,
 775                             dst, noreg, src,
 776                             _gct1);
 777     }
 778 
 779     void copy_store_at_8(Address dst, Register src) {
 780       _bs_asm->copy_store_at(_masm, _decorators, _type, 8,
 781                              dst, src, noreg,
 782                              _gct1, _gct2, _gct3);
 783     }
 784   };
 785 
 786   // Bulk copy of blocks of 8 words.
 787   //
 788   // count is a count of words.
 789   //
 790   // Precondition: count >= 8
 791   //
 792   // Postconditions:
 793   //
 794   // The least significant bit of count contains the remaining count
 795   // of words to copy.  The rest of count is trash.
 796   //
 797   // s and d are adjusted to point to the remaining words to copy
 798   //
 799   void generate_copy_longs(DecoratorSet decorators, BasicType type, Label &start, Register s, Register d, Register count,
 800                            copy_direction direction) {
 801     int unit = wordSize * direction;
 802     int bias = (UseSIMDForMemoryOps ? 4:2) * wordSize;
 803 
 804     const Register t0 = r3, t1 = r4, t2 = r5, t3 = r6,
 805       t4 = r7, t5 = r11, t6 = r12, t7 = r13;
 806     const Register stride = r14;
 807     const Register gct1 = rscratch1, gct2 = rscratch2, gct3 = r10;
 808     const FloatRegister gcvt1 = v6, gcvt2 = v7, gcvt3 = v16; // Note that v8-v15 are callee saved
 809     ArrayCopyBarrierSetHelper bs(_masm, decorators, type, gct1, gct2, gct3, gcvt1, gcvt2, gcvt3);
 810 
 811     assert_different_registers(rscratch1, rscratch2, t0, t1, t2, t3, t4, t5, t6, t7);
 812     assert_different_registers(s, d, count, rscratch1, rscratch2);
 813 
 814     Label again, drain;
 815     const char *stub_name;
 816     if (direction == copy_forwards)
 817       stub_name = "forward_copy_longs";
 818     else
 819       stub_name = "backward_copy_longs";
 820 
 821     __ align(CodeEntryAlignment);
 822 
 823     StubCodeMark mark(this, "StubRoutines", stub_name);
 824 
 825     __ bind(start);
 826 
 827     Label unaligned_copy_long;
 828     if (AvoidUnalignedAccesses) {
 829       __ tbnz(d, 3, unaligned_copy_long);
 830     }
 831 
 832     if (direction == copy_forwards) {
 833       __ sub(s, s, bias);
 834       __ sub(d, d, bias);
 835     }
 836 
 837 #ifdef ASSERT
 838     // Make sure we are never given < 8 words
 839     {
 840       Label L;
 841       __ cmp(count, (u1)8);
 842       __ br(Assembler::GE, L);
 843       __ stop("genrate_copy_longs called with < 8 words");
 844       __ bind(L);
 845     }
 846 #endif
 847 
 848     // Fill 8 registers
 849     if (UseSIMDForMemoryOps) {
 850       bs.copy_load_at_32(v0, v1, Address(s, 4 * unit));
 851       bs.copy_load_at_32(v2, v3, Address(__ pre(s, 8 * unit)));
 852     } else {
 853       bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
 854       bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
 855       bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
 856       bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
 857     }
 858 
 859     __ subs(count, count, 16);
 860     __ br(Assembler::LO, drain);
 861 
 862     int prefetch = PrefetchCopyIntervalInBytes;
 863     bool use_stride = false;
 864     if (direction == copy_backwards) {
 865        use_stride = prefetch > 256;
 866        prefetch = -prefetch;
 867        if (use_stride) __ mov(stride, prefetch);
 868     }
 869 
 870     __ bind(again);
 871 
 872     if (PrefetchCopyIntervalInBytes > 0)
 873       __ prfm(use_stride ? Address(s, stride) : Address(s, prefetch), PLDL1KEEP);
 874 
 875     if (UseSIMDForMemoryOps) {
 876       bs.copy_store_at_32(Address(d, 4 * unit), v0, v1);
 877       bs.copy_load_at_32(v0, v1, Address(s, 4 * unit));
 878       bs.copy_store_at_32(Address(__ pre(d, 8 * unit)), v2, v3);
 879       bs.copy_load_at_32(v2, v3, Address(__ pre(s, 8 * unit)));
 880     } else {
 881       bs.copy_store_at_16(Address(d, 2 * unit), t0, t1);
 882       bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
 883       bs.copy_store_at_16(Address(d, 4 * unit), t2, t3);
 884       bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
 885       bs.copy_store_at_16(Address(d, 6 * unit), t4, t5);
 886       bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
 887       bs.copy_store_at_16(Address(__ pre(d, 8 * unit)), t6, t7);
 888       bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
 889     }
 890 
 891     __ subs(count, count, 8);
 892     __ br(Assembler::HS, again);
 893 
 894     // Drain
 895     __ bind(drain);
 896     if (UseSIMDForMemoryOps) {
 897       bs.copy_store_at_32(Address(d, 4 * unit), v0, v1);
 898       bs.copy_store_at_32(Address(__ pre(d, 8 * unit)), v2, v3);
 899     } else {
 900       bs.copy_store_at_16(Address(d, 2 * unit), t0, t1);
 901       bs.copy_store_at_16(Address(d, 4 * unit), t2, t3);
 902       bs.copy_store_at_16(Address(d, 6 * unit), t4, t5);
 903       bs.copy_store_at_16(Address(__ pre(d, 8 * unit)), t6, t7);
 904     }
 905 
 906     {
 907       Label L1, L2;
 908       __ tbz(count, exact_log2(4), L1);
 909       if (UseSIMDForMemoryOps) {
 910         bs.copy_load_at_32(v0, v1, Address(__ pre(s, 4 * unit)));
 911         bs.copy_store_at_32(Address(__ pre(d, 4 * unit)), v0, v1);
 912       } else {
 913         bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
 914         bs.copy_load_at_16(t2, t3, Address(__ pre(s, 4 * unit)));
 915         bs.copy_store_at_16(Address(d, 2 * unit), t0, t1);
 916         bs.copy_store_at_16(Address(__ pre(d, 4 * unit)), t2, t3);
 917       }
 918       __ bind(L1);
 919 
 920       if (direction == copy_forwards) {
 921         __ add(s, s, bias);
 922         __ add(d, d, bias);
 923       }
 924 
 925       __ tbz(count, 1, L2);
 926       bs.copy_load_at_16(t0, t1, Address(__ adjust(s, 2 * unit, direction == copy_backwards)));
 927       bs.copy_store_at_16(Address(__ adjust(d, 2 * unit, direction == copy_backwards)), t0, t1);
 928       __ bind(L2);
 929     }
 930 
 931     __ ret(lr);
 932 
 933     if (AvoidUnalignedAccesses) {
 934       Label drain, again;
 935       // Register order for storing. Order is different for backward copy.
 936 
 937       __ bind(unaligned_copy_long);
 938 
 939       // source address is even aligned, target odd aligned
 940       //
 941       // when forward copying word pairs we read long pairs at offsets
 942       // {0, 2, 4, 6} (in long words). when backwards copying we read
 943       // long pairs at offsets {-2, -4, -6, -8}. We adjust the source
 944       // address by -2 in the forwards case so we can compute the
 945       // source offsets for both as {2, 4, 6, 8} * unit where unit = 1
 946       // or -1.
 947       //
 948       // when forward copying we need to store 1 word, 3 pairs and
 949       // then 1 word at offsets {0, 1, 3, 5, 7}. Rather than use a
 950       // zero offset We adjust the destination by -1 which means we
 951       // have to use offsets { 1, 2, 4, 6, 8} * unit for the stores.
 952       //
 953       // When backwards copyng we need to store 1 word, 3 pairs and
 954       // then 1 word at offsets {-1, -3, -5, -7, -8} i.e. we use
 955       // offsets {1, 3, 5, 7, 8} * unit.
 956 
 957       if (direction == copy_forwards) {
 958         __ sub(s, s, 16);
 959         __ sub(d, d, 8);
 960       }
 961 
 962       // Fill 8 registers
 963       //
 964       // for forwards copy s was offset by -16 from the original input
 965       // value of s so the register contents are at these offsets
 966       // relative to the 64 bit block addressed by that original input
 967       // and so on for each successive 64 byte block when s is updated
 968       //
 969       // t0 at offset 0,  t1 at offset 8
 970       // t2 at offset 16, t3 at offset 24
 971       // t4 at offset 32, t5 at offset 40
 972       // t6 at offset 48, t7 at offset 56
 973 
 974       // for backwards copy s was not offset so the register contents
 975       // are at these offsets into the preceding 64 byte block
 976       // relative to that original input and so on for each successive
 977       // preceding 64 byte block when s is updated. this explains the
 978       // slightly counter-intuitive looking pattern of register usage
 979       // in the stp instructions for backwards copy.
 980       //
 981       // t0 at offset -16, t1 at offset -8
 982       // t2 at offset -32, t3 at offset -24
 983       // t4 at offset -48, t5 at offset -40
 984       // t6 at offset -64, t7 at offset -56
 985 
 986       bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
 987       bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
 988       bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
 989       bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
 990 
 991       __ subs(count, count, 16);
 992       __ br(Assembler::LO, drain);
 993 
 994       int prefetch = PrefetchCopyIntervalInBytes;
 995       bool use_stride = false;
 996       if (direction == copy_backwards) {
 997          use_stride = prefetch > 256;
 998          prefetch = -prefetch;
 999          if (use_stride) __ mov(stride, prefetch);
1000       }
1001 
1002       __ bind(again);
1003 
1004       if (PrefetchCopyIntervalInBytes > 0)
1005         __ prfm(use_stride ? Address(s, stride) : Address(s, prefetch), PLDL1KEEP);
1006 
1007       if (direction == copy_forwards) {
1008        // allowing for the offset of -8 the store instructions place
1009        // registers into the target 64 bit block at the following
1010        // offsets
1011        //
1012        // t0 at offset 0
1013        // t1 at offset 8,  t2 at offset 16
1014        // t3 at offset 24, t4 at offset 32
1015        // t5 at offset 40, t6 at offset 48
1016        // t7 at offset 56
1017 
1018         bs.copy_store_at_8(Address(d, 1 * unit), t0);
1019         bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
1020         bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1021         bs.copy_store_at_16(Address(d, 4 * unit), t3, t4);
1022         bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1023         bs.copy_store_at_16(Address(d, 6 * unit), t5, t6);
1024         bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1025         bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t7);
1026         bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1027       } else {
1028        // d was not offset when we started so the registers are
1029        // written into the 64 bit block preceding d with the following
1030        // offsets
1031        //
1032        // t1 at offset -8
1033        // t3 at offset -24, t0 at offset -16
1034        // t5 at offset -48, t2 at offset -32
1035        // t7 at offset -56, t4 at offset -48
1036        //                   t6 at offset -64
1037        //
1038        // note that this matches the offsets previously noted for the
1039        // loads
1040 
1041         bs.copy_store_at_8(Address(d, 1 * unit), t1);
1042         bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
1043         bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1044         bs.copy_store_at_16(Address(d, 5 * unit), t5, t2);
1045         bs.copy_load_at_16(t2, t3, Address(s, 4 * unit));
1046         bs.copy_store_at_16(Address(d, 7 * unit), t7, t4);
1047         bs.copy_load_at_16(t4, t5, Address(s, 6 * unit));
1048         bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t6);
1049         bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
1050       }
1051 
1052       __ subs(count, count, 8);
1053       __ br(Assembler::HS, again);
1054 
1055       // Drain
1056       //
1057       // this uses the same pattern of offsets and register arguments
1058       // as above
1059       __ bind(drain);
1060       if (direction == copy_forwards) {
1061         bs.copy_store_at_8(Address(d, 1 * unit), t0);
1062         bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
1063         bs.copy_store_at_16(Address(d, 4 * unit), t3, t4);
1064         bs.copy_store_at_16(Address(d, 6 * unit), t5, t6);
1065         bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t7);
1066       } else {
1067         bs.copy_store_at_8(Address(d, 1 * unit), t1);
1068         bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
1069         bs.copy_store_at_16(Address(d, 5 * unit), t5, t2);
1070         bs.copy_store_at_16(Address(d, 7 * unit), t7, t4);
1071         bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t6);
1072       }
1073       // now we need to copy any remaining part block which may
1074       // include a 4 word block subblock and/or a 2 word subblock.
1075       // bits 2 and 1 in the count are the tell-tale for whether we
1076       // have each such subblock
1077       {
1078         Label L1, L2;
1079         __ tbz(count, exact_log2(4), L1);
1080        // this is the same as above but copying only 4 longs hence
1081        // with only one intervening stp between the str instructions
1082        // but note that the offsets and registers still follow the
1083        // same pattern
1084         bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
1085         bs.copy_load_at_16(t2, t3, Address(__ pre(s, 4 * unit)));
1086         if (direction == copy_forwards) {
1087           bs.copy_store_at_8(Address(d, 1 * unit), t0);
1088           bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
1089           bs.copy_store_at_8(Address(__ pre(d, 4 * unit)), t3);
1090         } else {
1091           bs.copy_store_at_8(Address(d, 1 * unit), t1);
1092           bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
1093           bs.copy_store_at_8(Address(__ pre(d, 4 * unit)), t2);
1094         }
1095         __ bind(L1);
1096 
1097         __ tbz(count, 1, L2);
1098        // this is the same as above but copying only 2 longs hence
1099        // there is no intervening stp between the str instructions
1100        // but note that the offset and register patterns are still
1101        // the same
1102         bs.copy_load_at_16(t0, t1, Address(__ pre(s, 2 * unit)));
1103         if (direction == copy_forwards) {
1104           bs.copy_store_at_8(Address(d, 1 * unit), t0);
1105           bs.copy_store_at_8(Address(__ pre(d, 2 * unit)), t1);
1106         } else {
1107           bs.copy_store_at_8(Address(d, 1 * unit), t1);
1108           bs.copy_store_at_8(Address(__ pre(d, 2 * unit)), t0);
1109         }
1110         __ bind(L2);
1111 
1112        // for forwards copy we need to re-adjust the offsets we
1113        // applied so that s and d are follow the last words written
1114 
1115        if (direction == copy_forwards) {
1116          __ add(s, s, 16);
1117          __ add(d, d, 8);
1118        }
1119 
1120       }
1121 
1122       __ ret(lr);
1123       }
1124   }
1125 
1126   // Small copy: less than 16 bytes.
1127   //
1128   // NB: Ignores all of the bits of count which represent more than 15
1129   // bytes, so a caller doesn't have to mask them.
1130 
1131   void copy_memory_small(DecoratorSet decorators, BasicType type, Register s, Register d, Register count, int step) {
1132     bool is_backwards = step < 0;
1133     size_t granularity = uabs(step);
1134     int direction = is_backwards ? -1 : 1;
1135 
1136     Label Lword, Lint, Lshort, Lbyte;
1137 
1138     assert(granularity
1139            && granularity <= sizeof (jlong), "Impossible granularity in copy_memory_small");
1140 
1141     const Register t0 = r3;
1142     const Register gct1 = rscratch1, gct2 = rscratch2, gct3 = r10;
1143     ArrayCopyBarrierSetHelper bs(_masm, decorators, type, gct1, gct2, gct3, fnoreg, fnoreg, fnoreg);
1144 
1145     // ??? I don't know if this bit-test-and-branch is the right thing
1146     // to do.  It does a lot of jumping, resulting in several
1147     // mispredicted branches.  It might make more sense to do this
1148     // with something like Duff's device with a single computed branch.
1149 
1150     __ tbz(count, 3 - exact_log2(granularity), Lword);
1151     bs.copy_load_at_8(t0, Address(__ adjust(s, direction * wordSize, is_backwards)));
1152     bs.copy_store_at_8(Address(__ adjust(d, direction * wordSize, is_backwards)), t0);
1153     __ bind(Lword);
1154 
1155     if (granularity <= sizeof (jint)) {
1156       __ tbz(count, 2 - exact_log2(granularity), Lint);
1157       __ ldrw(t0, Address(__ adjust(s, sizeof (jint) * direction, is_backwards)));
1158       __ strw(t0, Address(__ adjust(d, sizeof (jint) * direction, is_backwards)));
1159       __ bind(Lint);
1160     }
1161 
1162     if (granularity <= sizeof (jshort)) {
1163       __ tbz(count, 1 - exact_log2(granularity), Lshort);
1164       __ ldrh(t0, Address(__ adjust(s, sizeof (jshort) * direction, is_backwards)));
1165       __ strh(t0, Address(__ adjust(d, sizeof (jshort) * direction, is_backwards)));
1166       __ bind(Lshort);
1167     }
1168 
1169     if (granularity <= sizeof (jbyte)) {
1170       __ tbz(count, 0, Lbyte);
1171       __ ldrb(t0, Address(__ adjust(s, sizeof (jbyte) * direction, is_backwards)));
1172       __ strb(t0, Address(__ adjust(d, sizeof (jbyte) * direction, is_backwards)));
1173       __ bind(Lbyte);
1174     }
1175   }
1176 
1177   Label copy_f, copy_b;
1178   Label copy_obj_f, copy_obj_b;
1179   Label copy_obj_uninit_f, copy_obj_uninit_b;
1180 
1181   // All-singing all-dancing memory copy.
1182   //
1183   // Copy count units of memory from s to d.  The size of a unit is
1184   // step, which can be positive or negative depending on the direction
1185   // of copy.  If is_aligned is false, we align the source address.
1186   //
1187 
1188   void copy_memory(DecoratorSet decorators, BasicType type, bool is_aligned,
1189                    Register s, Register d, Register count, int step) {
1190     copy_direction direction = step < 0 ? copy_backwards : copy_forwards;
1191     bool is_backwards = step < 0;
1192     unsigned int granularity = uabs(step);
1193     const Register t0 = r3, t1 = r4;
1194 
1195     // <= 80 (or 96 for SIMD) bytes do inline. Direction doesn't matter because we always
1196     // load all the data before writing anything
1197     Label copy4, copy8, copy16, copy32, copy80, copy_big, finish;
1198     const Register t2 = r5, t3 = r6, t4 = r7, t5 = r11;
1199     const Register t6 = r12, t7 = r13, t8 = r14, t9 = r15;
1200     const Register send = r17, dend = r16;
1201     const Register gct1 = rscratch1, gct2 = rscratch2, gct3 = r10;
1202     const FloatRegister gcvt1 = v6, gcvt2 = v7, gcvt3 = v16; // Note that v8-v15 are callee saved
1203     ArrayCopyBarrierSetHelper bs(_masm, decorators, type, gct1, gct2, gct3, gcvt1, gcvt2, gcvt3);
1204 
1205     if (PrefetchCopyIntervalInBytes > 0)
1206       __ prfm(Address(s, 0), PLDL1KEEP);
1207     __ cmp(count, u1((UseSIMDForMemoryOps ? 96:80)/granularity));
1208     __ br(Assembler::HI, copy_big);
1209 
1210     __ lea(send, Address(s, count, Address::lsl(exact_log2(granularity))));
1211     __ lea(dend, Address(d, count, Address::lsl(exact_log2(granularity))));
1212 
1213     __ cmp(count, u1(16/granularity));
1214     __ br(Assembler::LS, copy16);
1215 
1216     __ cmp(count, u1(64/granularity));
1217     __ br(Assembler::HI, copy80);
1218 
1219     __ cmp(count, u1(32/granularity));
1220     __ br(Assembler::LS, copy32);
1221 
1222     // 33..64 bytes
1223     if (UseSIMDForMemoryOps) {
1224       bs.copy_load_at_32(v0, v1, Address(s, 0));
1225       bs.copy_load_at_32(v2, v3, Address(send, -32));
1226       bs.copy_store_at_32(Address(d, 0), v0, v1);
1227       bs.copy_store_at_32(Address(dend, -32), v2, v3);
1228     } else {
1229       bs.copy_load_at_16(t0, t1, Address(s, 0));
1230       bs.copy_load_at_16(t2, t3, Address(s, 16));
1231       bs.copy_load_at_16(t4, t5, Address(send, -32));
1232       bs.copy_load_at_16(t6, t7, Address(send, -16));
1233 
1234       bs.copy_store_at_16(Address(d, 0), t0, t1);
1235       bs.copy_store_at_16(Address(d, 16), t2, t3);
1236       bs.copy_store_at_16(Address(dend, -32), t4, t5);
1237       bs.copy_store_at_16(Address(dend, -16), t6, t7);
1238     }
1239     __ b(finish);
1240 
1241     // 17..32 bytes
1242     __ bind(copy32);
1243     bs.copy_load_at_16(t0, t1, Address(s, 0));
1244     bs.copy_load_at_16(t6, t7, Address(send, -16));
1245 
1246     bs.copy_store_at_16(Address(d, 0), t0, t1);
1247     bs.copy_store_at_16(Address(dend, -16), t6, t7);
1248     __ b(finish);
1249 
1250     // 65..80/96 bytes
1251     // (96 bytes if SIMD because we do 32 byes per instruction)
1252     __ bind(copy80);
1253     if (UseSIMDForMemoryOps) {
1254       bs.copy_load_at_32(v0, v1, Address(s, 0));
1255       bs.copy_load_at_32(v2, v3, Address(s, 32));
1256       // Unaligned pointers can be an issue for copying.
1257       // The issue has more chances to happen when granularity of data is
1258       // less than 4(sizeof(jint)). Pointers for arrays of jint are at least
1259       // 4 byte aligned. Pointers for arrays of jlong are 8 byte aligned.
1260       // The most performance drop has been seen for the range 65-80 bytes.
1261       // For such cases using the pair of ldp/stp instead of the third pair of
1262       // ldpq/stpq fixes the performance issue.
1263       if (granularity < sizeof (jint)) {
1264         Label copy96;
1265         __ cmp(count, u1(80/granularity));
1266         __ br(Assembler::HI, copy96);
1267         bs.copy_load_at_16(t0, t1, Address(send, -16));
1268 
1269         bs.copy_store_at_32(Address(d, 0), v0, v1);
1270         bs.copy_store_at_32(Address(d, 32), v2, v3);
1271 
1272         bs.copy_store_at_16(Address(dend, -16), t0, t1);
1273         __ b(finish);
1274 
1275         __ bind(copy96);
1276       }
1277       bs.copy_load_at_32(v4, v5, Address(send, -32));
1278 
1279       bs.copy_store_at_32(Address(d, 0), v0, v1);
1280       bs.copy_store_at_32(Address(d, 32), v2, v3);
1281 
1282       bs.copy_store_at_32(Address(dend, -32), v4, v5);
1283     } else {
1284       bs.copy_load_at_16(t0, t1, Address(s, 0));
1285       bs.copy_load_at_16(t2, t3, Address(s, 16));
1286       bs.copy_load_at_16(t4, t5, Address(s, 32));
1287       bs.copy_load_at_16(t6, t7, Address(s, 48));
1288       bs.copy_load_at_16(t8, t9, Address(send, -16));
1289 
1290       bs.copy_store_at_16(Address(d, 0), t0, t1);
1291       bs.copy_store_at_16(Address(d, 16), t2, t3);
1292       bs.copy_store_at_16(Address(d, 32), t4, t5);
1293       bs.copy_store_at_16(Address(d, 48), t6, t7);
1294       bs.copy_store_at_16(Address(dend, -16), t8, t9);
1295     }
1296     __ b(finish);
1297 
1298     // 0..16 bytes
1299     __ bind(copy16);
1300     __ cmp(count, u1(8/granularity));
1301     __ br(Assembler::LO, copy8);
1302 
1303     // 8..16 bytes
1304     bs.copy_load_at_8(t0, Address(s, 0));
1305     bs.copy_load_at_8(t1, Address(send, -8));
1306     bs.copy_store_at_8(Address(d, 0), t0);
1307     bs.copy_store_at_8(Address(dend, -8), t1);
1308     __ b(finish);
1309 
1310     if (granularity < 8) {
1311       // 4..7 bytes
1312       __ bind(copy8);
1313       __ tbz(count, 2 - exact_log2(granularity), copy4);
1314       __ ldrw(t0, Address(s, 0));
1315       __ ldrw(t1, Address(send, -4));
1316       __ strw(t0, Address(d, 0));
1317       __ strw(t1, Address(dend, -4));
1318       __ b(finish);
1319       if (granularity < 4) {
1320         // 0..3 bytes
1321         __ bind(copy4);
1322         __ cbz(count, finish); // get rid of 0 case
1323         if (granularity == 2) {
1324           __ ldrh(t0, Address(s, 0));
1325           __ strh(t0, Address(d, 0));
1326         } else { // granularity == 1
1327           // Now 1..3 bytes. Handle the 1 and 2 byte case by copying
1328           // the first and last byte.
1329           // Handle the 3 byte case by loading and storing base + count/2
1330           // (count == 1 (s+0)->(d+0), count == 2,3 (s+1) -> (d+1))
1331           // This does means in the 1 byte case we load/store the same
1332           // byte 3 times.
1333           __ lsr(count, count, 1);
1334           __ ldrb(t0, Address(s, 0));
1335           __ ldrb(t1, Address(send, -1));
1336           __ ldrb(t2, Address(s, count));
1337           __ strb(t0, Address(d, 0));
1338           __ strb(t1, Address(dend, -1));
1339           __ strb(t2, Address(d, count));
1340         }
1341         __ b(finish);
1342       }
1343     }
1344 
1345     __ bind(copy_big);
1346     if (is_backwards) {
1347       __ lea(s, Address(s, count, Address::lsl(exact_log2(-step))));
1348       __ lea(d, Address(d, count, Address::lsl(exact_log2(-step))));
1349     }
1350 
1351     // Now we've got the small case out of the way we can align the
1352     // source address on a 2-word boundary.
1353 
1354     // Here we will materialize a count in r15, which is used by copy_memory_small
1355     // and the various generate_copy_longs stubs that we use for 2 word aligned bytes.
1356     // Up until here, we have used t9, which aliases r15, but from here on, that register
1357     // can not be used as a temp register, as it contains the count.
1358 
1359     Label aligned;
1360 
1361     if (is_aligned) {
1362       // We may have to adjust by 1 word to get s 2-word-aligned.
1363       __ tbz(s, exact_log2(wordSize), aligned);
1364       bs.copy_load_at_8(t0, Address(__ adjust(s, direction * wordSize, is_backwards)));
1365       bs.copy_store_at_8(Address(__ adjust(d, direction * wordSize, is_backwards)), t0);
1366       __ sub(count, count, wordSize/granularity);
1367     } else {
1368       if (is_backwards) {
1369         __ andr(r15, s, 2 * wordSize - 1);
1370       } else {
1371         __ neg(r15, s);
1372         __ andr(r15, r15, 2 * wordSize - 1);
1373       }
1374       // r15 is the byte adjustment needed to align s.
1375       __ cbz(r15, aligned);
1376       int shift = exact_log2(granularity);
1377       if (shift)  __ lsr(r15, r15, shift);
1378       __ sub(count, count, r15);
1379 
1380 #if 0
1381       // ?? This code is only correct for a disjoint copy.  It may or
1382       // may not make sense to use it in that case.
1383 
1384       // Copy the first pair; s and d may not be aligned.
1385       __ ldp(t0, t1, Address(s, is_backwards ? -2 * wordSize : 0));
1386       __ stp(t0, t1, Address(d, is_backwards ? -2 * wordSize : 0));
1387 
1388       // Align s and d, adjust count
1389       if (is_backwards) {
1390         __ sub(s, s, r15);
1391         __ sub(d, d, r15);
1392       } else {
1393         __ add(s, s, r15);
1394         __ add(d, d, r15);
1395       }
1396 #else
1397       copy_memory_small(decorators, type, s, d, r15, step);
1398 #endif
1399     }
1400 
1401     __ bind(aligned);
1402 
1403     // s is now 2-word-aligned.
1404 
1405     // We have a count of units and some trailing bytes.  Adjust the
1406     // count and do a bulk copy of words.
1407     __ lsr(r15, count, exact_log2(wordSize/granularity));
1408     if (direction == copy_forwards) {
1409       if (type != T_OBJECT) {
1410         __ bl(copy_f);
1411       } else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
1412         __ bl(copy_obj_uninit_f);
1413       } else {
1414         __ bl(copy_obj_f);
1415       }
1416     } else {
1417       if (type != T_OBJECT) {
1418         __ bl(copy_b);
1419       } else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
1420         __ bl(copy_obj_uninit_b);
1421       } else {
1422         __ bl(copy_obj_b);
1423       }
1424     }
1425 
1426     // And the tail.
1427     copy_memory_small(decorators, type, s, d, count, step);
1428 
1429     if (granularity >= 8) __ bind(copy8);
1430     if (granularity >= 4) __ bind(copy4);
1431     __ bind(finish);
1432   }
1433 
1434 
1435   void clobber_registers() {
1436 #ifdef ASSERT
1437     RegSet clobbered
1438       = MacroAssembler::call_clobbered_gp_registers() - rscratch1;
1439     __ mov(rscratch1, (uint64_t)0xdeadbeef);
1440     __ orr(rscratch1, rscratch1, rscratch1, Assembler::LSL, 32);
1441     for (RegSetIterator<Register> it = clobbered.begin(); *it != noreg; ++it) {
1442       __ mov(*it, rscratch1);
1443     }
1444 #endif
1445 
1446   }
1447 
1448   // Scan over array at a for count oops, verifying each one.
1449   // Preserves a and count, clobbers rscratch1 and rscratch2.
1450   void verify_oop_array (int size, Register a, Register count, Register temp) {
1451     Label loop, end;
1452     __ mov(rscratch1, a);
1453     __ mov(rscratch2, zr);
1454     __ bind(loop);
1455     __ cmp(rscratch2, count);
1456     __ br(Assembler::HS, end);
1457     if (size == wordSize) {
1458       __ ldr(temp, Address(a, rscratch2, Address::lsl(exact_log2(size))));
1459       __ verify_oop(temp);
1460     } else {
1461       __ ldrw(temp, Address(a, rscratch2, Address::lsl(exact_log2(size))));
1462       __ decode_heap_oop(temp); // calls verify_oop
1463     }
1464     __ add(rscratch2, rscratch2, 1);
1465     __ b(loop);
1466     __ bind(end);
1467   }
1468 
1469   // Arguments:
1470   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1471   //             ignored
1472   //   is_oop  - true => oop array, so generate store check code
1473   //   name    - stub name string
1474   //
1475   // Inputs:
1476   //   c_rarg0   - source array address
1477   //   c_rarg1   - destination array address
1478   //   c_rarg2   - element count, treated as ssize_t, can be zero
1479   //
1480   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1481   // the hardware handle it.  The two dwords within qwords that span
1482   // cache line boundaries will still be loaded and stored atomically.
1483   //
1484   // Side Effects:
1485   //   disjoint_int_copy_entry is set to the no-overlap entry point
1486   //   used by generate_conjoint_int_oop_copy().
1487   //
1488   address generate_disjoint_copy(int size, bool aligned, bool is_oop, address *entry,
1489                                   const char *name, bool dest_uninitialized = false) {
1490     Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1491     RegSet saved_reg = RegSet::of(s, d, count);
1492     __ align(CodeEntryAlignment);
1493     StubCodeMark mark(this, "StubRoutines", name);
1494     address start = __ pc();
1495     __ enter();
1496 
1497     if (entry != nullptr) {
1498       *entry = __ pc();
1499       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1500       BLOCK_COMMENT("Entry:");
1501     }
1502 
1503     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
1504     if (dest_uninitialized) {
1505       decorators |= IS_DEST_UNINITIALIZED;
1506     }
1507     if (aligned) {
1508       decorators |= ARRAYCOPY_ALIGNED;
1509     }
1510 
1511     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1512     bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_reg);
1513 
1514     if (is_oop) {
1515       // save regs before copy_memory
1516       __ push(RegSet::of(d, count), sp);
1517     }
1518     {
1519       // UnsafeMemoryAccess page error: continue after unsafe access
1520       bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size);
1521       UnsafeMemoryAccessMark umam(this, add_entry, true);
1522       copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, size);
1523     }
1524 
1525     if (is_oop) {
1526       __ pop(RegSet::of(d, count), sp);
1527       if (VerifyOops)
1528         verify_oop_array(size, d, count, r16);
1529     }
1530 
1531     bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1, RegSet());
1532 
1533     __ leave();
1534     __ mov(r0, zr); // return 0
1535     __ ret(lr);
1536     return start;
1537   }
1538 
1539   // Arguments:
1540   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1541   //             ignored
1542   //   is_oop  - true => oop array, so generate store check code
1543   //   name    - stub name string
1544   //
1545   // Inputs:
1546   //   c_rarg0   - source array address
1547   //   c_rarg1   - destination array address
1548   //   c_rarg2   - element count, treated as ssize_t, can be zero
1549   //
1550   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1551   // the hardware handle it.  The two dwords within qwords that span
1552   // cache line boundaries will still be loaded and stored atomically.
1553   //
1554   address generate_conjoint_copy(int size, bool aligned, bool is_oop, address nooverlap_target,
1555                                  address *entry, const char *name,
1556                                  bool dest_uninitialized = false) {
1557     Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
1558     RegSet saved_regs = RegSet::of(s, d, count);
1559     StubCodeMark mark(this, "StubRoutines", name);
1560     address start = __ pc();
1561     __ enter();
1562 
1563     if (entry != nullptr) {
1564       *entry = __ pc();
1565       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1566       BLOCK_COMMENT("Entry:");
1567     }
1568 
1569     // use fwd copy when (d-s) above_equal (count*size)
1570     __ sub(rscratch1, d, s);
1571     __ cmp(rscratch1, count, Assembler::LSL, exact_log2(size));
1572     __ br(Assembler::HS, nooverlap_target);
1573 
1574     DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1575     if (dest_uninitialized) {
1576       decorators |= IS_DEST_UNINITIALIZED;
1577     }
1578     if (aligned) {
1579       decorators |= ARRAYCOPY_ALIGNED;
1580     }
1581 
1582     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1583     bs->arraycopy_prologue(_masm, decorators, is_oop, s, d, count, saved_regs);
1584 
1585     if (is_oop) {
1586       // save regs before copy_memory
1587       __ push(RegSet::of(d, count), sp);
1588     }
1589     {
1590       // UnsafeMemoryAccess page error: continue after unsafe access
1591       bool add_entry = !is_oop && (!aligned || sizeof(jlong) == size);
1592       UnsafeMemoryAccessMark umam(this, add_entry, true);
1593       copy_memory(decorators, is_oop ? T_OBJECT : T_BYTE, aligned, s, d, count, -size);
1594     }
1595     if (is_oop) {
1596       __ pop(RegSet::of(d, count), sp);
1597       if (VerifyOops)
1598         verify_oop_array(size, d, count, r16);
1599     }
1600     bs->arraycopy_epilogue(_masm, decorators, is_oop, d, count, rscratch1, RegSet());
1601     __ leave();
1602     __ mov(r0, zr); // return 0
1603     __ ret(lr);
1604     return start;
1605 }
1606 
1607   // Arguments:
1608   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1609   //             ignored
1610   //   name    - stub name string
1611   //
1612   // Inputs:
1613   //   c_rarg0   - source array address
1614   //   c_rarg1   - destination array address
1615   //   c_rarg2   - element count, treated as ssize_t, can be zero
1616   //
1617   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1618   // we let the hardware handle it.  The one to eight bytes within words,
1619   // dwords or qwords that span cache line boundaries will still be loaded
1620   // and stored atomically.
1621   //
1622   // Side Effects:
1623   //   disjoint_byte_copy_entry is set to the no-overlap entry point  //
1624   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1625   // we let the hardware handle it.  The one to eight bytes within words,
1626   // dwords or qwords that span cache line boundaries will still be loaded
1627   // and stored atomically.
1628   //
1629   // Side Effects:
1630   //   disjoint_byte_copy_entry is set to the no-overlap entry point
1631   //   used by generate_conjoint_byte_copy().
1632   //
1633   address generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) {
1634     const bool not_oop = false;
1635     return generate_disjoint_copy(sizeof (jbyte), aligned, not_oop, entry, name);
1636   }
1637 
1638   // Arguments:
1639   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1640   //             ignored
1641   //   name    - stub name string
1642   //
1643   // Inputs:
1644   //   c_rarg0   - source array address
1645   //   c_rarg1   - destination array address
1646   //   c_rarg2   - element count, treated as ssize_t, can be zero
1647   //
1648   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1649   // we let the hardware handle it.  The one to eight bytes within words,
1650   // dwords or qwords that span cache line boundaries will still be loaded
1651   // and stored atomically.
1652   //
1653   address generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
1654                                       address* entry, const char *name) {
1655     const bool not_oop = false;
1656     return generate_conjoint_copy(sizeof (jbyte), aligned, not_oop, nooverlap_target, entry, name);
1657   }
1658 
1659   // Arguments:
1660   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1661   //             ignored
1662   //   name    - stub name string
1663   //
1664   // Inputs:
1665   //   c_rarg0   - source array address
1666   //   c_rarg1   - destination array address
1667   //   c_rarg2   - element count, treated as ssize_t, can be zero
1668   //
1669   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1670   // let the hardware handle it.  The two or four words within dwords
1671   // or qwords that span cache line boundaries will still be loaded
1672   // and stored atomically.
1673   //
1674   // Side Effects:
1675   //   disjoint_short_copy_entry is set to the no-overlap entry point
1676   //   used by generate_conjoint_short_copy().
1677   //
1678   address generate_disjoint_short_copy(bool aligned,
1679                                        address* entry, const char *name) {
1680     const bool not_oop = false;
1681     return generate_disjoint_copy(sizeof (jshort), aligned, not_oop, entry, name);
1682   }
1683 
1684   // Arguments:
1685   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1686   //             ignored
1687   //   name    - stub name string
1688   //
1689   // Inputs:
1690   //   c_rarg0   - source array address
1691   //   c_rarg1   - destination array address
1692   //   c_rarg2   - element count, treated as ssize_t, can be zero
1693   //
1694   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1695   // let the hardware handle it.  The two or four words within dwords
1696   // or qwords that span cache line boundaries will still be loaded
1697   // and stored atomically.
1698   //
1699   address generate_conjoint_short_copy(bool aligned, address nooverlap_target,
1700                                        address *entry, const char *name) {
1701     const bool not_oop = false;
1702     return generate_conjoint_copy(sizeof (jshort), aligned, not_oop, nooverlap_target, entry, name);
1703 
1704   }
1705   // Arguments:
1706   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1707   //             ignored
1708   //   name    - stub name string
1709   //
1710   // Inputs:
1711   //   c_rarg0   - source array address
1712   //   c_rarg1   - destination array address
1713   //   c_rarg2   - element count, treated as ssize_t, can be zero
1714   //
1715   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1716   // the hardware handle it.  The two dwords within qwords that span
1717   // cache line boundaries will still be loaded and stored atomically.
1718   //
1719   // Side Effects:
1720   //   disjoint_int_copy_entry is set to the no-overlap entry point
1721   //   used by generate_conjoint_int_oop_copy().
1722   //
1723   address generate_disjoint_int_copy(bool aligned, address *entry,
1724                                          const char *name, bool dest_uninitialized = false) {
1725     const bool not_oop = false;
1726     return generate_disjoint_copy(sizeof (jint), aligned, not_oop, entry, name);
1727   }
1728 
1729   // Arguments:
1730   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1731   //             ignored
1732   //   name    - stub name string
1733   //
1734   // Inputs:
1735   //   c_rarg0   - source array address
1736   //   c_rarg1   - destination array address
1737   //   c_rarg2   - element count, treated as ssize_t, can be zero
1738   //
1739   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1740   // the hardware handle it.  The two dwords within qwords that span
1741   // cache line boundaries will still be loaded and stored atomically.
1742   //
1743   address generate_conjoint_int_copy(bool aligned, address nooverlap_target,
1744                                      address *entry, const char *name,
1745                                      bool dest_uninitialized = false) {
1746     const bool not_oop = false;
1747     return generate_conjoint_copy(sizeof (jint), aligned, not_oop, nooverlap_target, entry, name);
1748   }
1749 
1750 
1751   // Arguments:
1752   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1753   //             ignored
1754   //   name    - stub name string
1755   //
1756   // Inputs:
1757   //   c_rarg0   - source array address
1758   //   c_rarg1   - destination array address
1759   //   c_rarg2   - element count, treated as size_t, can be zero
1760   //
1761   // Side Effects:
1762   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
1763   //   no-overlap entry point used by generate_conjoint_long_oop_copy().
1764   //
1765   address generate_disjoint_long_copy(bool aligned, address *entry,
1766                                           const char *name, bool dest_uninitialized = false) {
1767     const bool not_oop = false;
1768     return generate_disjoint_copy(sizeof (jlong), aligned, not_oop, entry, name);
1769   }
1770 
1771   // Arguments:
1772   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1773   //             ignored
1774   //   name    - stub name string
1775   //
1776   // Inputs:
1777   //   c_rarg0   - source array address
1778   //   c_rarg1   - destination array address
1779   //   c_rarg2   - element count, treated as size_t, can be zero
1780   //
1781   address generate_conjoint_long_copy(bool aligned,
1782                                       address nooverlap_target, address *entry,
1783                                       const char *name, bool dest_uninitialized = false) {
1784     const bool not_oop = false;
1785     return generate_conjoint_copy(sizeof (jlong), aligned, not_oop, nooverlap_target, entry, name);
1786   }
1787 
1788   // Arguments:
1789   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1790   //             ignored
1791   //   name    - stub name string
1792   //
1793   // Inputs:
1794   //   c_rarg0   - source array address
1795   //   c_rarg1   - destination array address
1796   //   c_rarg2   - element count, treated as size_t, can be zero
1797   //
1798   // Side Effects:
1799   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
1800   //   no-overlap entry point used by generate_conjoint_long_oop_copy().
1801   //
1802   address generate_disjoint_oop_copy(bool aligned, address *entry,
1803                                      const char *name, bool dest_uninitialized) {
1804     const bool is_oop = true;
1805     const int size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1806     return generate_disjoint_copy(size, aligned, is_oop, entry, name, dest_uninitialized);
1807   }
1808 
1809   // Arguments:
1810   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1811   //             ignored
1812   //   name    - stub name string
1813   //
1814   // Inputs:
1815   //   c_rarg0   - source array address
1816   //   c_rarg1   - destination array address
1817   //   c_rarg2   - element count, treated as size_t, can be zero
1818   //
1819   address generate_conjoint_oop_copy(bool aligned,
1820                                      address nooverlap_target, address *entry,
1821                                      const char *name, bool dest_uninitialized) {
1822     const bool is_oop = true;
1823     const int size = UseCompressedOops ? sizeof (jint) : sizeof (jlong);
1824     return generate_conjoint_copy(size, aligned, is_oop, nooverlap_target, entry,
1825                                   name, dest_uninitialized);
1826   }
1827 
1828 
1829   // Helper for generating a dynamic type check.
1830   // Smashes rscratch1, rscratch2.
1831   void generate_type_check(Register sub_klass,
1832                            Register super_check_offset,
1833                            Register super_klass,
1834                            Label& L_success) {
1835     assert_different_registers(sub_klass, super_check_offset, super_klass);
1836 
1837     BLOCK_COMMENT("type_check:");
1838 
1839     Label L_miss;
1840 
1841     __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg,        &L_success, &L_miss, nullptr,
1842                                      super_check_offset);
1843     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, nullptr);
1844 
1845     // Fall through on failure!
1846     __ BIND(L_miss);
1847   }
1848 
1849   //
1850   //  Generate checkcasting array copy stub
1851   //
1852   //  Input:
1853   //    c_rarg0   - source array address
1854   //    c_rarg1   - destination array address
1855   //    c_rarg2   - element count, treated as ssize_t, can be zero
1856   //    c_rarg3   - size_t ckoff (super_check_offset)
1857   //    c_rarg4   - oop ckval (super_klass)
1858   //
1859   //  Output:
1860   //    r0 ==  0  -  success
1861   //    r0 == -1^K - failure, where K is partial transfer count
1862   //
1863   address generate_checkcast_copy(const char *name, address *entry,
1864                                   bool dest_uninitialized = false) {
1865 
1866     Label L_load_element, L_store_element, L_do_card_marks, L_done, L_done_pop;
1867 
1868     // Input registers (after setup_arg_regs)
1869     const Register from        = c_rarg0;   // source array address
1870     const Register to          = c_rarg1;   // destination array address
1871     const Register count       = c_rarg2;   // elementscount
1872     const Register ckoff       = c_rarg3;   // super_check_offset
1873     const Register ckval       = c_rarg4;   // super_klass
1874 
1875     RegSet wb_pre_saved_regs = RegSet::range(c_rarg0, c_rarg4);
1876     RegSet wb_post_saved_regs = RegSet::of(count);
1877 
1878     // Registers used as temps (r19, r20, r21, r22 are save-on-entry)
1879     const Register copied_oop  = r22;       // actual oop copied
1880     const Register count_save  = r21;       // orig elementscount
1881     const Register start_to    = r20;       // destination array start address
1882     const Register r19_klass   = r19;       // oop._klass
1883 
1884     // Registers used as gc temps (r5, r6, r7 are save-on-call)
1885     const Register gct1 = r5, gct2 = r6, gct3 = r7;
1886 
1887     //---------------------------------------------------------------
1888     // Assembler stub will be used for this call to arraycopy
1889     // if the two arrays are subtypes of Object[] but the
1890     // destination array type is not equal to or a supertype
1891     // of the source type.  Each element must be separately
1892     // checked.
1893 
1894     assert_different_registers(from, to, count, ckoff, ckval, start_to,
1895                                copied_oop, r19_klass, count_save);
1896 
1897     __ align(CodeEntryAlignment);
1898     StubCodeMark mark(this, "StubRoutines", name);
1899     address start = __ pc();
1900 
1901     __ enter(); // required for proper stackwalking of RuntimeStub frame
1902 
1903 #ifdef ASSERT
1904     // caller guarantees that the arrays really are different
1905     // otherwise, we would have to make conjoint checks
1906     { Label L;
1907       __ b(L);                  // conjoint check not yet implemented
1908       __ stop("checkcast_copy within a single array");
1909       __ bind(L);
1910     }
1911 #endif //ASSERT
1912 
1913     // Caller of this entry point must set up the argument registers.
1914     if (entry != nullptr) {
1915       *entry = __ pc();
1916       BLOCK_COMMENT("Entry:");
1917     }
1918 
1919      // Empty array:  Nothing to do.
1920     __ cbz(count, L_done);
1921     __ push(RegSet::of(r19, r20, r21, r22), sp);
1922 
1923 #ifdef ASSERT
1924     BLOCK_COMMENT("assert consistent ckoff/ckval");
1925     // The ckoff and ckval must be mutually consistent,
1926     // even though caller generates both.
1927     { Label L;
1928       int sco_offset = in_bytes(Klass::super_check_offset_offset());
1929       __ ldrw(start_to, Address(ckval, sco_offset));
1930       __ cmpw(ckoff, start_to);
1931       __ br(Assembler::EQ, L);
1932       __ stop("super_check_offset inconsistent");
1933       __ bind(L);
1934     }
1935 #endif //ASSERT
1936 
1937     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT;
1938     bool is_oop = true;
1939     int element_size = UseCompressedOops ? 4 : 8;
1940     if (dest_uninitialized) {
1941       decorators |= IS_DEST_UNINITIALIZED;
1942     }
1943 
1944     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1945     bs->arraycopy_prologue(_masm, decorators, is_oop, from, to, count, wb_pre_saved_regs);
1946 
1947     // save the original count
1948     __ mov(count_save, count);
1949 
1950     // Copy from low to high addresses
1951     __ mov(start_to, to);              // Save destination array start address
1952     __ b(L_load_element);
1953 
1954     // ======== begin loop ========
1955     // (Loop is rotated; its entry is L_load_element.)
1956     // Loop control:
1957     //   for (; count != 0; count--) {
1958     //     copied_oop = load_heap_oop(from++);
1959     //     ... generate_type_check ...;
1960     //     store_heap_oop(to++, copied_oop);
1961     //   }
1962     __ align(OptoLoopAlignment);
1963 
1964     __ BIND(L_store_element);
1965     bs->copy_store_at(_masm, decorators, T_OBJECT, element_size,
1966                       __ post(to, element_size), copied_oop, noreg,
1967                       gct1, gct2, gct3);
1968     __ sub(count, count, 1);
1969     __ cbz(count, L_do_card_marks);
1970 
1971     // ======== loop entry is here ========
1972     __ BIND(L_load_element);
1973     bs->copy_load_at(_masm, decorators, T_OBJECT, element_size,
1974                      copied_oop, noreg, __ post(from, element_size),
1975                      gct1);
1976     __ cbz(copied_oop, L_store_element);
1977 
1978     __ load_klass(r19_klass, copied_oop);// query the object klass
1979     generate_type_check(r19_klass, ckoff, ckval, L_store_element);
1980     // ======== end loop ========
1981 
1982     // It was a real error; we must depend on the caller to finish the job.
1983     // Register count = remaining oops, count_orig = total oops.
1984     // Emit GC store barriers for the oops we have copied and report
1985     // their number to the caller.
1986 
1987     __ subs(count, count_save, count);     // K = partially copied oop count
1988     __ eon(count, count, zr);                   // report (-1^K) to caller
1989     __ br(Assembler::EQ, L_done_pop);
1990 
1991     __ BIND(L_do_card_marks);
1992     bs->arraycopy_epilogue(_masm, decorators, is_oop, start_to, count_save, rscratch1, wb_post_saved_regs);
1993 
1994     __ bind(L_done_pop);
1995     __ pop(RegSet::of(r19, r20, r21, r22), sp);
1996     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1997 
1998     __ bind(L_done);
1999     __ mov(r0, count);
2000     __ leave();
2001     __ ret(lr);
2002 
2003     return start;
2004   }
2005 
2006   // Perform range checks on the proposed arraycopy.
2007   // Kills temp, but nothing else.
2008   // Also, clean the sign bits of src_pos and dst_pos.
2009   void arraycopy_range_checks(Register src,     // source array oop (c_rarg0)
2010                               Register src_pos, // source position (c_rarg1)
2011                               Register dst,     // destination array oo (c_rarg2)
2012                               Register dst_pos, // destination position (c_rarg3)
2013                               Register length,
2014                               Register temp,
2015                               Label& L_failed) {
2016     BLOCK_COMMENT("arraycopy_range_checks:");
2017 
2018     assert_different_registers(rscratch1, temp);
2019 
2020     //  if (src_pos + length > arrayOop(src)->length())  FAIL;
2021     __ ldrw(rscratch1, Address(src, arrayOopDesc::length_offset_in_bytes()));
2022     __ addw(temp, length, src_pos);
2023     __ cmpw(temp, rscratch1);
2024     __ br(Assembler::HI, L_failed);
2025 
2026     //  if (dst_pos + length > arrayOop(dst)->length())  FAIL;
2027     __ ldrw(rscratch1, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2028     __ addw(temp, length, dst_pos);
2029     __ cmpw(temp, rscratch1);
2030     __ br(Assembler::HI, L_failed);
2031 
2032     // Have to clean up high 32 bits of 'src_pos' and 'dst_pos'.
2033     __ movw(src_pos, src_pos);
2034     __ movw(dst_pos, dst_pos);
2035 
2036     BLOCK_COMMENT("arraycopy_range_checks done");
2037   }
2038 
2039   // These stubs get called from some dumb test routine.
2040   // I'll write them properly when they're called from
2041   // something that's actually doing something.
2042   static void fake_arraycopy_stub(address src, address dst, int count) {
2043     assert(count == 0, "huh?");
2044   }
2045 
2046 
2047   //
2048   //  Generate 'unsafe' array copy stub
2049   //  Though just as safe as the other stubs, it takes an unscaled
2050   //  size_t argument instead of an element count.
2051   //
2052   //  Input:
2053   //    c_rarg0   - source array address
2054   //    c_rarg1   - destination array address
2055   //    c_rarg2   - byte count, treated as ssize_t, can be zero
2056   //
2057   // Examines the alignment of the operands and dispatches
2058   // to a long, int, short, or byte copy loop.
2059   //
2060   address generate_unsafe_copy(const char *name,
2061                                address byte_copy_entry,
2062                                address short_copy_entry,
2063                                address int_copy_entry,
2064                                address long_copy_entry) {
2065     Label L_long_aligned, L_int_aligned, L_short_aligned;
2066     Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
2067 
2068     __ align(CodeEntryAlignment);
2069     StubCodeMark mark(this, "StubRoutines", name);
2070     address start = __ pc();
2071     __ enter(); // required for proper stackwalking of RuntimeStub frame
2072 
2073     // bump this on entry, not on exit:
2074     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2075 
2076     __ orr(rscratch1, s, d);
2077     __ orr(rscratch1, rscratch1, count);
2078 
2079     __ andr(rscratch1, rscratch1, BytesPerLong-1);
2080     __ cbz(rscratch1, L_long_aligned);
2081     __ andr(rscratch1, rscratch1, BytesPerInt-1);
2082     __ cbz(rscratch1, L_int_aligned);
2083     __ tbz(rscratch1, 0, L_short_aligned);
2084     __ b(RuntimeAddress(byte_copy_entry));
2085 
2086     __ BIND(L_short_aligned);
2087     __ lsr(count, count, LogBytesPerShort);  // size => short_count
2088     __ b(RuntimeAddress(short_copy_entry));
2089     __ BIND(L_int_aligned);
2090     __ lsr(count, count, LogBytesPerInt);    // size => int_count
2091     __ b(RuntimeAddress(int_copy_entry));
2092     __ BIND(L_long_aligned);
2093     __ lsr(count, count, LogBytesPerLong);   // size => long_count
2094     __ b(RuntimeAddress(long_copy_entry));
2095 
2096     return start;
2097   }
2098 
2099   //
2100   //  Generate generic array copy stubs
2101   //
2102   //  Input:
2103   //    c_rarg0    -  src oop
2104   //    c_rarg1    -  src_pos (32-bits)
2105   //    c_rarg2    -  dst oop
2106   //    c_rarg3    -  dst_pos (32-bits)
2107   //    c_rarg4    -  element count (32-bits)
2108   //
2109   //  Output:
2110   //    r0 ==  0  -  success
2111   //    r0 == -1^K - failure, where K is partial transfer count
2112   //
2113   address generate_generic_copy(const char *name,
2114                                 address byte_copy_entry, address short_copy_entry,
2115                                 address int_copy_entry, address oop_copy_entry,
2116                                 address long_copy_entry, address checkcast_copy_entry) {
2117 
2118     Label L_failed, L_objArray;
2119     Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
2120 
2121     // Input registers
2122     const Register src        = c_rarg0;  // source array oop
2123     const Register src_pos    = c_rarg1;  // source position
2124     const Register dst        = c_rarg2;  // destination array oop
2125     const Register dst_pos    = c_rarg3;  // destination position
2126     const Register length     = c_rarg4;
2127 
2128 
2129     // Registers used as temps
2130     const Register dst_klass  = c_rarg5;
2131 
2132     __ align(CodeEntryAlignment);
2133 
2134     StubCodeMark mark(this, "StubRoutines", name);
2135 
2136     address start = __ pc();
2137 
2138     __ enter(); // required for proper stackwalking of RuntimeStub frame
2139 
2140     // bump this on entry, not on exit:
2141     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
2142 
2143     //-----------------------------------------------------------------------
2144     // Assembler stub will be used for this call to arraycopy
2145     // if the following conditions are met:
2146     //
2147     // (1) src and dst must not be null.
2148     // (2) src_pos must not be negative.
2149     // (3) dst_pos must not be negative.
2150     // (4) length  must not be negative.
2151     // (5) src klass and dst klass should be the same and not null.
2152     // (6) src and dst should be arrays.
2153     // (7) src_pos + length must not exceed length of src.
2154     // (8) dst_pos + length must not exceed length of dst.
2155     //
2156 
2157     //  if (src == nullptr) return -1;
2158     __ cbz(src, L_failed);
2159 
2160     //  if (src_pos < 0) return -1;
2161     __ tbnz(src_pos, 31, L_failed);  // i.e. sign bit set
2162 
2163     //  if (dst == nullptr) return -1;
2164     __ cbz(dst, L_failed);
2165 
2166     //  if (dst_pos < 0) return -1;
2167     __ tbnz(dst_pos, 31, L_failed);  // i.e. sign bit set
2168 
2169     // registers used as temp
2170     const Register scratch_length    = r16; // elements count to copy
2171     const Register scratch_src_klass = r17; // array klass
2172     const Register lh                = r15; // layout helper
2173 
2174     //  if (length < 0) return -1;
2175     __ movw(scratch_length, length);        // length (elements count, 32-bits value)
2176     __ tbnz(scratch_length, 31, L_failed);  // i.e. sign bit set
2177 
2178     __ load_klass(scratch_src_klass, src);
2179 #ifdef ASSERT
2180     //  assert(src->klass() != nullptr);
2181     {
2182       BLOCK_COMMENT("assert klasses not null {");
2183       Label L1, L2;
2184       __ cbnz(scratch_src_klass, L2);   // it is broken if klass is null
2185       __ bind(L1);
2186       __ stop("broken null klass");
2187       __ bind(L2);
2188       __ load_klass(rscratch1, dst);
2189       __ cbz(rscratch1, L1);     // this would be broken also
2190       BLOCK_COMMENT("} assert klasses not null done");
2191     }
2192 #endif
2193 
2194     // Load layout helper (32-bits)
2195     //
2196     //  |array_tag|     | header_size | element_type |     |log2_element_size|
2197     // 32        30    24            16              8     2                 0
2198     //
2199     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2200     //
2201 
2202     const int lh_offset = in_bytes(Klass::layout_helper_offset());
2203 
2204     // Handle objArrays completely differently...
2205     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2206     __ ldrw(lh, Address(scratch_src_klass, lh_offset));
2207     __ movw(rscratch1, objArray_lh);
2208     __ eorw(rscratch2, lh, rscratch1);
2209     __ cbzw(rscratch2, L_objArray);
2210 
2211     //  if (src->klass() != dst->klass()) return -1;
2212     __ load_klass(rscratch2, dst);
2213     __ eor(rscratch2, rscratch2, scratch_src_klass);
2214     __ cbnz(rscratch2, L_failed);
2215 
2216     //  if (!src->is_Array()) return -1;
2217     __ tbz(lh, 31, L_failed);  // i.e. (lh >= 0)
2218 
2219     // At this point, it is known to be a typeArray (array_tag 0x3).
2220 #ifdef ASSERT
2221     {
2222       BLOCK_COMMENT("assert primitive array {");
2223       Label L;
2224       __ movw(rscratch2, Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift);
2225       __ cmpw(lh, rscratch2);
2226       __ br(Assembler::GE, L);
2227       __ stop("must be a primitive array");
2228       __ bind(L);
2229       BLOCK_COMMENT("} assert primitive array done");
2230     }
2231 #endif
2232 
2233     arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2234                            rscratch2, L_failed);
2235 
2236     // TypeArrayKlass
2237     //
2238     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2239     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2240     //
2241 
2242     const Register rscratch1_offset = rscratch1;    // array offset
2243     const Register r15_elsize = lh; // element size
2244 
2245     __ ubfx(rscratch1_offset, lh, Klass::_lh_header_size_shift,
2246            exact_log2(Klass::_lh_header_size_mask+1));   // array_offset
2247     __ add(src, src, rscratch1_offset);           // src array offset
2248     __ add(dst, dst, rscratch1_offset);           // dst array offset
2249     BLOCK_COMMENT("choose copy loop based on element size");
2250 
2251     // next registers should be set before the jump to corresponding stub
2252     const Register from     = c_rarg0;  // source array address
2253     const Register to       = c_rarg1;  // destination array address
2254     const Register count    = c_rarg2;  // elements count
2255 
2256     // 'from', 'to', 'count' registers should be set in such order
2257     // since they are the same as 'src', 'src_pos', 'dst'.
2258 
2259     assert(Klass::_lh_log2_element_size_shift == 0, "fix this code");
2260 
2261     // The possible values of elsize are 0-3, i.e. exact_log2(element
2262     // size in bytes).  We do a simple bitwise binary search.
2263   __ BIND(L_copy_bytes);
2264     __ tbnz(r15_elsize, 1, L_copy_ints);
2265     __ tbnz(r15_elsize, 0, L_copy_shorts);
2266     __ lea(from, Address(src, src_pos));// src_addr
2267     __ lea(to,   Address(dst, dst_pos));// dst_addr
2268     __ movw(count, scratch_length); // length
2269     __ b(RuntimeAddress(byte_copy_entry));
2270 
2271   __ BIND(L_copy_shorts);
2272     __ lea(from, Address(src, src_pos, Address::lsl(1)));// src_addr
2273     __ lea(to,   Address(dst, dst_pos, Address::lsl(1)));// dst_addr
2274     __ movw(count, scratch_length); // length
2275     __ b(RuntimeAddress(short_copy_entry));
2276 
2277   __ BIND(L_copy_ints);
2278     __ tbnz(r15_elsize, 0, L_copy_longs);
2279     __ lea(from, Address(src, src_pos, Address::lsl(2)));// src_addr
2280     __ lea(to,   Address(dst, dst_pos, Address::lsl(2)));// dst_addr
2281     __ movw(count, scratch_length); // length
2282     __ b(RuntimeAddress(int_copy_entry));
2283 
2284   __ BIND(L_copy_longs);
2285 #ifdef ASSERT
2286     {
2287       BLOCK_COMMENT("assert long copy {");
2288       Label L;
2289       __ andw(lh, lh, Klass::_lh_log2_element_size_mask); // lh -> r15_elsize
2290       __ cmpw(r15_elsize, LogBytesPerLong);
2291       __ br(Assembler::EQ, L);
2292       __ stop("must be long copy, but elsize is wrong");
2293       __ bind(L);
2294       BLOCK_COMMENT("} assert long copy done");
2295     }
2296 #endif
2297     __ lea(from, Address(src, src_pos, Address::lsl(3)));// src_addr
2298     __ lea(to,   Address(dst, dst_pos, Address::lsl(3)));// dst_addr
2299     __ movw(count, scratch_length); // length
2300     __ b(RuntimeAddress(long_copy_entry));
2301 
2302     // ObjArrayKlass
2303   __ BIND(L_objArray);
2304     // live at this point:  scratch_src_klass, scratch_length, src[_pos], dst[_pos]
2305 
2306     Label L_plain_copy, L_checkcast_copy;
2307     //  test array classes for subtyping
2308     __ load_klass(r15, dst);
2309     __ cmp(scratch_src_klass, r15); // usual case is exact equality
2310     __ br(Assembler::NE, L_checkcast_copy);
2311 
2312     // Identically typed arrays can be copied without element-wise checks.
2313     arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2314                            rscratch2, L_failed);
2315 
2316     __ lea(from, Address(src, src_pos, Address::lsl(LogBytesPerHeapOop)));
2317     __ add(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2318     __ lea(to, Address(dst, dst_pos, Address::lsl(LogBytesPerHeapOop)));
2319     __ add(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2320     __ movw(count, scratch_length); // length
2321   __ BIND(L_plain_copy);
2322     __ b(RuntimeAddress(oop_copy_entry));
2323 
2324   __ BIND(L_checkcast_copy);
2325     // live at this point:  scratch_src_klass, scratch_length, r15 (dst_klass)
2326     {
2327       // Before looking at dst.length, make sure dst is also an objArray.
2328       __ ldrw(rscratch1, Address(r15, lh_offset));
2329       __ movw(rscratch2, objArray_lh);
2330       __ eorw(rscratch1, rscratch1, rscratch2);
2331       __ cbnzw(rscratch1, L_failed);
2332 
2333       // It is safe to examine both src.length and dst.length.
2334       arraycopy_range_checks(src, src_pos, dst, dst_pos, scratch_length,
2335                              r15, L_failed);
2336 
2337       __ load_klass(dst_klass, dst); // reload
2338 
2339       // Marshal the base address arguments now, freeing registers.
2340       __ lea(from, Address(src, src_pos, Address::lsl(LogBytesPerHeapOop)));
2341       __ add(from, from, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2342       __ lea(to, Address(dst, dst_pos, Address::lsl(LogBytesPerHeapOop)));
2343       __ add(to, to, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2344       __ movw(count, length);           // length (reloaded)
2345       Register sco_temp = c_rarg3;      // this register is free now
2346       assert_different_registers(from, to, count, sco_temp,
2347                                  dst_klass, scratch_src_klass);
2348       // assert_clean_int(count, sco_temp);
2349 
2350       // Generate the type check.
2351       const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2352       __ ldrw(sco_temp, Address(dst_klass, sco_offset));
2353 
2354       // Smashes rscratch1, rscratch2
2355       generate_type_check(scratch_src_klass, sco_temp, dst_klass, L_plain_copy);
2356 
2357       // Fetch destination element klass from the ObjArrayKlass header.
2358       int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2359       __ ldr(dst_klass, Address(dst_klass, ek_offset));
2360       __ ldrw(sco_temp, Address(dst_klass, sco_offset));
2361 
2362       // the checkcast_copy loop needs two extra arguments:
2363       assert(c_rarg3 == sco_temp, "#3 already in place");
2364       // Set up arguments for checkcast_copy_entry.
2365       __ mov(c_rarg4, dst_klass);  // dst.klass.element_klass
2366       __ b(RuntimeAddress(checkcast_copy_entry));
2367     }
2368 
2369   __ BIND(L_failed);
2370     __ mov(r0, -1);
2371     __ leave();   // required for proper stackwalking of RuntimeStub frame
2372     __ ret(lr);
2373 
2374     return start;
2375   }
2376 
2377   //
2378   // Generate stub for array fill. If "aligned" is true, the
2379   // "to" address is assumed to be heapword aligned.
2380   //
2381   // Arguments for generated stub:
2382   //   to:    c_rarg0
2383   //   value: c_rarg1
2384   //   count: c_rarg2 treated as signed
2385   //
2386   address generate_fill(BasicType t, bool aligned, const char *name) {
2387     __ align(CodeEntryAlignment);
2388     StubCodeMark mark(this, "StubRoutines", name);
2389     address start = __ pc();
2390 
2391     BLOCK_COMMENT("Entry:");
2392 
2393     const Register to        = c_rarg0;  // source array address
2394     const Register value     = c_rarg1;  // value
2395     const Register count     = c_rarg2;  // elements count
2396 
2397     const Register bz_base = r10;        // base for block_zero routine
2398     const Register cnt_words = r11;      // temp register
2399 
2400     __ enter();
2401 
2402     Label L_fill_elements, L_exit1;
2403 
2404     int shift = -1;
2405     switch (t) {
2406       case T_BYTE:
2407         shift = 0;
2408         __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
2409         __ bfi(value, value, 8, 8);   // 8 bit -> 16 bit
2410         __ bfi(value, value, 16, 16); // 16 bit -> 32 bit
2411         __ br(Assembler::LO, L_fill_elements);
2412         break;
2413       case T_SHORT:
2414         shift = 1;
2415         __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
2416         __ bfi(value, value, 16, 16); // 16 bit -> 32 bit
2417         __ br(Assembler::LO, L_fill_elements);
2418         break;
2419       case T_INT:
2420         shift = 2;
2421         __ cmpw(count, 8 >> shift); // Short arrays (< 8 bytes) fill by element
2422         __ br(Assembler::LO, L_fill_elements);
2423         break;
2424       default: ShouldNotReachHere();
2425     }
2426 
2427     // Align source address at 8 bytes address boundary.
2428     Label L_skip_align1, L_skip_align2, L_skip_align4;
2429     if (!aligned) {
2430       switch (t) {
2431         case T_BYTE:
2432           // One byte misalignment happens only for byte arrays.
2433           __ tbz(to, 0, L_skip_align1);
2434           __ strb(value, Address(__ post(to, 1)));
2435           __ subw(count, count, 1);
2436           __ bind(L_skip_align1);
2437           // Fallthrough
2438         case T_SHORT:
2439           // Two bytes misalignment happens only for byte and short (char) arrays.
2440           __ tbz(to, 1, L_skip_align2);
2441           __ strh(value, Address(__ post(to, 2)));
2442           __ subw(count, count, 2 >> shift);
2443           __ bind(L_skip_align2);
2444           // Fallthrough
2445         case T_INT:
2446           // Align to 8 bytes, we know we are 4 byte aligned to start.
2447           __ tbz(to, 2, L_skip_align4);
2448           __ strw(value, Address(__ post(to, 4)));
2449           __ subw(count, count, 4 >> shift);
2450           __ bind(L_skip_align4);
2451           break;
2452         default: ShouldNotReachHere();
2453       }
2454     }
2455 
2456     //
2457     //  Fill large chunks
2458     //
2459     __ lsrw(cnt_words, count, 3 - shift); // number of words
2460     __ bfi(value, value, 32, 32);         // 32 bit -> 64 bit
2461     __ subw(count, count, cnt_words, Assembler::LSL, 3 - shift);
2462     if (UseBlockZeroing) {
2463       Label non_block_zeroing, rest;
2464       // If the fill value is zero we can use the fast zero_words().
2465       __ cbnz(value, non_block_zeroing);
2466       __ mov(bz_base, to);
2467       __ add(to, to, cnt_words, Assembler::LSL, LogBytesPerWord);
2468       address tpc = __ zero_words(bz_base, cnt_words);
2469       if (tpc == nullptr) {
2470         fatal("CodeCache is full at generate_fill");
2471       }
2472       __ b(rest);
2473       __ bind(non_block_zeroing);
2474       __ fill_words(to, cnt_words, value);
2475       __ bind(rest);
2476     } else {
2477       __ fill_words(to, cnt_words, value);
2478     }
2479 
2480     // Remaining count is less than 8 bytes. Fill it by a single store.
2481     // Note that the total length is no less than 8 bytes.
2482     if (t == T_BYTE || t == T_SHORT) {
2483       Label L_exit1;
2484       __ cbzw(count, L_exit1);
2485       __ add(to, to, count, Assembler::LSL, shift); // points to the end
2486       __ str(value, Address(to, -8));    // overwrite some elements
2487       __ bind(L_exit1);
2488       __ leave();
2489       __ ret(lr);
2490     }
2491 
2492     // Handle copies less than 8 bytes.
2493     Label L_fill_2, L_fill_4, L_exit2;
2494     __ bind(L_fill_elements);
2495     switch (t) {
2496       case T_BYTE:
2497         __ tbz(count, 0, L_fill_2);
2498         __ strb(value, Address(__ post(to, 1)));
2499         __ bind(L_fill_2);
2500         __ tbz(count, 1, L_fill_4);
2501         __ strh(value, Address(__ post(to, 2)));
2502         __ bind(L_fill_4);
2503         __ tbz(count, 2, L_exit2);
2504         __ strw(value, Address(to));
2505         break;
2506       case T_SHORT:
2507         __ tbz(count, 0, L_fill_4);
2508         __ strh(value, Address(__ post(to, 2)));
2509         __ bind(L_fill_4);
2510         __ tbz(count, 1, L_exit2);
2511         __ strw(value, Address(to));
2512         break;
2513       case T_INT:
2514         __ cbzw(count, L_exit2);
2515         __ strw(value, Address(to));
2516         break;
2517       default: ShouldNotReachHere();
2518     }
2519     __ bind(L_exit2);
2520     __ leave();
2521     __ ret(lr);
2522     return start;
2523   }
2524 
2525   address generate_data_cache_writeback() {
2526     const Register line        = c_rarg0;  // address of line to write back
2527 
2528     __ align(CodeEntryAlignment);
2529 
2530     StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback");
2531 
2532     address start = __ pc();
2533     __ enter();
2534     __ cache_wb(Address(line, 0));
2535     __ leave();
2536     __ ret(lr);
2537 
2538     return start;
2539   }
2540 
2541   address generate_data_cache_writeback_sync() {
2542     const Register is_pre     = c_rarg0;  // pre or post sync
2543 
2544     __ align(CodeEntryAlignment);
2545 
2546     StubCodeMark mark(this, "StubRoutines", "_data_cache_writeback_sync");
2547 
2548     // pre wbsync is a no-op
2549     // post wbsync translates to an sfence
2550 
2551     Label skip;
2552     address start = __ pc();
2553     __ enter();
2554     __ cbnz(is_pre, skip);
2555     __ cache_wbsync(false);
2556     __ bind(skip);
2557     __ leave();
2558     __ ret(lr);
2559 
2560     return start;
2561   }
2562 
2563   void generate_arraycopy_stubs() {
2564     address entry;
2565     address entry_jbyte_arraycopy;
2566     address entry_jshort_arraycopy;
2567     address entry_jint_arraycopy;
2568     address entry_oop_arraycopy;
2569     address entry_jlong_arraycopy;
2570     address entry_checkcast_arraycopy;
2571 
2572     generate_copy_longs(IN_HEAP | IS_ARRAY, T_BYTE, copy_f, r0, r1, r15, copy_forwards);
2573     generate_copy_longs(IN_HEAP | IS_ARRAY, T_BYTE, copy_b, r0, r1, r15, copy_backwards);
2574 
2575     generate_copy_longs(IN_HEAP | IS_ARRAY, T_OBJECT, copy_obj_f, r0, r1, r15, copy_forwards);
2576     generate_copy_longs(IN_HEAP | IS_ARRAY, T_OBJECT, copy_obj_b, r0, r1, r15, copy_backwards);
2577 
2578     generate_copy_longs(IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, T_OBJECT, copy_obj_uninit_f, r0, r1, r15, copy_forwards);
2579     generate_copy_longs(IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, T_OBJECT, copy_obj_uninit_b, r0, r1, r15, copy_backwards);
2580 
2581     StubRoutines::aarch64::_zero_blocks = generate_zero_blocks();
2582 
2583     //*** jbyte
2584     // Always need aligned and unaligned versions
2585     StubRoutines::_jbyte_disjoint_arraycopy         = generate_disjoint_byte_copy(false, &entry,
2586                                                                                   "jbyte_disjoint_arraycopy");
2587     StubRoutines::_jbyte_arraycopy                  = generate_conjoint_byte_copy(false, entry,
2588                                                                                   &entry_jbyte_arraycopy,
2589                                                                                   "jbyte_arraycopy");
2590     StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(true, &entry,
2591                                                                                   "arrayof_jbyte_disjoint_arraycopy");
2592     StubRoutines::_arrayof_jbyte_arraycopy          = generate_conjoint_byte_copy(true, entry, nullptr,
2593                                                                                   "arrayof_jbyte_arraycopy");
2594 
2595     //*** jshort
2596     // Always need aligned and unaligned versions
2597     StubRoutines::_jshort_disjoint_arraycopy         = generate_disjoint_short_copy(false, &entry,
2598                                                                                     "jshort_disjoint_arraycopy");
2599     StubRoutines::_jshort_arraycopy                  = generate_conjoint_short_copy(false, entry,
2600                                                                                     &entry_jshort_arraycopy,
2601                                                                                     "jshort_arraycopy");
2602     StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_short_copy(true, &entry,
2603                                                                                     "arrayof_jshort_disjoint_arraycopy");
2604     StubRoutines::_arrayof_jshort_arraycopy          = generate_conjoint_short_copy(true, entry, nullptr,
2605                                                                                     "arrayof_jshort_arraycopy");
2606 
2607     //*** jint
2608     // Aligned versions
2609     StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_int_copy(true, &entry,
2610                                                                                 "arrayof_jint_disjoint_arraycopy");
2611     StubRoutines::_arrayof_jint_arraycopy          = generate_conjoint_int_copy(true, entry, &entry_jint_arraycopy,
2612                                                                                 "arrayof_jint_arraycopy");
2613     // In 64 bit we need both aligned and unaligned versions of jint arraycopy.
2614     // entry_jint_arraycopy always points to the unaligned version
2615     StubRoutines::_jint_disjoint_arraycopy         = generate_disjoint_int_copy(false, &entry,
2616                                                                                 "jint_disjoint_arraycopy");
2617     StubRoutines::_jint_arraycopy                  = generate_conjoint_int_copy(false, entry,
2618                                                                                 &entry_jint_arraycopy,
2619                                                                                 "jint_arraycopy");
2620 
2621     //*** jlong
2622     // It is always aligned
2623     StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_long_copy(true, &entry,
2624                                                                                   "arrayof_jlong_disjoint_arraycopy");
2625     StubRoutines::_arrayof_jlong_arraycopy          = generate_conjoint_long_copy(true, entry, &entry_jlong_arraycopy,
2626                                                                                   "arrayof_jlong_arraycopy");
2627     StubRoutines::_jlong_disjoint_arraycopy         = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
2628     StubRoutines::_jlong_arraycopy                  = StubRoutines::_arrayof_jlong_arraycopy;
2629 
2630     //*** oops
2631     {
2632       // With compressed oops we need unaligned versions; notice that
2633       // we overwrite entry_oop_arraycopy.
2634       bool aligned = !UseCompressedOops;
2635 
2636       StubRoutines::_arrayof_oop_disjoint_arraycopy
2637         = generate_disjoint_oop_copy(aligned, &entry, "arrayof_oop_disjoint_arraycopy",
2638                                      /*dest_uninitialized*/false);
2639       StubRoutines::_arrayof_oop_arraycopy
2640         = generate_conjoint_oop_copy(aligned, entry, &entry_oop_arraycopy, "arrayof_oop_arraycopy",
2641                                      /*dest_uninitialized*/false);
2642       // Aligned versions without pre-barriers
2643       StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit
2644         = generate_disjoint_oop_copy(aligned, &entry, "arrayof_oop_disjoint_arraycopy_uninit",
2645                                      /*dest_uninitialized*/true);
2646       StubRoutines::_arrayof_oop_arraycopy_uninit
2647         = generate_conjoint_oop_copy(aligned, entry, nullptr, "arrayof_oop_arraycopy_uninit",
2648                                      /*dest_uninitialized*/true);
2649     }
2650 
2651     StubRoutines::_oop_disjoint_arraycopy            = StubRoutines::_arrayof_oop_disjoint_arraycopy;
2652     StubRoutines::_oop_arraycopy                     = StubRoutines::_arrayof_oop_arraycopy;
2653     StubRoutines::_oop_disjoint_arraycopy_uninit     = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
2654     StubRoutines::_oop_arraycopy_uninit              = StubRoutines::_arrayof_oop_arraycopy_uninit;
2655 
2656     StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2657     StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", nullptr,
2658                                                                         /*dest_uninitialized*/true);
2659 
2660     StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy("unsafe_arraycopy",
2661                                                               entry_jbyte_arraycopy,
2662                                                               entry_jshort_arraycopy,
2663                                                               entry_jint_arraycopy,
2664                                                               entry_jlong_arraycopy);
2665 
2666     StubRoutines::_generic_arraycopy   = generate_generic_copy("generic_arraycopy",
2667                                                                entry_jbyte_arraycopy,
2668                                                                entry_jshort_arraycopy,
2669                                                                entry_jint_arraycopy,
2670                                                                entry_oop_arraycopy,
2671                                                                entry_jlong_arraycopy,
2672                                                                entry_checkcast_arraycopy);
2673 
2674     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2675     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2676     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2677     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2678     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2679     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2680   }
2681 
2682   void generate_math_stubs() { Unimplemented(); }
2683 
2684   // Arguments:
2685   //
2686   // Inputs:
2687   //   c_rarg0   - source byte array address
2688   //   c_rarg1   - destination byte array address
2689   //   c_rarg2   - K (key) in little endian int array
2690   //
2691   address generate_aescrypt_encryptBlock() {
2692     __ align(CodeEntryAlignment);
2693     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2694 
2695     const Register from        = c_rarg0;  // source array address
2696     const Register to          = c_rarg1;  // destination array address
2697     const Register key         = c_rarg2;  // key array address
2698     const Register keylen      = rscratch1;
2699 
2700     address start = __ pc();
2701     __ enter();
2702 
2703     __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2704 
2705     __ aesenc_loadkeys(key, keylen);
2706     __ aesecb_encrypt(from, to, keylen);
2707 
2708     __ mov(r0, 0);
2709 
2710     __ leave();
2711     __ ret(lr);
2712 
2713     return start;
2714   }
2715 
2716   // Arguments:
2717   //
2718   // Inputs:
2719   //   c_rarg0   - source byte array address
2720   //   c_rarg1   - destination byte array address
2721   //   c_rarg2   - K (key) in little endian int array
2722   //
2723   address generate_aescrypt_decryptBlock() {
2724     assert(UseAES, "need AES cryptographic extension support");
2725     __ align(CodeEntryAlignment);
2726     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
2727     Label L_doLast;
2728 
2729     const Register from        = c_rarg0;  // source array address
2730     const Register to          = c_rarg1;  // destination array address
2731     const Register key         = c_rarg2;  // key array address
2732     const Register keylen      = rscratch1;
2733 
2734     address start = __ pc();
2735     __ enter(); // required for proper stackwalking of RuntimeStub frame
2736 
2737     __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2738 
2739     __ aesecb_decrypt(from, to, key, keylen);
2740 
2741     __ mov(r0, 0);
2742 
2743     __ leave();
2744     __ ret(lr);
2745 
2746     return start;
2747   }
2748 
2749   // Arguments:
2750   //
2751   // Inputs:
2752   //   c_rarg0   - source byte array address
2753   //   c_rarg1   - destination byte array address
2754   //   c_rarg2   - K (key) in little endian int array
2755   //   c_rarg3   - r vector byte array address
2756   //   c_rarg4   - input length
2757   //
2758   // Output:
2759   //   x0        - input length
2760   //
2761   address generate_cipherBlockChaining_encryptAESCrypt() {
2762     assert(UseAES, "need AES cryptographic extension support");
2763     __ align(CodeEntryAlignment);
2764     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
2765 
2766     Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52;
2767 
2768     const Register from        = c_rarg0;  // source array address
2769     const Register to          = c_rarg1;  // destination array address
2770     const Register key         = c_rarg2;  // key array address
2771     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
2772                                            // and left with the results of the last encryption block
2773     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
2774     const Register keylen      = rscratch1;
2775 
2776     address start = __ pc();
2777 
2778       __ enter();
2779 
2780       __ movw(rscratch2, len_reg);
2781 
2782       __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2783 
2784       __ ld1(v0, __ T16B, rvec);
2785 
2786       __ cmpw(keylen, 52);
2787       __ br(Assembler::CC, L_loadkeys_44);
2788       __ br(Assembler::EQ, L_loadkeys_52);
2789 
2790       __ ld1(v17, v18, __ T16B, __ post(key, 32));
2791       __ rev32(v17, __ T16B, v17);
2792       __ rev32(v18, __ T16B, v18);
2793     __ BIND(L_loadkeys_52);
2794       __ ld1(v19, v20, __ T16B, __ post(key, 32));
2795       __ rev32(v19, __ T16B, v19);
2796       __ rev32(v20, __ T16B, v20);
2797     __ BIND(L_loadkeys_44);
2798       __ ld1(v21, v22, v23, v24, __ T16B, __ post(key, 64));
2799       __ rev32(v21, __ T16B, v21);
2800       __ rev32(v22, __ T16B, v22);
2801       __ rev32(v23, __ T16B, v23);
2802       __ rev32(v24, __ T16B, v24);
2803       __ ld1(v25, v26, v27, v28, __ T16B, __ post(key, 64));
2804       __ rev32(v25, __ T16B, v25);
2805       __ rev32(v26, __ T16B, v26);
2806       __ rev32(v27, __ T16B, v27);
2807       __ rev32(v28, __ T16B, v28);
2808       __ ld1(v29, v30, v31, __ T16B, key);
2809       __ rev32(v29, __ T16B, v29);
2810       __ rev32(v30, __ T16B, v30);
2811       __ rev32(v31, __ T16B, v31);
2812 
2813     __ BIND(L_aes_loop);
2814       __ ld1(v1, __ T16B, __ post(from, 16));
2815       __ eor(v0, __ T16B, v0, v1);
2816 
2817       __ br(Assembler::CC, L_rounds_44);
2818       __ br(Assembler::EQ, L_rounds_52);
2819 
2820       __ aese(v0, v17); __ aesmc(v0, v0);
2821       __ aese(v0, v18); __ aesmc(v0, v0);
2822     __ BIND(L_rounds_52);
2823       __ aese(v0, v19); __ aesmc(v0, v0);
2824       __ aese(v0, v20); __ aesmc(v0, v0);
2825     __ BIND(L_rounds_44);
2826       __ aese(v0, v21); __ aesmc(v0, v0);
2827       __ aese(v0, v22); __ aesmc(v0, v0);
2828       __ aese(v0, v23); __ aesmc(v0, v0);
2829       __ aese(v0, v24); __ aesmc(v0, v0);
2830       __ aese(v0, v25); __ aesmc(v0, v0);
2831       __ aese(v0, v26); __ aesmc(v0, v0);
2832       __ aese(v0, v27); __ aesmc(v0, v0);
2833       __ aese(v0, v28); __ aesmc(v0, v0);
2834       __ aese(v0, v29); __ aesmc(v0, v0);
2835       __ aese(v0, v30);
2836       __ eor(v0, __ T16B, v0, v31);
2837 
2838       __ st1(v0, __ T16B, __ post(to, 16));
2839 
2840       __ subw(len_reg, len_reg, 16);
2841       __ cbnzw(len_reg, L_aes_loop);
2842 
2843       __ st1(v0, __ T16B, rvec);
2844 
2845       __ mov(r0, rscratch2);
2846 
2847       __ leave();
2848       __ ret(lr);
2849 
2850       return start;
2851   }
2852 
2853   // Arguments:
2854   //
2855   // Inputs:
2856   //   c_rarg0   - source byte array address
2857   //   c_rarg1   - destination byte array address
2858   //   c_rarg2   - K (key) in little endian int array
2859   //   c_rarg3   - r vector byte array address
2860   //   c_rarg4   - input length
2861   //
2862   // Output:
2863   //   r0        - input length
2864   //
2865   address generate_cipherBlockChaining_decryptAESCrypt() {
2866     assert(UseAES, "need AES cryptographic extension support");
2867     __ align(CodeEntryAlignment);
2868     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
2869 
2870     Label L_loadkeys_44, L_loadkeys_52, L_aes_loop, L_rounds_44, L_rounds_52;
2871 
2872     const Register from        = c_rarg0;  // source array address
2873     const Register to          = c_rarg1;  // destination array address
2874     const Register key         = c_rarg2;  // key array address
2875     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
2876                                            // and left with the results of the last encryption block
2877     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
2878     const Register keylen      = rscratch1;
2879 
2880     address start = __ pc();
2881 
2882       __ enter();
2883 
2884       __ movw(rscratch2, len_reg);
2885 
2886       __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2887 
2888       __ ld1(v2, __ T16B, rvec);
2889 
2890       __ ld1(v31, __ T16B, __ post(key, 16));
2891       __ rev32(v31, __ T16B, v31);
2892 
2893       __ cmpw(keylen, 52);
2894       __ br(Assembler::CC, L_loadkeys_44);
2895       __ br(Assembler::EQ, L_loadkeys_52);
2896 
2897       __ ld1(v17, v18, __ T16B, __ post(key, 32));
2898       __ rev32(v17, __ T16B, v17);
2899       __ rev32(v18, __ T16B, v18);
2900     __ BIND(L_loadkeys_52);
2901       __ ld1(v19, v20, __ T16B, __ post(key, 32));
2902       __ rev32(v19, __ T16B, v19);
2903       __ rev32(v20, __ T16B, v20);
2904     __ BIND(L_loadkeys_44);
2905       __ ld1(v21, v22, v23, v24, __ T16B, __ post(key, 64));
2906       __ rev32(v21, __ T16B, v21);
2907       __ rev32(v22, __ T16B, v22);
2908       __ rev32(v23, __ T16B, v23);
2909       __ rev32(v24, __ T16B, v24);
2910       __ ld1(v25, v26, v27, v28, __ T16B, __ post(key, 64));
2911       __ rev32(v25, __ T16B, v25);
2912       __ rev32(v26, __ T16B, v26);
2913       __ rev32(v27, __ T16B, v27);
2914       __ rev32(v28, __ T16B, v28);
2915       __ ld1(v29, v30, __ T16B, key);
2916       __ rev32(v29, __ T16B, v29);
2917       __ rev32(v30, __ T16B, v30);
2918 
2919     __ BIND(L_aes_loop);
2920       __ ld1(v0, __ T16B, __ post(from, 16));
2921       __ orr(v1, __ T16B, v0, v0);
2922 
2923       __ br(Assembler::CC, L_rounds_44);
2924       __ br(Assembler::EQ, L_rounds_52);
2925 
2926       __ aesd(v0, v17); __ aesimc(v0, v0);
2927       __ aesd(v0, v18); __ aesimc(v0, v0);
2928     __ BIND(L_rounds_52);
2929       __ aesd(v0, v19); __ aesimc(v0, v0);
2930       __ aesd(v0, v20); __ aesimc(v0, v0);
2931     __ BIND(L_rounds_44);
2932       __ aesd(v0, v21); __ aesimc(v0, v0);
2933       __ aesd(v0, v22); __ aesimc(v0, v0);
2934       __ aesd(v0, v23); __ aesimc(v0, v0);
2935       __ aesd(v0, v24); __ aesimc(v0, v0);
2936       __ aesd(v0, v25); __ aesimc(v0, v0);
2937       __ aesd(v0, v26); __ aesimc(v0, v0);
2938       __ aesd(v0, v27); __ aesimc(v0, v0);
2939       __ aesd(v0, v28); __ aesimc(v0, v0);
2940       __ aesd(v0, v29); __ aesimc(v0, v0);
2941       __ aesd(v0, v30);
2942       __ eor(v0, __ T16B, v0, v31);
2943       __ eor(v0, __ T16B, v0, v2);
2944 
2945       __ st1(v0, __ T16B, __ post(to, 16));
2946       __ orr(v2, __ T16B, v1, v1);
2947 
2948       __ subw(len_reg, len_reg, 16);
2949       __ cbnzw(len_reg, L_aes_loop);
2950 
2951       __ st1(v2, __ T16B, rvec);
2952 
2953       __ mov(r0, rscratch2);
2954 
2955       __ leave();
2956       __ ret(lr);
2957 
2958     return start;
2959   }
2960 
2961   // Big-endian 128-bit + 64-bit -> 128-bit addition.
2962   // Inputs: 128-bits. in is preserved.
2963   // The least-significant 64-bit word is in the upper dword of each vector.
2964   // inc (the 64-bit increment) is preserved. Its lower dword must be zero.
2965   // Output: result
2966   void be_add_128_64(FloatRegister result, FloatRegister in,
2967                      FloatRegister inc, FloatRegister tmp) {
2968     assert_different_registers(result, tmp, inc);
2969 
2970     __ addv(result, __ T2D, in, inc);      // Add inc to the least-significant dword of
2971                                            // input
2972     __ cm(__ HI, tmp, __ T2D, inc, result);// Check for result overflowing
2973     __ ext(tmp, __ T16B, tmp, tmp, 0x08);  // Swap LSD of comparison result to MSD and
2974                                            // MSD == 0 (must be!) to LSD
2975     __ subv(result, __ T2D, result, tmp);  // Subtract -1 from MSD if there was an overflow
2976   }
2977 
2978   // CTR AES crypt.
2979   // Arguments:
2980   //
2981   // Inputs:
2982   //   c_rarg0   - source byte array address
2983   //   c_rarg1   - destination byte array address
2984   //   c_rarg2   - K (key) in little endian int array
2985   //   c_rarg3   - counter vector byte array address
2986   //   c_rarg4   - input length
2987   //   c_rarg5   - saved encryptedCounter start
2988   //   c_rarg6   - saved used length
2989   //
2990   // Output:
2991   //   r0       - input length
2992   //
2993   address generate_counterMode_AESCrypt() {
2994     const Register in = c_rarg0;
2995     const Register out = c_rarg1;
2996     const Register key = c_rarg2;
2997     const Register counter = c_rarg3;
2998     const Register saved_len = c_rarg4, len = r10;
2999     const Register saved_encrypted_ctr = c_rarg5;
3000     const Register used_ptr = c_rarg6, used = r12;
3001 
3002     const Register offset = r7;
3003     const Register keylen = r11;
3004 
3005     const unsigned char block_size = 16;
3006     const int bulk_width = 4;
3007     // NB: bulk_width can be 4 or 8. 8 gives slightly faster
3008     // performance with larger data sizes, but it also means that the
3009     // fast path isn't used until you have at least 8 blocks, and up
3010     // to 127 bytes of data will be executed on the slow path. For
3011     // that reason, and also so as not to blow away too much icache, 4
3012     // blocks seems like a sensible compromise.
3013 
3014     // Algorithm:
3015     //
3016     //    if (len == 0) {
3017     //        goto DONE;
3018     //    }
3019     //    int result = len;
3020     //    do {
3021     //        if (used >= blockSize) {
3022     //            if (len >= bulk_width * blockSize) {
3023     //                CTR_large_block();
3024     //                if (len == 0)
3025     //                    goto DONE;
3026     //            }
3027     //            for (;;) {
3028     //                16ByteVector v0 = counter;
3029     //                embeddedCipher.encryptBlock(v0, 0, encryptedCounter, 0);
3030     //                used = 0;
3031     //                if (len < blockSize)
3032     //                    break;    /* goto NEXT */
3033     //                16ByteVector v1 = load16Bytes(in, offset);
3034     //                v1 = v1 ^ encryptedCounter;
3035     //                store16Bytes(out, offset);
3036     //                used = blockSize;
3037     //                offset += blockSize;
3038     //                len -= blockSize;
3039     //                if (len == 0)
3040     //                    goto DONE;
3041     //            }
3042     //        }
3043     //      NEXT:
3044     //        out[outOff++] = (byte)(in[inOff++] ^ encryptedCounter[used++]);
3045     //        len--;
3046     //    } while (len != 0);
3047     //  DONE:
3048     //    return result;
3049     //
3050     // CTR_large_block()
3051     //    Wide bulk encryption of whole blocks.
3052 
3053     __ align(CodeEntryAlignment);
3054     StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
3055     const address start = __ pc();
3056     __ enter();
3057 
3058     Label DONE, CTR_large_block, large_block_return;
3059     __ ldrw(used, Address(used_ptr));
3060     __ cbzw(saved_len, DONE);
3061 
3062     __ mov(len, saved_len);
3063     __ mov(offset, 0);
3064 
3065     // Compute #rounds for AES based on the length of the key array
3066     __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3067 
3068     __ aesenc_loadkeys(key, keylen);
3069 
3070     {
3071       Label L_CTR_loop, NEXT;
3072 
3073       __ bind(L_CTR_loop);
3074 
3075       __ cmp(used, block_size);
3076       __ br(__ LO, NEXT);
3077 
3078       // Maybe we have a lot of data
3079       __ subsw(rscratch1, len, bulk_width * block_size);
3080       __ br(__ HS, CTR_large_block);
3081       __ BIND(large_block_return);
3082       __ cbzw(len, DONE);
3083 
3084       // Setup the counter
3085       __ movi(v4, __ T4S, 0);
3086       __ movi(v5, __ T4S, 1);
3087       __ ins(v4, __ S, v5, 2, 2); // v4 contains { 0, 1 }
3088 
3089       // 128-bit big-endian increment
3090       __ ld1(v0, __ T16B, counter);
3091       __ rev64(v16, __ T16B, v0);
3092       be_add_128_64(v16, v16, v4, /*tmp*/v5);
3093       __ rev64(v16, __ T16B, v16);
3094       __ st1(v16, __ T16B, counter);
3095       // Previous counter value is in v0
3096       // v4 contains { 0, 1 }
3097 
3098       {
3099         // We have fewer than bulk_width blocks of data left. Encrypt
3100         // them one by one until there is less than a full block
3101         // remaining, being careful to save both the encrypted counter
3102         // and the counter.
3103 
3104         Label inner_loop;
3105         __ bind(inner_loop);
3106         // Counter to encrypt is in v0
3107         __ aesecb_encrypt(noreg, noreg, keylen);
3108         __ st1(v0, __ T16B, saved_encrypted_ctr);
3109 
3110         // Do we have a remaining full block?
3111 
3112         __ mov(used, 0);
3113         __ cmp(len, block_size);
3114         __ br(__ LO, NEXT);
3115 
3116         // Yes, we have a full block
3117         __ ldrq(v1, Address(in, offset));
3118         __ eor(v1, __ T16B, v1, v0);
3119         __ strq(v1, Address(out, offset));
3120         __ mov(used, block_size);
3121         __ add(offset, offset, block_size);
3122 
3123         __ subw(len, len, block_size);
3124         __ cbzw(len, DONE);
3125 
3126         // Increment the counter, store it back
3127         __ orr(v0, __ T16B, v16, v16);
3128         __ rev64(v16, __ T16B, v16);
3129         be_add_128_64(v16, v16, v4, /*tmp*/v5);
3130         __ rev64(v16, __ T16B, v16);
3131         __ st1(v16, __ T16B, counter); // Save the incremented counter back
3132 
3133         __ b(inner_loop);
3134       }
3135 
3136       __ BIND(NEXT);
3137 
3138       // Encrypt a single byte, and loop.
3139       // We expect this to be a rare event.
3140       __ ldrb(rscratch1, Address(in, offset));
3141       __ ldrb(rscratch2, Address(saved_encrypted_ctr, used));
3142       __ eor(rscratch1, rscratch1, rscratch2);
3143       __ strb(rscratch1, Address(out, offset));
3144       __ add(offset, offset, 1);
3145       __ add(used, used, 1);
3146       __ subw(len, len,1);
3147       __ cbnzw(len, L_CTR_loop);
3148     }
3149 
3150     __ bind(DONE);
3151     __ strw(used, Address(used_ptr));
3152     __ mov(r0, saved_len);
3153 
3154     __ leave(); // required for proper stackwalking of RuntimeStub frame
3155     __ ret(lr);
3156 
3157     // Bulk encryption
3158 
3159     __ BIND (CTR_large_block);
3160     assert(bulk_width == 4 || bulk_width == 8, "must be");
3161 
3162     if (bulk_width == 8) {
3163       __ sub(sp, sp, 4 * 16);
3164       __ st1(v12, v13, v14, v15, __ T16B, Address(sp));
3165     }
3166     __ sub(sp, sp, 4 * 16);
3167     __ st1(v8, v9, v10, v11, __ T16B, Address(sp));
3168     RegSet saved_regs = (RegSet::of(in, out, offset)
3169                          + RegSet::of(saved_encrypted_ctr, used_ptr, len));
3170     __ push(saved_regs, sp);
3171     __ andr(len, len, -16 * bulk_width);  // 8/4 encryptions, 16 bytes per encryption
3172     __ add(in, in, offset);
3173     __ add(out, out, offset);
3174 
3175     // Keys should already be loaded into the correct registers
3176 
3177     __ ld1(v0, __ T16B, counter); // v0 contains the first counter
3178     __ rev64(v16, __ T16B, v0); // v16 contains byte-reversed counter
3179 
3180     // AES/CTR loop
3181     {
3182       Label L_CTR_loop;
3183       __ BIND(L_CTR_loop);
3184 
3185       // Setup the counters
3186       __ movi(v8, __ T4S, 0);
3187       __ movi(v9, __ T4S, 1);
3188       __ ins(v8, __ S, v9, 2, 2); // v8 contains { 0, 1 }
3189 
3190       for (int i = 0; i < bulk_width; i++) {
3191         FloatRegister v0_ofs = as_FloatRegister(v0->encoding() + i);
3192         __ rev64(v0_ofs, __ T16B, v16);
3193         be_add_128_64(v16, v16, v8, /*tmp*/v9);
3194       }
3195 
3196       __ ld1(v8, v9, v10, v11, __ T16B, __ post(in, 4 * 16));
3197 
3198       // Encrypt the counters
3199       __ aesecb_encrypt(noreg, noreg, keylen, v0, bulk_width);
3200 
3201       if (bulk_width == 8) {
3202         __ ld1(v12, v13, v14, v15, __ T16B, __ post(in, 4 * 16));
3203       }
3204 
3205       // XOR the encrypted counters with the inputs
3206       for (int i = 0; i < bulk_width; i++) {
3207         FloatRegister v0_ofs = as_FloatRegister(v0->encoding() + i);
3208         FloatRegister v8_ofs = as_FloatRegister(v8->encoding() + i);
3209         __ eor(v0_ofs, __ T16B, v0_ofs, v8_ofs);
3210       }
3211 
3212       // Write the encrypted data
3213       __ st1(v0, v1, v2, v3, __ T16B, __ post(out, 4 * 16));
3214       if (bulk_width == 8) {
3215         __ st1(v4, v5, v6, v7, __ T16B, __ post(out, 4 * 16));
3216       }
3217 
3218       __ subw(len, len, 16 * bulk_width);
3219       __ cbnzw(len, L_CTR_loop);
3220     }
3221 
3222     // Save the counter back where it goes
3223     __ rev64(v16, __ T16B, v16);
3224     __ st1(v16, __ T16B, counter);
3225 
3226     __ pop(saved_regs, sp);
3227 
3228     __ ld1(v8, v9, v10, v11, __ T16B, __ post(sp, 4 * 16));
3229     if (bulk_width == 8) {
3230       __ ld1(v12, v13, v14, v15, __ T16B, __ post(sp, 4 * 16));
3231     }
3232 
3233     __ andr(rscratch1, len, -16 * bulk_width);
3234     __ sub(len, len, rscratch1);
3235     __ add(offset, offset, rscratch1);
3236     __ mov(used, 16);
3237     __ strw(used, Address(used_ptr));
3238     __ b(large_block_return);
3239 
3240     return start;
3241   }
3242 
3243   // Vector AES Galois Counter Mode implementation. Parameters:
3244   //
3245   // in = c_rarg0
3246   // len = c_rarg1
3247   // ct = c_rarg2 - ciphertext that ghash will read (in for encrypt, out for decrypt)
3248   // out = c_rarg3
3249   // key = c_rarg4
3250   // state = c_rarg5 - GHASH.state
3251   // subkeyHtbl = c_rarg6 - powers of H
3252   // counter = c_rarg7 - 16 bytes of CTR
3253   // return - number of processed bytes
3254   address generate_galoisCounterMode_AESCrypt() {
3255     address ghash_polynomial = __ pc();
3256     __ emit_int64(0x87);  // The low-order bits of the field
3257                           // polynomial (i.e. p = z^7+z^2+z+1)
3258                           // repeated in the low and high parts of a
3259                           // 128-bit vector
3260     __ emit_int64(0x87);
3261 
3262     __ align(CodeEntryAlignment);
3263      StubCodeMark mark(this, "StubRoutines", "galoisCounterMode_AESCrypt");
3264     address start = __ pc();
3265     __ enter();
3266 
3267     const Register in = c_rarg0;
3268     const Register len = c_rarg1;
3269     const Register ct = c_rarg2;
3270     const Register out = c_rarg3;
3271     // and updated with the incremented counter in the end
3272 
3273     const Register key = c_rarg4;
3274     const Register state = c_rarg5;
3275 
3276     const Register subkeyHtbl = c_rarg6;
3277 
3278     const Register counter = c_rarg7;
3279 
3280     const Register keylen = r10;
3281     // Save state before entering routine
3282     __ sub(sp, sp, 4 * 16);
3283     __ st1(v12, v13, v14, v15, __ T16B, Address(sp));
3284     __ sub(sp, sp, 4 * 16);
3285     __ st1(v8, v9, v10, v11, __ T16B, Address(sp));
3286 
3287     // __ andr(len, len, -512);
3288     __ andr(len, len, -16 * 8);  // 8 encryptions, 16 bytes per encryption
3289     __ str(len, __ pre(sp, -2 * wordSize));
3290 
3291     Label DONE;
3292     __ cbz(len, DONE);
3293 
3294     // Compute #rounds for AES based on the length of the key array
3295     __ ldrw(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3296 
3297     __ aesenc_loadkeys(key, keylen);
3298     __ ld1(v0, __ T16B, counter); // v0 contains the first counter
3299     __ rev32(v16, __ T16B, v0); // v16 contains byte-reversed counter
3300 
3301     // AES/CTR loop
3302     {
3303       Label L_CTR_loop;
3304       __ BIND(L_CTR_loop);
3305 
3306       // Setup the counters
3307       __ movi(v8, __ T4S, 0);
3308       __ movi(v9, __ T4S, 1);
3309       __ ins(v8, __ S, v9, 3, 3); // v8 contains { 0, 0, 0, 1 }
3310 
3311       assert(v0->encoding() < v8->encoding(), "");
3312       for (int i = v0->encoding(); i < v8->encoding(); i++) {
3313         FloatRegister f = as_FloatRegister(i);
3314         __ rev32(f, __ T16B, v16);
3315         __ addv(v16, __ T4S, v16, v8);
3316       }
3317 
3318       __ ld1(v8, v9, v10, v11, __ T16B, __ post(in, 4 * 16));
3319 
3320       // Encrypt the counters
3321       __ aesecb_encrypt(noreg, noreg, keylen, v0, /*unrolls*/8);
3322 
3323       __ ld1(v12, v13, v14, v15, __ T16B, __ post(in, 4 * 16));
3324 
3325       // XOR the encrypted counters with the inputs
3326       for (int i = 0; i < 8; i++) {
3327         FloatRegister v0_ofs = as_FloatRegister(v0->encoding() + i);
3328         FloatRegister v8_ofs = as_FloatRegister(v8->encoding() + i);
3329         __ eor(v0_ofs, __ T16B, v0_ofs, v8_ofs);
3330       }
3331       __ st1(v0, v1, v2, v3, __ T16B, __ post(out, 4 * 16));
3332       __ st1(v4, v5, v6, v7, __ T16B, __ post(out, 4 * 16));
3333 
3334       __ subw(len, len, 16 * 8);
3335       __ cbnzw(len, L_CTR_loop);
3336     }
3337 
3338     __ rev32(v16, __ T16B, v16);
3339     __ st1(v16, __ T16B, counter);
3340 
3341     __ ldr(len, Address(sp));
3342     __ lsr(len, len, exact_log2(16));  // We want the count of blocks
3343 
3344     // GHASH/CTR loop
3345     __ ghash_processBlocks_wide(ghash_polynomial, state, subkeyHtbl, ct,
3346                                 len, /*unrolls*/4);
3347 
3348 #ifdef ASSERT
3349     { Label L;
3350       __ cmp(len, (unsigned char)0);
3351       __ br(Assembler::EQ, L);
3352       __ stop("stubGenerator: abort");
3353       __ bind(L);
3354   }
3355 #endif
3356 
3357   __ bind(DONE);
3358     // Return the number of bytes processed
3359     __ ldr(r0, __ post(sp, 2 * wordSize));
3360 
3361     __ ld1(v8, v9, v10, v11, __ T16B, __ post(sp, 4 * 16));
3362     __ ld1(v12, v13, v14, v15, __ T16B, __ post(sp, 4 * 16));
3363 
3364     __ leave(); // required for proper stackwalking of RuntimeStub frame
3365     __ ret(lr);
3366      return start;
3367   }
3368 
3369   class Cached64Bytes {
3370   private:
3371     MacroAssembler *_masm;
3372     Register _regs[8];
3373 
3374   public:
3375     Cached64Bytes(MacroAssembler *masm, RegSet rs): _masm(masm) {
3376       assert(rs.size() == 8, "%u registers are used to cache 16 4-byte data", rs.size());
3377       auto it = rs.begin();
3378       for (auto &r: _regs) {
3379         r = *it;
3380         ++it;
3381       }
3382     }
3383 
3384     void gen_loads(Register base) {
3385       for (int i = 0; i < 8; i += 2) {
3386         __ ldp(_regs[i], _regs[i + 1], Address(base, 8 * i));
3387       }
3388     }
3389 
3390     // Generate code extracting i-th unsigned word (4 bytes) from cached 64 bytes.
3391     void extract_u32(Register dest, int i) {
3392       __ ubfx(dest, _regs[i / 2], 32 * (i % 2), 32);
3393     }
3394   };
3395 
3396   // Utility routines for md5.
3397   // Clobbers r10 and r11.
3398   void md5_FF(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
3399               int k, int s, int t) {
3400     Register rscratch3 = r10;
3401     Register rscratch4 = r11;
3402 
3403     __ eorw(rscratch3, r3, r4);
3404     __ movw(rscratch2, t);
3405     __ andw(rscratch3, rscratch3, r2);
3406     __ addw(rscratch4, r1, rscratch2);
3407     reg_cache.extract_u32(rscratch1, k);
3408     __ eorw(rscratch3, rscratch3, r4);
3409     __ addw(rscratch4, rscratch4, rscratch1);
3410     __ addw(rscratch3, rscratch3, rscratch4);
3411     __ rorw(rscratch2, rscratch3, 32 - s);
3412     __ addw(r1, rscratch2, r2);
3413   }
3414 
3415   void md5_GG(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
3416               int k, int s, int t) {
3417     Register rscratch3 = r10;
3418     Register rscratch4 = r11;
3419 
3420     reg_cache.extract_u32(rscratch1, k);
3421     __ movw(rscratch2, t);
3422     __ addw(rscratch4, r1, rscratch2);
3423     __ addw(rscratch4, rscratch4, rscratch1);
3424     __ bicw(rscratch2, r3, r4);
3425     __ andw(rscratch3, r2, r4);
3426     __ addw(rscratch2, rscratch2, rscratch4);
3427     __ addw(rscratch2, rscratch2, rscratch3);
3428     __ rorw(rscratch2, rscratch2, 32 - s);
3429     __ addw(r1, rscratch2, r2);
3430   }
3431 
3432   void md5_HH(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
3433               int k, int s, int t) {
3434     Register rscratch3 = r10;
3435     Register rscratch4 = r11;
3436 
3437     __ eorw(rscratch3, r3, r4);
3438     __ movw(rscratch2, t);
3439     __ addw(rscratch4, r1, rscratch2);
3440     reg_cache.extract_u32(rscratch1, k);
3441     __ eorw(rscratch3, rscratch3, r2);
3442     __ addw(rscratch4, rscratch4, rscratch1);
3443     __ addw(rscratch3, rscratch3, rscratch4);
3444     __ rorw(rscratch2, rscratch3, 32 - s);
3445     __ addw(r1, rscratch2, r2);
3446   }
3447 
3448   void md5_II(Cached64Bytes& reg_cache, Register r1, Register r2, Register r3, Register r4,
3449               int k, int s, int t) {
3450     Register rscratch3 = r10;
3451     Register rscratch4 = r11;
3452 
3453     __ movw(rscratch3, t);
3454     __ ornw(rscratch2, r2, r4);
3455     __ addw(rscratch4, r1, rscratch3);
3456     reg_cache.extract_u32(rscratch1, k);
3457     __ eorw(rscratch3, rscratch2, r3);
3458     __ addw(rscratch4, rscratch4, rscratch1);
3459     __ addw(rscratch3, rscratch3, rscratch4);
3460     __ rorw(rscratch2, rscratch3, 32 - s);
3461     __ addw(r1, rscratch2, r2);
3462   }
3463 
3464   // Arguments:
3465   //
3466   // Inputs:
3467   //   c_rarg0   - byte[]  source+offset
3468   //   c_rarg1   - int[]   SHA.state
3469   //   c_rarg2   - int     offset
3470   //   c_rarg3   - int     limit
3471   //
3472   address generate_md5_implCompress(bool multi_block, const char *name) {
3473     __ align(CodeEntryAlignment);
3474     StubCodeMark mark(this, "StubRoutines", name);
3475     address start = __ pc();
3476 
3477     Register buf       = c_rarg0;
3478     Register state     = c_rarg1;
3479     Register ofs       = c_rarg2;
3480     Register limit     = c_rarg3;
3481     Register a         = r4;
3482     Register b         = r5;
3483     Register c         = r6;
3484     Register d         = r7;
3485     Register rscratch3 = r10;
3486     Register rscratch4 = r11;
3487 
3488     Register state_regs[2] = { r12, r13 };
3489     RegSet saved_regs = RegSet::range(r16, r22) - r18_tls;
3490     Cached64Bytes reg_cache(_masm, RegSet::of(r14, r15) + saved_regs);  // using 8 registers
3491 
3492     __ push(saved_regs, sp);
3493 
3494     __ ldp(state_regs[0], state_regs[1], Address(state));
3495     __ ubfx(a, state_regs[0],  0, 32);
3496     __ ubfx(b, state_regs[0], 32, 32);
3497     __ ubfx(c, state_regs[1],  0, 32);
3498     __ ubfx(d, state_regs[1], 32, 32);
3499 
3500     Label md5_loop;
3501     __ BIND(md5_loop);
3502 
3503     reg_cache.gen_loads(buf);
3504 
3505     // Round 1
3506     md5_FF(reg_cache, a, b, c, d,  0,  7, 0xd76aa478);
3507     md5_FF(reg_cache, d, a, b, c,  1, 12, 0xe8c7b756);
3508     md5_FF(reg_cache, c, d, a, b,  2, 17, 0x242070db);
3509     md5_FF(reg_cache, b, c, d, a,  3, 22, 0xc1bdceee);
3510     md5_FF(reg_cache, a, b, c, d,  4,  7, 0xf57c0faf);
3511     md5_FF(reg_cache, d, a, b, c,  5, 12, 0x4787c62a);
3512     md5_FF(reg_cache, c, d, a, b,  6, 17, 0xa8304613);
3513     md5_FF(reg_cache, b, c, d, a,  7, 22, 0xfd469501);
3514     md5_FF(reg_cache, a, b, c, d,  8,  7, 0x698098d8);
3515     md5_FF(reg_cache, d, a, b, c,  9, 12, 0x8b44f7af);
3516     md5_FF(reg_cache, c, d, a, b, 10, 17, 0xffff5bb1);
3517     md5_FF(reg_cache, b, c, d, a, 11, 22, 0x895cd7be);
3518     md5_FF(reg_cache, a, b, c, d, 12,  7, 0x6b901122);
3519     md5_FF(reg_cache, d, a, b, c, 13, 12, 0xfd987193);
3520     md5_FF(reg_cache, c, d, a, b, 14, 17, 0xa679438e);
3521     md5_FF(reg_cache, b, c, d, a, 15, 22, 0x49b40821);
3522 
3523     // Round 2
3524     md5_GG(reg_cache, a, b, c, d,  1,  5, 0xf61e2562);
3525     md5_GG(reg_cache, d, a, b, c,  6,  9, 0xc040b340);
3526     md5_GG(reg_cache, c, d, a, b, 11, 14, 0x265e5a51);
3527     md5_GG(reg_cache, b, c, d, a,  0, 20, 0xe9b6c7aa);
3528     md5_GG(reg_cache, a, b, c, d,  5,  5, 0xd62f105d);
3529     md5_GG(reg_cache, d, a, b, c, 10,  9, 0x02441453);
3530     md5_GG(reg_cache, c, d, a, b, 15, 14, 0xd8a1e681);
3531     md5_GG(reg_cache, b, c, d, a,  4, 20, 0xe7d3fbc8);
3532     md5_GG(reg_cache, a, b, c, d,  9,  5, 0x21e1cde6);
3533     md5_GG(reg_cache, d, a, b, c, 14,  9, 0xc33707d6);
3534     md5_GG(reg_cache, c, d, a, b,  3, 14, 0xf4d50d87);
3535     md5_GG(reg_cache, b, c, d, a,  8, 20, 0x455a14ed);
3536     md5_GG(reg_cache, a, b, c, d, 13,  5, 0xa9e3e905);
3537     md5_GG(reg_cache, d, a, b, c,  2,  9, 0xfcefa3f8);
3538     md5_GG(reg_cache, c, d, a, b,  7, 14, 0x676f02d9);
3539     md5_GG(reg_cache, b, c, d, a, 12, 20, 0x8d2a4c8a);
3540 
3541     // Round 3
3542     md5_HH(reg_cache, a, b, c, d,  5,  4, 0xfffa3942);
3543     md5_HH(reg_cache, d, a, b, c,  8, 11, 0x8771f681);
3544     md5_HH(reg_cache, c, d, a, b, 11, 16, 0x6d9d6122);
3545     md5_HH(reg_cache, b, c, d, a, 14, 23, 0xfde5380c);
3546     md5_HH(reg_cache, a, b, c, d,  1,  4, 0xa4beea44);
3547     md5_HH(reg_cache, d, a, b, c,  4, 11, 0x4bdecfa9);
3548     md5_HH(reg_cache, c, d, a, b,  7, 16, 0xf6bb4b60);
3549     md5_HH(reg_cache, b, c, d, a, 10, 23, 0xbebfbc70);
3550     md5_HH(reg_cache, a, b, c, d, 13,  4, 0x289b7ec6);
3551     md5_HH(reg_cache, d, a, b, c,  0, 11, 0xeaa127fa);
3552     md5_HH(reg_cache, c, d, a, b,  3, 16, 0xd4ef3085);
3553     md5_HH(reg_cache, b, c, d, a,  6, 23, 0x04881d05);
3554     md5_HH(reg_cache, a, b, c, d,  9,  4, 0xd9d4d039);
3555     md5_HH(reg_cache, d, a, b, c, 12, 11, 0xe6db99e5);
3556     md5_HH(reg_cache, c, d, a, b, 15, 16, 0x1fa27cf8);
3557     md5_HH(reg_cache, b, c, d, a,  2, 23, 0xc4ac5665);
3558 
3559     // Round 4
3560     md5_II(reg_cache, a, b, c, d,  0,  6, 0xf4292244);
3561     md5_II(reg_cache, d, a, b, c,  7, 10, 0x432aff97);
3562     md5_II(reg_cache, c, d, a, b, 14, 15, 0xab9423a7);
3563     md5_II(reg_cache, b, c, d, a,  5, 21, 0xfc93a039);
3564     md5_II(reg_cache, a, b, c, d, 12,  6, 0x655b59c3);
3565     md5_II(reg_cache, d, a, b, c,  3, 10, 0x8f0ccc92);
3566     md5_II(reg_cache, c, d, a, b, 10, 15, 0xffeff47d);
3567     md5_II(reg_cache, b, c, d, a,  1, 21, 0x85845dd1);
3568     md5_II(reg_cache, a, b, c, d,  8,  6, 0x6fa87e4f);
3569     md5_II(reg_cache, d, a, b, c, 15, 10, 0xfe2ce6e0);
3570     md5_II(reg_cache, c, d, a, b,  6, 15, 0xa3014314);
3571     md5_II(reg_cache, b, c, d, a, 13, 21, 0x4e0811a1);
3572     md5_II(reg_cache, a, b, c, d,  4,  6, 0xf7537e82);
3573     md5_II(reg_cache, d, a, b, c, 11, 10, 0xbd3af235);
3574     md5_II(reg_cache, c, d, a, b,  2, 15, 0x2ad7d2bb);
3575     md5_II(reg_cache, b, c, d, a,  9, 21, 0xeb86d391);
3576 
3577     __ addw(a, state_regs[0], a);
3578     __ ubfx(rscratch2, state_regs[0], 32, 32);
3579     __ addw(b, rscratch2, b);
3580     __ addw(c, state_regs[1], c);
3581     __ ubfx(rscratch4, state_regs[1], 32, 32);
3582     __ addw(d, rscratch4, d);
3583 
3584     __ orr(state_regs[0], a, b, Assembler::LSL, 32);
3585     __ orr(state_regs[1], c, d, Assembler::LSL, 32);
3586 
3587     if (multi_block) {
3588       __ add(buf, buf, 64);
3589       __ add(ofs, ofs, 64);
3590       __ cmp(ofs, limit);
3591       __ br(Assembler::LE, md5_loop);
3592       __ mov(c_rarg0, ofs); // return ofs
3593     }
3594 
3595     // write hash values back in the correct order
3596     __ stp(state_regs[0], state_regs[1], Address(state));
3597 
3598     __ pop(saved_regs, sp);
3599 
3600     __ ret(lr);
3601 
3602     return start;
3603   }
3604 
3605   // Arguments:
3606   //
3607   // Inputs:
3608   //   c_rarg0   - byte[]  source+offset
3609   //   c_rarg1   - int[]   SHA.state
3610   //   c_rarg2   - int     offset
3611   //   c_rarg3   - int     limit
3612   //
3613   address generate_sha1_implCompress(bool multi_block, const char *name) {
3614     __ align(CodeEntryAlignment);
3615     StubCodeMark mark(this, "StubRoutines", name);
3616     address start = __ pc();
3617 
3618     Register buf   = c_rarg0;
3619     Register state = c_rarg1;
3620     Register ofs   = c_rarg2;
3621     Register limit = c_rarg3;
3622 
3623     Label keys;
3624     Label sha1_loop;
3625 
3626     // load the keys into v0..v3
3627     __ adr(rscratch1, keys);
3628     __ ld4r(v0, v1, v2, v3, __ T4S, Address(rscratch1));
3629     // load 5 words state into v6, v7
3630     __ ldrq(v6, Address(state, 0));
3631     __ ldrs(v7, Address(state, 16));
3632 
3633 
3634     __ BIND(sha1_loop);
3635     // load 64 bytes of data into v16..v19
3636     __ ld1(v16, v17, v18, v19, __ T4S, multi_block ? __ post(buf, 64) : buf);
3637     __ rev32(v16, __ T16B, v16);
3638     __ rev32(v17, __ T16B, v17);
3639     __ rev32(v18, __ T16B, v18);
3640     __ rev32(v19, __ T16B, v19);
3641 
3642     // do the sha1
3643     __ addv(v4, __ T4S, v16, v0);
3644     __ orr(v20, __ T16B, v6, v6);
3645 
3646     FloatRegister d0 = v16;
3647     FloatRegister d1 = v17;
3648     FloatRegister d2 = v18;
3649     FloatRegister d3 = v19;
3650 
3651     for (int round = 0; round < 20; round++) {
3652       FloatRegister tmp1 = (round & 1) ? v4 : v5;
3653       FloatRegister tmp2 = (round & 1) ? v21 : v22;
3654       FloatRegister tmp3 = round ? ((round & 1) ? v22 : v21) : v7;
3655       FloatRegister tmp4 = (round & 1) ? v5 : v4;
3656       FloatRegister key = (round < 4) ? v0 : ((round < 9) ? v1 : ((round < 14) ? v2 : v3));
3657 
3658       if (round < 16) __ sha1su0(d0, __ T4S, d1, d2);
3659       if (round < 19) __ addv(tmp1, __ T4S, d1, key);
3660       __ sha1h(tmp2, __ T4S, v20);
3661       if (round < 5)
3662         __ sha1c(v20, __ T4S, tmp3, tmp4);
3663       else if (round < 10 || round >= 15)
3664         __ sha1p(v20, __ T4S, tmp3, tmp4);
3665       else
3666         __ sha1m(v20, __ T4S, tmp3, tmp4);
3667       if (round < 16) __ sha1su1(d0, __ T4S, d3);
3668 
3669       tmp1 = d0; d0 = d1; d1 = d2; d2 = d3; d3 = tmp1;
3670     }
3671 
3672     __ addv(v7, __ T2S, v7, v21);
3673     __ addv(v6, __ T4S, v6, v20);
3674 
3675     if (multi_block) {
3676       __ add(ofs, ofs, 64);
3677       __ cmp(ofs, limit);
3678       __ br(Assembler::LE, sha1_loop);
3679       __ mov(c_rarg0, ofs); // return ofs
3680     }
3681 
3682     __ strq(v6, Address(state, 0));
3683     __ strs(v7, Address(state, 16));
3684 
3685     __ ret(lr);
3686 
3687     __ bind(keys);
3688     __ emit_int32(0x5a827999);
3689     __ emit_int32(0x6ed9eba1);
3690     __ emit_int32(0x8f1bbcdc);
3691     __ emit_int32(0xca62c1d6);
3692 
3693     return start;
3694   }
3695 
3696 
3697   // Arguments:
3698   //
3699   // Inputs:
3700   //   c_rarg0   - byte[]  source+offset
3701   //   c_rarg1   - int[]   SHA.state
3702   //   c_rarg2   - int     offset
3703   //   c_rarg3   - int     limit
3704   //
3705   address generate_sha256_implCompress(bool multi_block, const char *name) {
3706     static const uint32_t round_consts[64] = {
3707       0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
3708       0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
3709       0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
3710       0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
3711       0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
3712       0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
3713       0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
3714       0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
3715       0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
3716       0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
3717       0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
3718       0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
3719       0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
3720       0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
3721       0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
3722       0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
3723     };
3724     __ align(CodeEntryAlignment);
3725     StubCodeMark mark(this, "StubRoutines", name);
3726     address start = __ pc();
3727 
3728     Register buf   = c_rarg0;
3729     Register state = c_rarg1;
3730     Register ofs   = c_rarg2;
3731     Register limit = c_rarg3;
3732 
3733     Label sha1_loop;
3734 
3735     __ stpd(v8, v9, __ pre(sp, -32));
3736     __ stpd(v10, v11, Address(sp, 16));
3737 
3738 // dga == v0
3739 // dgb == v1
3740 // dg0 == v2
3741 // dg1 == v3
3742 // dg2 == v4
3743 // t0 == v6
3744 // t1 == v7
3745 
3746     // load 16 keys to v16..v31
3747     __ lea(rscratch1, ExternalAddress((address)round_consts));
3748     __ ld1(v16, v17, v18, v19, __ T4S, __ post(rscratch1, 64));
3749     __ ld1(v20, v21, v22, v23, __ T4S, __ post(rscratch1, 64));
3750     __ ld1(v24, v25, v26, v27, __ T4S, __ post(rscratch1, 64));
3751     __ ld1(v28, v29, v30, v31, __ T4S, rscratch1);
3752 
3753     // load 8 words (256 bits) state
3754     __ ldpq(v0, v1, state);
3755 
3756     __ BIND(sha1_loop);
3757     // load 64 bytes of data into v8..v11
3758     __ ld1(v8, v9, v10, v11, __ T4S, multi_block ? __ post(buf, 64) : buf);
3759     __ rev32(v8, __ T16B, v8);
3760     __ rev32(v9, __ T16B, v9);
3761     __ rev32(v10, __ T16B, v10);
3762     __ rev32(v11, __ T16B, v11);
3763 
3764     __ addv(v6, __ T4S, v8, v16);
3765     __ orr(v2, __ T16B, v0, v0);
3766     __ orr(v3, __ T16B, v1, v1);
3767 
3768     FloatRegister d0 = v8;
3769     FloatRegister d1 = v9;
3770     FloatRegister d2 = v10;
3771     FloatRegister d3 = v11;
3772 
3773 
3774     for (int round = 0; round < 16; round++) {
3775       FloatRegister tmp1 = (round & 1) ? v6 : v7;
3776       FloatRegister tmp2 = (round & 1) ? v7 : v6;
3777       FloatRegister tmp3 = (round & 1) ? v2 : v4;
3778       FloatRegister tmp4 = (round & 1) ? v4 : v2;
3779 
3780       if (round < 12) __ sha256su0(d0, __ T4S, d1);
3781        __ orr(v4, __ T16B, v2, v2);
3782       if (round < 15)
3783         __ addv(tmp1, __ T4S, d1, as_FloatRegister(round + 17));
3784       __ sha256h(v2, __ T4S, v3, tmp2);
3785       __ sha256h2(v3, __ T4S, v4, tmp2);
3786       if (round < 12) __ sha256su1(d0, __ T4S, d2, d3);
3787 
3788       tmp1 = d0; d0 = d1; d1 = d2; d2 = d3; d3 = tmp1;
3789     }
3790 
3791     __ addv(v0, __ T4S, v0, v2);
3792     __ addv(v1, __ T4S, v1, v3);
3793 
3794     if (multi_block) {
3795       __ add(ofs, ofs, 64);
3796       __ cmp(ofs, limit);
3797       __ br(Assembler::LE, sha1_loop);
3798       __ mov(c_rarg0, ofs); // return ofs
3799     }
3800 
3801     __ ldpd(v10, v11, Address(sp, 16));
3802     __ ldpd(v8, v9, __ post(sp, 32));
3803 
3804     __ stpq(v0, v1, state);
3805 
3806     __ ret(lr);
3807 
3808     return start;
3809   }
3810 
3811   // Double rounds for sha512.
3812   void sha512_dround(int dr,
3813                      FloatRegister vi0, FloatRegister vi1,
3814                      FloatRegister vi2, FloatRegister vi3,
3815                      FloatRegister vi4, FloatRegister vrc0,
3816                      FloatRegister vrc1, FloatRegister vin0,
3817                      FloatRegister vin1, FloatRegister vin2,
3818                      FloatRegister vin3, FloatRegister vin4) {
3819       if (dr < 36) {
3820         __ ld1(vrc1, __ T2D, __ post(rscratch2, 16));
3821       }
3822       __ addv(v5, __ T2D, vrc0, vin0);
3823       __ ext(v6, __ T16B, vi2, vi3, 8);
3824       __ ext(v5, __ T16B, v5, v5, 8);
3825       __ ext(v7, __ T16B, vi1, vi2, 8);
3826       __ addv(vi3, __ T2D, vi3, v5);
3827       if (dr < 32) {
3828         __ ext(v5, __ T16B, vin3, vin4, 8);
3829         __ sha512su0(vin0, __ T2D, vin1);
3830       }
3831       __ sha512h(vi3, __ T2D, v6, v7);
3832       if (dr < 32) {
3833         __ sha512su1(vin0, __ T2D, vin2, v5);
3834       }
3835       __ addv(vi4, __ T2D, vi1, vi3);
3836       __ sha512h2(vi3, __ T2D, vi1, vi0);
3837   }
3838 
3839   // Arguments:
3840   //
3841   // Inputs:
3842   //   c_rarg0   - byte[]  source+offset
3843   //   c_rarg1   - int[]   SHA.state
3844   //   c_rarg2   - int     offset
3845   //   c_rarg3   - int     limit
3846   //
3847   address generate_sha512_implCompress(bool multi_block, const char *name) {
3848     static const uint64_t round_consts[80] = {
3849       0x428A2F98D728AE22L, 0x7137449123EF65CDL, 0xB5C0FBCFEC4D3B2FL,
3850       0xE9B5DBA58189DBBCL, 0x3956C25BF348B538L, 0x59F111F1B605D019L,
3851       0x923F82A4AF194F9BL, 0xAB1C5ED5DA6D8118L, 0xD807AA98A3030242L,
3852       0x12835B0145706FBEL, 0x243185BE4EE4B28CL, 0x550C7DC3D5FFB4E2L,
3853       0x72BE5D74F27B896FL, 0x80DEB1FE3B1696B1L, 0x9BDC06A725C71235L,
3854       0xC19BF174CF692694L, 0xE49B69C19EF14AD2L, 0xEFBE4786384F25E3L,
3855       0x0FC19DC68B8CD5B5L, 0x240CA1CC77AC9C65L, 0x2DE92C6F592B0275L,
3856       0x4A7484AA6EA6E483L, 0x5CB0A9DCBD41FBD4L, 0x76F988DA831153B5L,
3857       0x983E5152EE66DFABL, 0xA831C66D2DB43210L, 0xB00327C898FB213FL,
3858       0xBF597FC7BEEF0EE4L, 0xC6E00BF33DA88FC2L, 0xD5A79147930AA725L,
3859       0x06CA6351E003826FL, 0x142929670A0E6E70L, 0x27B70A8546D22FFCL,
3860       0x2E1B21385C26C926L, 0x4D2C6DFC5AC42AEDL, 0x53380D139D95B3DFL,
3861       0x650A73548BAF63DEL, 0x766A0ABB3C77B2A8L, 0x81C2C92E47EDAEE6L,
3862       0x92722C851482353BL, 0xA2BFE8A14CF10364L, 0xA81A664BBC423001L,
3863       0xC24B8B70D0F89791L, 0xC76C51A30654BE30L, 0xD192E819D6EF5218L,
3864       0xD69906245565A910L, 0xF40E35855771202AL, 0x106AA07032BBD1B8L,
3865       0x19A4C116B8D2D0C8L, 0x1E376C085141AB53L, 0x2748774CDF8EEB99L,
3866       0x34B0BCB5E19B48A8L, 0x391C0CB3C5C95A63L, 0x4ED8AA4AE3418ACBL,
3867       0x5B9CCA4F7763E373L, 0x682E6FF3D6B2B8A3L, 0x748F82EE5DEFB2FCL,
3868       0x78A5636F43172F60L, 0x84C87814A1F0AB72L, 0x8CC702081A6439ECL,
3869       0x90BEFFFA23631E28L, 0xA4506CEBDE82BDE9L, 0xBEF9A3F7B2C67915L,
3870       0xC67178F2E372532BL, 0xCA273ECEEA26619CL, 0xD186B8C721C0C207L,
3871       0xEADA7DD6CDE0EB1EL, 0xF57D4F7FEE6ED178L, 0x06F067AA72176FBAL,
3872       0x0A637DC5A2C898A6L, 0x113F9804BEF90DAEL, 0x1B710B35131C471BL,
3873       0x28DB77F523047D84L, 0x32CAAB7B40C72493L, 0x3C9EBE0A15C9BEBCL,
3874       0x431D67C49C100D4CL, 0x4CC5D4BECB3E42B6L, 0x597F299CFC657E2AL,
3875       0x5FCB6FAB3AD6FAECL, 0x6C44198C4A475817L
3876     };
3877 
3878     __ align(CodeEntryAlignment);
3879     StubCodeMark mark(this, "StubRoutines", name);
3880     address start = __ pc();
3881 
3882     Register buf   = c_rarg0;
3883     Register state = c_rarg1;
3884     Register ofs   = c_rarg2;
3885     Register limit = c_rarg3;
3886 
3887     __ stpd(v8, v9, __ pre(sp, -64));
3888     __ stpd(v10, v11, Address(sp, 16));
3889     __ stpd(v12, v13, Address(sp, 32));
3890     __ stpd(v14, v15, Address(sp, 48));
3891 
3892     Label sha512_loop;
3893 
3894     // load state
3895     __ ld1(v8, v9, v10, v11, __ T2D, state);
3896 
3897     // load first 4 round constants
3898     __ lea(rscratch1, ExternalAddress((address)round_consts));
3899     __ ld1(v20, v21, v22, v23, __ T2D, __ post(rscratch1, 64));
3900 
3901     __ BIND(sha512_loop);
3902     // load 128B of data into v12..v19
3903     __ ld1(v12, v13, v14, v15, __ T2D, __ post(buf, 64));
3904     __ ld1(v16, v17, v18, v19, __ T2D, __ post(buf, 64));
3905     __ rev64(v12, __ T16B, v12);
3906     __ rev64(v13, __ T16B, v13);
3907     __ rev64(v14, __ T16B, v14);
3908     __ rev64(v15, __ T16B, v15);
3909     __ rev64(v16, __ T16B, v16);
3910     __ rev64(v17, __ T16B, v17);
3911     __ rev64(v18, __ T16B, v18);
3912     __ rev64(v19, __ T16B, v19);
3913 
3914     __ mov(rscratch2, rscratch1);
3915 
3916     __ mov(v0, __ T16B, v8);
3917     __ mov(v1, __ T16B, v9);
3918     __ mov(v2, __ T16B, v10);
3919     __ mov(v3, __ T16B, v11);
3920 
3921     sha512_dround( 0, v0, v1, v2, v3, v4, v20, v24, v12, v13, v19, v16, v17);
3922     sha512_dround( 1, v3, v0, v4, v2, v1, v21, v25, v13, v14, v12, v17, v18);
3923     sha512_dround( 2, v2, v3, v1, v4, v0, v22, v26, v14, v15, v13, v18, v19);
3924     sha512_dround( 3, v4, v2, v0, v1, v3, v23, v27, v15, v16, v14, v19, v12);
3925     sha512_dround( 4, v1, v4, v3, v0, v2, v24, v28, v16, v17, v15, v12, v13);
3926     sha512_dround( 5, v0, v1, v2, v3, v4, v25, v29, v17, v18, v16, v13, v14);
3927     sha512_dround( 6, v3, v0, v4, v2, v1, v26, v30, v18, v19, v17, v14, v15);
3928     sha512_dround( 7, v2, v3, v1, v4, v0, v27, v31, v19, v12, v18, v15, v16);
3929     sha512_dround( 8, v4, v2, v0, v1, v3, v28, v24, v12, v13, v19, v16, v17);
3930     sha512_dround( 9, v1, v4, v3, v0, v2, v29, v25, v13, v14, v12, v17, v18);
3931     sha512_dround(10, v0, v1, v2, v3, v4, v30, v26, v14, v15, v13, v18, v19);
3932     sha512_dround(11, v3, v0, v4, v2, v1, v31, v27, v15, v16, v14, v19, v12);
3933     sha512_dround(12, v2, v3, v1, v4, v0, v24, v28, v16, v17, v15, v12, v13);
3934     sha512_dround(13, v4, v2, v0, v1, v3, v25, v29, v17, v18, v16, v13, v14);
3935     sha512_dround(14, v1, v4, v3, v0, v2, v26, v30, v18, v19, v17, v14, v15);
3936     sha512_dround(15, v0, v1, v2, v3, v4, v27, v31, v19, v12, v18, v15, v16);
3937     sha512_dround(16, v3, v0, v4, v2, v1, v28, v24, v12, v13, v19, v16, v17);
3938     sha512_dround(17, v2, v3, v1, v4, v0, v29, v25, v13, v14, v12, v17, v18);
3939     sha512_dround(18, v4, v2, v0, v1, v3, v30, v26, v14, v15, v13, v18, v19);
3940     sha512_dround(19, v1, v4, v3, v0, v2, v31, v27, v15, v16, v14, v19, v12);
3941     sha512_dround(20, v0, v1, v2, v3, v4, v24, v28, v16, v17, v15, v12, v13);
3942     sha512_dround(21, v3, v0, v4, v2, v1, v25, v29, v17, v18, v16, v13, v14);
3943     sha512_dround(22, v2, v3, v1, v4, v0, v26, v30, v18, v19, v17, v14, v15);
3944     sha512_dround(23, v4, v2, v0, v1, v3, v27, v31, v19, v12, v18, v15, v16);
3945     sha512_dround(24, v1, v4, v3, v0, v2, v28, v24, v12, v13, v19, v16, v17);
3946     sha512_dround(25, v0, v1, v2, v3, v4, v29, v25, v13, v14, v12, v17, v18);
3947     sha512_dround(26, v3, v0, v4, v2, v1, v30, v26, v14, v15, v13, v18, v19);
3948     sha512_dround(27, v2, v3, v1, v4, v0, v31, v27, v15, v16, v14, v19, v12);
3949     sha512_dround(28, v4, v2, v0, v1, v3, v24, v28, v16, v17, v15, v12, v13);
3950     sha512_dround(29, v1, v4, v3, v0, v2, v25, v29, v17, v18, v16, v13, v14);
3951     sha512_dround(30, v0, v1, v2, v3, v4, v26, v30, v18, v19, v17, v14, v15);
3952     sha512_dround(31, v3, v0, v4, v2, v1, v27, v31, v19, v12, v18, v15, v16);
3953     sha512_dround(32, v2, v3, v1, v4, v0, v28, v24, v12,  v0,  v0,  v0,  v0);
3954     sha512_dround(33, v4, v2, v0, v1, v3, v29, v25, v13,  v0,  v0,  v0,  v0);
3955     sha512_dround(34, v1, v4, v3, v0, v2, v30, v26, v14,  v0,  v0,  v0,  v0);
3956     sha512_dround(35, v0, v1, v2, v3, v4, v31, v27, v15,  v0,  v0,  v0,  v0);
3957     sha512_dround(36, v3, v0, v4, v2, v1, v24,  v0, v16,  v0,  v0,  v0,  v0);
3958     sha512_dround(37, v2, v3, v1, v4, v0, v25,  v0, v17,  v0,  v0,  v0,  v0);
3959     sha512_dround(38, v4, v2, v0, v1, v3, v26,  v0, v18,  v0,  v0,  v0,  v0);
3960     sha512_dround(39, v1, v4, v3, v0, v2, v27,  v0, v19,  v0,  v0,  v0,  v0);
3961 
3962     __ addv(v8, __ T2D, v8, v0);
3963     __ addv(v9, __ T2D, v9, v1);
3964     __ addv(v10, __ T2D, v10, v2);
3965     __ addv(v11, __ T2D, v11, v3);
3966 
3967     if (multi_block) {
3968       __ add(ofs, ofs, 128);
3969       __ cmp(ofs, limit);
3970       __ br(Assembler::LE, sha512_loop);
3971       __ mov(c_rarg0, ofs); // return ofs
3972     }
3973 
3974     __ st1(v8, v9, v10, v11, __ T2D, state);
3975 
3976     __ ldpd(v14, v15, Address(sp, 48));
3977     __ ldpd(v12, v13, Address(sp, 32));
3978     __ ldpd(v10, v11, Address(sp, 16));
3979     __ ldpd(v8, v9, __ post(sp, 64));
3980 
3981     __ ret(lr);
3982 
3983     return start;
3984   }
3985 
3986   // Arguments:
3987   //
3988   // Inputs:
3989   //   c_rarg0   - byte[]  source+offset
3990   //   c_rarg1   - byte[]  SHA.state
3991   //   c_rarg2   - int     block_size
3992   //   c_rarg3   - int     offset
3993   //   c_rarg4   - int     limit
3994   //
3995   address generate_sha3_implCompress(bool multi_block, const char *name) {
3996     static const uint64_t round_consts[24] = {
3997       0x0000000000000001L, 0x0000000000008082L, 0x800000000000808AL,
3998       0x8000000080008000L, 0x000000000000808BL, 0x0000000080000001L,
3999       0x8000000080008081L, 0x8000000000008009L, 0x000000000000008AL,
4000       0x0000000000000088L, 0x0000000080008009L, 0x000000008000000AL,
4001       0x000000008000808BL, 0x800000000000008BL, 0x8000000000008089L,
4002       0x8000000000008003L, 0x8000000000008002L, 0x8000000000000080L,
4003       0x000000000000800AL, 0x800000008000000AL, 0x8000000080008081L,
4004       0x8000000000008080L, 0x0000000080000001L, 0x8000000080008008L
4005     };
4006 
4007     __ align(CodeEntryAlignment);
4008     StubCodeMark mark(this, "StubRoutines", name);
4009     address start = __ pc();
4010 
4011     Register buf           = c_rarg0;
4012     Register state         = c_rarg1;
4013     Register block_size    = c_rarg2;
4014     Register ofs           = c_rarg3;
4015     Register limit         = c_rarg4;
4016 
4017     Label sha3_loop, rounds24_loop;
4018     Label sha3_512_or_sha3_384, shake128;
4019 
4020     __ stpd(v8, v9, __ pre(sp, -64));
4021     __ stpd(v10, v11, Address(sp, 16));
4022     __ stpd(v12, v13, Address(sp, 32));
4023     __ stpd(v14, v15, Address(sp, 48));
4024 
4025     // load state
4026     __ add(rscratch1, state, 32);
4027     __ ld1(v0, v1, v2,  v3,  __ T1D, state);
4028     __ ld1(v4, v5, v6,  v7,  __ T1D, __ post(rscratch1, 32));
4029     __ ld1(v8, v9, v10, v11, __ T1D, __ post(rscratch1, 32));
4030     __ ld1(v12, v13, v14, v15, __ T1D, __ post(rscratch1, 32));
4031     __ ld1(v16, v17, v18, v19, __ T1D, __ post(rscratch1, 32));
4032     __ ld1(v20, v21, v22, v23, __ T1D, __ post(rscratch1, 32));
4033     __ ld1(v24, __ T1D, rscratch1);
4034 
4035     __ BIND(sha3_loop);
4036 
4037     // 24 keccak rounds
4038     __ movw(rscratch2, 24);
4039 
4040     // load round_constants base
4041     __ lea(rscratch1, ExternalAddress((address) round_consts));
4042 
4043     // load input
4044     __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32));
4045     __ ld1(v29, v30, v31, __ T8B, __ post(buf, 24));
4046     __ eor(v0, __ T8B, v0, v25);
4047     __ eor(v1, __ T8B, v1, v26);
4048     __ eor(v2, __ T8B, v2, v27);
4049     __ eor(v3, __ T8B, v3, v28);
4050     __ eor(v4, __ T8B, v4, v29);
4051     __ eor(v5, __ T8B, v5, v30);
4052     __ eor(v6, __ T8B, v6, v31);
4053 
4054     // block_size == 72, SHA3-512; block_size == 104, SHA3-384
4055     __ tbz(block_size, 7, sha3_512_or_sha3_384);
4056 
4057     __ ld1(v25, v26, v27, v28, __ T8B, __ post(buf, 32));
4058     __ ld1(v29, v30, v31, __ T8B, __ post(buf, 24));
4059     __ eor(v7, __ T8B, v7, v25);
4060     __ eor(v8, __ T8B, v8, v26);
4061     __ eor(v9, __ T8B, v9, v27);
4062     __ eor(v10, __ T8B, v10, v28);
4063     __ eor(v11, __ T8B, v11, v29);
4064     __ eor(v12, __ T8B, v12, v30);
4065     __ eor(v13, __ T8B, v13, v31);
4066 
4067     __ ld1(v25, v26, v27,  __ T8B, __ post(buf, 24));
4068     __ eor(v14, __ T8B, v14, v25);
4069     __ eor(v15, __ T8B, v15, v26);
4070     __ eor(v16, __ T8B, v16, v27);
4071 
4072     // block_size == 136, bit4 == 0 and bit5 == 0, SHA3-256 or SHAKE256
4073     __ andw(c_rarg5, block_size, 48);
4074     __ cbzw(c_rarg5, rounds24_loop);
4075 
4076     __ tbnz(block_size, 5, shake128);
4077     // block_size == 144, bit5 == 0, SHA3-244
4078     __ ldrd(v28, __ post(buf, 8));
4079     __ eor(v17, __ T8B, v17, v28);
4080     __ b(rounds24_loop);
4081 
4082     __ BIND(shake128);
4083     __ ld1(v28, v29, v30, v31, __ T8B, __ post(buf, 32));
4084     __ eor(v17, __ T8B, v17, v28);
4085     __ eor(v18, __ T8B, v18, v29);
4086     __ eor(v19, __ T8B, v19, v30);
4087     __ eor(v20, __ T8B, v20, v31);
4088     __ b(rounds24_loop); // block_size == 168, SHAKE128
4089 
4090     __ BIND(sha3_512_or_sha3_384);
4091     __ ld1(v25, v26, __ T8B, __ post(buf, 16));
4092     __ eor(v7, __ T8B, v7, v25);
4093     __ eor(v8, __ T8B, v8, v26);
4094     __ tbz(block_size, 5, rounds24_loop); // SHA3-512
4095 
4096     // SHA3-384
4097     __ ld1(v27, v28, v29, v30, __ T8B, __ post(buf, 32));
4098     __ eor(v9,  __ T8B, v9,  v27);
4099     __ eor(v10, __ T8B, v10, v28);
4100     __ eor(v11, __ T8B, v11, v29);
4101     __ eor(v12, __ T8B, v12, v30);
4102 
4103     __ BIND(rounds24_loop);
4104     __ subw(rscratch2, rscratch2, 1);
4105 
4106     __ eor3(v29, __ T16B, v4, v9, v14);
4107     __ eor3(v26, __ T16B, v1, v6, v11);
4108     __ eor3(v28, __ T16B, v3, v8, v13);
4109     __ eor3(v25, __ T16B, v0, v5, v10);
4110     __ eor3(v27, __ T16B, v2, v7, v12);
4111     __ eor3(v29, __ T16B, v29, v19, v24);
4112     __ eor3(v26, __ T16B, v26, v16, v21);
4113     __ eor3(v28, __ T16B, v28, v18, v23);
4114     __ eor3(v25, __ T16B, v25, v15, v20);
4115     __ eor3(v27, __ T16B, v27, v17, v22);
4116 
4117     __ rax1(v30, __ T2D, v29, v26);
4118     __ rax1(v26, __ T2D, v26, v28);
4119     __ rax1(v28, __ T2D, v28, v25);
4120     __ rax1(v25, __ T2D, v25, v27);
4121     __ rax1(v27, __ T2D, v27, v29);
4122 
4123     __ eor(v0, __ T16B, v0, v30);
4124     __ xar(v29, __ T2D, v1,  v25, (64 - 1));
4125     __ xar(v1,  __ T2D, v6,  v25, (64 - 44));
4126     __ xar(v6,  __ T2D, v9,  v28, (64 - 20));
4127     __ xar(v9,  __ T2D, v22, v26, (64 - 61));
4128     __ xar(v22, __ T2D, v14, v28, (64 - 39));
4129     __ xar(v14, __ T2D, v20, v30, (64 - 18));
4130     __ xar(v31, __ T2D, v2,  v26, (64 - 62));
4131     __ xar(v2,  __ T2D, v12, v26, (64 - 43));
4132     __ xar(v12, __ T2D, v13, v27, (64 - 25));
4133     __ xar(v13, __ T2D, v19, v28, (64 - 8));
4134     __ xar(v19, __ T2D, v23, v27, (64 - 56));
4135     __ xar(v23, __ T2D, v15, v30, (64 - 41));
4136     __ xar(v15, __ T2D, v4,  v28, (64 - 27));
4137     __ xar(v28, __ T2D, v24, v28, (64 - 14));
4138     __ xar(v24, __ T2D, v21, v25, (64 - 2));
4139     __ xar(v8,  __ T2D, v8,  v27, (64 - 55));
4140     __ xar(v4,  __ T2D, v16, v25, (64 - 45));
4141     __ xar(v16, __ T2D, v5,  v30, (64 - 36));
4142     __ xar(v5,  __ T2D, v3,  v27, (64 - 28));
4143     __ xar(v27, __ T2D, v18, v27, (64 - 21));
4144     __ xar(v3,  __ T2D, v17, v26, (64 - 15));
4145     __ xar(v25, __ T2D, v11, v25, (64 - 10));
4146     __ xar(v26, __ T2D, v7,  v26, (64 - 6));
4147     __ xar(v30, __ T2D, v10, v30, (64 - 3));
4148 
4149     __ bcax(v20, __ T16B, v31, v22, v8);
4150     __ bcax(v21, __ T16B, v8,  v23, v22);
4151     __ bcax(v22, __ T16B, v22, v24, v23);
4152     __ bcax(v23, __ T16B, v23, v31, v24);
4153     __ bcax(v24, __ T16B, v24, v8,  v31);
4154 
4155     __ ld1r(v31, __ T2D, __ post(rscratch1, 8));
4156 
4157     __ bcax(v17, __ T16B, v25, v19, v3);
4158     __ bcax(v18, __ T16B, v3,  v15, v19);
4159     __ bcax(v19, __ T16B, v19, v16, v15);
4160     __ bcax(v15, __ T16B, v15, v25, v16);
4161     __ bcax(v16, __ T16B, v16, v3,  v25);
4162 
4163     __ bcax(v10, __ T16B, v29, v12, v26);
4164     __ bcax(v11, __ T16B, v26, v13, v12);
4165     __ bcax(v12, __ T16B, v12, v14, v13);
4166     __ bcax(v13, __ T16B, v13, v29, v14);
4167     __ bcax(v14, __ T16B, v14, v26, v29);
4168 
4169     __ bcax(v7, __ T16B, v30, v9,  v4);
4170     __ bcax(v8, __ T16B, v4,  v5,  v9);
4171     __ bcax(v9, __ T16B, v9,  v6,  v5);
4172     __ bcax(v5, __ T16B, v5,  v30, v6);
4173     __ bcax(v6, __ T16B, v6,  v4,  v30);
4174 
4175     __ bcax(v3, __ T16B, v27, v0,  v28);
4176     __ bcax(v4, __ T16B, v28, v1,  v0);
4177     __ bcax(v0, __ T16B, v0,  v2,  v1);
4178     __ bcax(v1, __ T16B, v1,  v27, v2);
4179     __ bcax(v2, __ T16B, v2,  v28, v27);
4180 
4181     __ eor(v0, __ T16B, v0, v31);
4182 
4183     __ cbnzw(rscratch2, rounds24_loop);
4184 
4185     if (multi_block) {
4186       __ add(ofs, ofs, block_size);
4187       __ cmp(ofs, limit);
4188       __ br(Assembler::LE, sha3_loop);
4189       __ mov(c_rarg0, ofs); // return ofs
4190     }
4191 
4192     __ st1(v0, v1, v2,  v3,  __ T1D, __ post(state, 32));
4193     __ st1(v4, v5, v6,  v7,  __ T1D, __ post(state, 32));
4194     __ st1(v8, v9, v10, v11, __ T1D, __ post(state, 32));
4195     __ st1(v12, v13, v14, v15, __ T1D, __ post(state, 32));
4196     __ st1(v16, v17, v18, v19, __ T1D, __ post(state, 32));
4197     __ st1(v20, v21, v22, v23, __ T1D, __ post(state, 32));
4198     __ st1(v24, __ T1D, state);
4199 
4200     __ ldpd(v14, v15, Address(sp, 48));
4201     __ ldpd(v12, v13, Address(sp, 32));
4202     __ ldpd(v10, v11, Address(sp, 16));
4203     __ ldpd(v8, v9, __ post(sp, 64));
4204 
4205     __ ret(lr);
4206 
4207     return start;
4208   }
4209 
4210   /**
4211    *  Arguments:
4212    *
4213    * Inputs:
4214    *   c_rarg0   - int crc
4215    *   c_rarg1   - byte* buf
4216    *   c_rarg2   - int length
4217    *
4218    * Output:
4219    *       rax   - int crc result
4220    */
4221   address generate_updateBytesCRC32() {
4222     assert(UseCRC32Intrinsics, "what are we doing here?");
4223 
4224     __ align(CodeEntryAlignment);
4225     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
4226 
4227     address start = __ pc();
4228 
4229     const Register crc   = c_rarg0;  // crc
4230     const Register buf   = c_rarg1;  // source java byte array address
4231     const Register len   = c_rarg2;  // length
4232     const Register table0 = c_rarg3; // crc_table address
4233     const Register table1 = c_rarg4;
4234     const Register table2 = c_rarg5;
4235     const Register table3 = c_rarg6;
4236     const Register tmp3 = c_rarg7;
4237 
4238     BLOCK_COMMENT("Entry:");
4239     __ enter(); // required for proper stackwalking of RuntimeStub frame
4240 
4241     __ kernel_crc32(crc, buf, len,
4242               table0, table1, table2, table3, rscratch1, rscratch2, tmp3);
4243 
4244     __ leave(); // required for proper stackwalking of RuntimeStub frame
4245     __ ret(lr);
4246 
4247     return start;
4248   }
4249 
4250   // ChaCha20 block function.  This version parallelizes by loading
4251   // individual 32-bit state elements into vectors for four blocks
4252   // (e.g. all four blocks' worth of state[0] in one register, etc.)
4253   //
4254   // state (int[16]) = c_rarg0
4255   // keystream (byte[1024]) = c_rarg1
4256   // return - number of bytes of keystream (always 256)
4257   address generate_chacha20Block_blockpar() {
4258     Label L_twoRounds, L_cc20_const;
4259     // The constant data is broken into two 128-bit segments to be loaded
4260     // onto FloatRegisters.  The first 128 bits are a counter add overlay
4261     // that adds +0/+1/+2/+3 to the vector holding replicated state[12].
4262     // The second 128-bits is a table constant used for 8-bit left rotations.
4263     __ BIND(L_cc20_const);
4264     __ emit_int64(0x0000000100000000UL);
4265     __ emit_int64(0x0000000300000002UL);
4266     __ emit_int64(0x0605040702010003UL);
4267     __ emit_int64(0x0E0D0C0F0A09080BUL);
4268 
4269     __ align(CodeEntryAlignment);
4270     StubCodeMark mark(this, "StubRoutines", "chacha20Block");
4271     address start = __ pc();
4272     __ enter();
4273 
4274     int i, j;
4275     const Register state = c_rarg0;
4276     const Register keystream = c_rarg1;
4277     const Register loopCtr = r10;
4278     const Register tmpAddr = r11;
4279 
4280     const FloatRegister stateFirst = v0;
4281     const FloatRegister stateSecond = v1;
4282     const FloatRegister stateThird = v2;
4283     const FloatRegister stateFourth = v3;
4284     const FloatRegister origCtrState = v28;
4285     const FloatRegister scratch = v29;
4286     const FloatRegister lrot8Tbl = v30;
4287 
4288     // Organize SIMD registers in an array that facilitates
4289     // putting repetitive opcodes into loop structures.  It is
4290     // important that each grouping of 4 registers is monotonically
4291     // increasing to support the requirements of multi-register
4292     // instructions (e.g. ld4r, st4, etc.)
4293     const FloatRegister workSt[16] = {
4294          v4,  v5,  v6,  v7, v16, v17, v18, v19,
4295         v20, v21, v22, v23, v24, v25, v26, v27
4296     };
4297 
4298     // Load from memory and interlace across 16 SIMD registers,
4299     // With each word from memory being broadcast to all lanes of
4300     // each successive SIMD register.
4301     //      Addr(0) -> All lanes in workSt[i]
4302     //      Addr(4) -> All lanes workSt[i + 1], etc.
4303     __ mov(tmpAddr, state);
4304     for (i = 0; i < 16; i += 4) {
4305       __ ld4r(workSt[i], workSt[i + 1], workSt[i + 2], workSt[i + 3], __ T4S,
4306           __ post(tmpAddr, 16));
4307     }
4308 
4309     // Pull in constant data.  The first 16 bytes are the add overlay
4310     // which is applied to the vector holding the counter (state[12]).
4311     // The second 16 bytes is the index register for the 8-bit left
4312     // rotation tbl instruction.
4313     __ adr(tmpAddr, L_cc20_const);
4314     __ ldpq(origCtrState, lrot8Tbl, Address(tmpAddr));
4315     __ addv(workSt[12], __ T4S, workSt[12], origCtrState);
4316 
4317     // Set up the 10 iteration loop and perform all 8 quarter round ops
4318     __ mov(loopCtr, 10);
4319     __ BIND(L_twoRounds);
4320 
4321     __ cc20_quarter_round(workSt[0], workSt[4], workSt[8], workSt[12],
4322         scratch, lrot8Tbl);
4323     __ cc20_quarter_round(workSt[1], workSt[5], workSt[9], workSt[13],
4324         scratch, lrot8Tbl);
4325     __ cc20_quarter_round(workSt[2], workSt[6], workSt[10], workSt[14],
4326         scratch, lrot8Tbl);
4327     __ cc20_quarter_round(workSt[3], workSt[7], workSt[11], workSt[15],
4328         scratch, lrot8Tbl);
4329 
4330     __ cc20_quarter_round(workSt[0], workSt[5], workSt[10], workSt[15],
4331         scratch, lrot8Tbl);
4332     __ cc20_quarter_round(workSt[1], workSt[6], workSt[11], workSt[12],
4333         scratch, lrot8Tbl);
4334     __ cc20_quarter_round(workSt[2], workSt[7], workSt[8], workSt[13],
4335         scratch, lrot8Tbl);
4336     __ cc20_quarter_round(workSt[3], workSt[4], workSt[9], workSt[14],
4337         scratch, lrot8Tbl);
4338 
4339     // Decrement and iterate
4340     __ sub(loopCtr, loopCtr, 1);
4341     __ cbnz(loopCtr, L_twoRounds);
4342 
4343     __ mov(tmpAddr, state);
4344 
4345     // Add the starting state back to the post-loop keystream
4346     // state.  We read/interlace the state array from memory into
4347     // 4 registers similar to what we did in the beginning.  Then
4348     // add the counter overlay onto workSt[12] at the end.
4349     for (i = 0; i < 16; i += 4) {
4350       __ ld4r(stateFirst, stateSecond, stateThird, stateFourth, __ T4S,
4351           __ post(tmpAddr, 16));
4352       __ addv(workSt[i], __ T4S, workSt[i], stateFirst);
4353       __ addv(workSt[i + 1], __ T4S, workSt[i + 1], stateSecond);
4354       __ addv(workSt[i + 2], __ T4S, workSt[i + 2], stateThird);
4355       __ addv(workSt[i + 3], __ T4S, workSt[i + 3], stateFourth);
4356     }
4357     __ addv(workSt[12], __ T4S, workSt[12], origCtrState);    // Add ctr mask
4358 
4359     // Write to key stream, storing the same element out of workSt[0..15]
4360     // to consecutive 4-byte offsets in the key stream buffer, then repeating
4361     // for the next element position.
4362     for (i = 0; i < 4; i++) {
4363       for (j = 0; j < 16; j += 4) {
4364         __ st4(workSt[j], workSt[j + 1], workSt[j + 2], workSt[j + 3], __ S, i,
4365             __ post(keystream, 16));
4366       }
4367     }
4368 
4369     __ mov(r0, 256);             // Return length of output keystream
4370     __ leave();
4371     __ ret(lr);
4372 
4373     return start;
4374   }
4375 
4376   /**
4377    *  Arguments:
4378    *
4379    * Inputs:
4380    *   c_rarg0   - int crc
4381    *   c_rarg1   - byte* buf
4382    *   c_rarg2   - int length
4383    *   c_rarg3   - int* table
4384    *
4385    * Output:
4386    *       r0   - int crc result
4387    */
4388   address generate_updateBytesCRC32C() {
4389     assert(UseCRC32CIntrinsics, "what are we doing here?");
4390 
4391     __ align(CodeEntryAlignment);
4392     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
4393 
4394     address start = __ pc();
4395 
4396     const Register crc   = c_rarg0;  // crc
4397     const Register buf   = c_rarg1;  // source java byte array address
4398     const Register len   = c_rarg2;  // length
4399     const Register table0 = c_rarg3; // crc_table address
4400     const Register table1 = c_rarg4;
4401     const Register table2 = c_rarg5;
4402     const Register table3 = c_rarg6;
4403     const Register tmp3 = c_rarg7;
4404 
4405     BLOCK_COMMENT("Entry:");
4406     __ enter(); // required for proper stackwalking of RuntimeStub frame
4407 
4408     __ kernel_crc32c(crc, buf, len,
4409               table0, table1, table2, table3, rscratch1, rscratch2, tmp3);
4410 
4411     __ leave(); // required for proper stackwalking of RuntimeStub frame
4412     __ ret(lr);
4413 
4414     return start;
4415   }
4416 
4417   /***
4418    *  Arguments:
4419    *
4420    *  Inputs:
4421    *   c_rarg0   - int   adler
4422    *   c_rarg1   - byte* buff
4423    *   c_rarg2   - int   len
4424    *
4425    * Output:
4426    *   c_rarg0   - int adler result
4427    */
4428   address generate_updateBytesAdler32() {
4429     __ align(CodeEntryAlignment);
4430     StubCodeMark mark(this, "StubRoutines", "updateBytesAdler32");
4431     address start = __ pc();
4432 
4433     Label L_simple_by1_loop, L_nmax, L_nmax_loop, L_by16, L_by16_loop, L_by1_loop, L_do_mod, L_combine, L_by1;
4434 
4435     // Aliases
4436     Register adler  = c_rarg0;
4437     Register s1     = c_rarg0;
4438     Register s2     = c_rarg3;
4439     Register buff   = c_rarg1;
4440     Register len    = c_rarg2;
4441     Register nmax  = r4;
4442     Register base  = r5;
4443     Register count = r6;
4444     Register temp0 = rscratch1;
4445     Register temp1 = rscratch2;
4446     FloatRegister vbytes = v0;
4447     FloatRegister vs1acc = v1;
4448     FloatRegister vs2acc = v2;
4449     FloatRegister vtable = v3;
4450 
4451     // Max number of bytes we can process before having to take the mod
4452     // 0x15B0 is 5552 in decimal, the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1
4453     uint64_t BASE = 0xfff1;
4454     uint64_t NMAX = 0x15B0;
4455 
4456     __ mov(base, BASE);
4457     __ mov(nmax, NMAX);
4458 
4459     // Load accumulation coefficients for the upper 16 bits
4460     __ lea(temp0, ExternalAddress((address) StubRoutines::aarch64::_adler_table));
4461     __ ld1(vtable, __ T16B, Address(temp0));
4462 
4463     // s1 is initialized to the lower 16 bits of adler
4464     // s2 is initialized to the upper 16 bits of adler
4465     __ ubfx(s2, adler, 16, 16);  // s2 = ((adler >> 16) & 0xffff)
4466     __ uxth(s1, adler);          // s1 = (adler & 0xffff)
4467 
4468     // The pipelined loop needs at least 16 elements for 1 iteration
4469     // It does check this, but it is more effective to skip to the cleanup loop
4470     __ cmp(len, (u1)16);
4471     __ br(Assembler::HS, L_nmax);
4472     __ cbz(len, L_combine);
4473 
4474     __ bind(L_simple_by1_loop);
4475     __ ldrb(temp0, Address(__ post(buff, 1)));
4476     __ add(s1, s1, temp0);
4477     __ add(s2, s2, s1);
4478     __ subs(len, len, 1);
4479     __ br(Assembler::HI, L_simple_by1_loop);
4480 
4481     // s1 = s1 % BASE
4482     __ subs(temp0, s1, base);
4483     __ csel(s1, temp0, s1, Assembler::HS);
4484 
4485     // s2 = s2 % BASE
4486     __ lsr(temp0, s2, 16);
4487     __ lsl(temp1, temp0, 4);
4488     __ sub(temp1, temp1, temp0);
4489     __ add(s2, temp1, s2, ext::uxth);
4490 
4491     __ subs(temp0, s2, base);
4492     __ csel(s2, temp0, s2, Assembler::HS);
4493 
4494     __ b(L_combine);
4495 
4496     __ bind(L_nmax);
4497     __ subs(len, len, nmax);
4498     __ sub(count, nmax, 16);
4499     __ br(Assembler::LO, L_by16);
4500 
4501     __ bind(L_nmax_loop);
4502 
4503     generate_updateBytesAdler32_accum(s1, s2, buff, temp0, temp1,
4504                                       vbytes, vs1acc, vs2acc, vtable);
4505 
4506     __ subs(count, count, 16);
4507     __ br(Assembler::HS, L_nmax_loop);
4508 
4509     // s1 = s1 % BASE
4510     __ lsr(temp0, s1, 16);
4511     __ lsl(temp1, temp0, 4);
4512     __ sub(temp1, temp1, temp0);
4513     __ add(temp1, temp1, s1, ext::uxth);
4514 
4515     __ lsr(temp0, temp1, 16);
4516     __ lsl(s1, temp0, 4);
4517     __ sub(s1, s1, temp0);
4518     __ add(s1, s1, temp1, ext:: uxth);
4519 
4520     __ subs(temp0, s1, base);
4521     __ csel(s1, temp0, s1, Assembler::HS);
4522 
4523     // s2 = s2 % BASE
4524     __ lsr(temp0, s2, 16);
4525     __ lsl(temp1, temp0, 4);
4526     __ sub(temp1, temp1, temp0);
4527     __ add(temp1, temp1, s2, ext::uxth);
4528 
4529     __ lsr(temp0, temp1, 16);
4530     __ lsl(s2, temp0, 4);
4531     __ sub(s2, s2, temp0);
4532     __ add(s2, s2, temp1, ext:: uxth);
4533 
4534     __ subs(temp0, s2, base);
4535     __ csel(s2, temp0, s2, Assembler::HS);
4536 
4537     __ subs(len, len, nmax);
4538     __ sub(count, nmax, 16);
4539     __ br(Assembler::HS, L_nmax_loop);
4540 
4541     __ bind(L_by16);
4542     __ adds(len, len, count);
4543     __ br(Assembler::LO, L_by1);
4544 
4545     __ bind(L_by16_loop);
4546 
4547     generate_updateBytesAdler32_accum(s1, s2, buff, temp0, temp1,
4548                                       vbytes, vs1acc, vs2acc, vtable);
4549 
4550     __ subs(len, len, 16);
4551     __ br(Assembler::HS, L_by16_loop);
4552 
4553     __ bind(L_by1);
4554     __ adds(len, len, 15);
4555     __ br(Assembler::LO, L_do_mod);
4556 
4557     __ bind(L_by1_loop);
4558     __ ldrb(temp0, Address(__ post(buff, 1)));
4559     __ add(s1, temp0, s1);
4560     __ add(s2, s2, s1);
4561     __ subs(len, len, 1);
4562     __ br(Assembler::HS, L_by1_loop);
4563 
4564     __ bind(L_do_mod);
4565     // s1 = s1 % BASE
4566     __ lsr(temp0, s1, 16);
4567     __ lsl(temp1, temp0, 4);
4568     __ sub(temp1, temp1, temp0);
4569     __ add(temp1, temp1, s1, ext::uxth);
4570 
4571     __ lsr(temp0, temp1, 16);
4572     __ lsl(s1, temp0, 4);
4573     __ sub(s1, s1, temp0);
4574     __ add(s1, s1, temp1, ext:: uxth);
4575 
4576     __ subs(temp0, s1, base);
4577     __ csel(s1, temp0, s1, Assembler::HS);
4578 
4579     // s2 = s2 % BASE
4580     __ lsr(temp0, s2, 16);
4581     __ lsl(temp1, temp0, 4);
4582     __ sub(temp1, temp1, temp0);
4583     __ add(temp1, temp1, s2, ext::uxth);
4584 
4585     __ lsr(temp0, temp1, 16);
4586     __ lsl(s2, temp0, 4);
4587     __ sub(s2, s2, temp0);
4588     __ add(s2, s2, temp1, ext:: uxth);
4589 
4590     __ subs(temp0, s2, base);
4591     __ csel(s2, temp0, s2, Assembler::HS);
4592 
4593     // Combine lower bits and higher bits
4594     __ bind(L_combine);
4595     __ orr(s1, s1, s2, Assembler::LSL, 16); // adler = s1 | (s2 << 16)
4596 
4597     __ ret(lr);
4598 
4599     return start;
4600   }
4601 
4602   void generate_updateBytesAdler32_accum(Register s1, Register s2, Register buff,
4603           Register temp0, Register temp1, FloatRegister vbytes,
4604           FloatRegister vs1acc, FloatRegister vs2acc, FloatRegister vtable) {
4605     // Below is a vectorized implementation of updating s1 and s2 for 16 bytes.
4606     // We use b1, b2, ..., b16 to denote the 16 bytes loaded in each iteration.
4607     // In non-vectorized code, we update s1 and s2 as:
4608     //   s1 <- s1 + b1
4609     //   s2 <- s2 + s1
4610     //   s1 <- s1 + b2
4611     //   s2 <- s2 + b1
4612     //   ...
4613     //   s1 <- s1 + b16
4614     //   s2 <- s2 + s1
4615     // Putting above assignments together, we have:
4616     //   s1_new = s1 + b1 + b2 + ... + b16
4617     //   s2_new = s2 + (s1 + b1) + (s1 + b1 + b2) + ... + (s1 + b1 + b2 + ... + b16)
4618     //          = s2 + s1 * 16 + (b1 * 16 + b2 * 15 + ... + b16 * 1)
4619     //          = s2 + s1 * 16 + (b1, b2, ... b16) dot (16, 15, ... 1)
4620     __ ld1(vbytes, __ T16B, Address(__ post(buff, 16)));
4621 
4622     // s2 = s2 + s1 * 16
4623     __ add(s2, s2, s1, Assembler::LSL, 4);
4624 
4625     // vs1acc = b1 + b2 + b3 + ... + b16
4626     // vs2acc = (b1 * 16) + (b2 * 15) + (b3 * 14) + ... + (b16 * 1)
4627     __ umullv(vs2acc, __ T8B, vtable, vbytes);
4628     __ umlalv(vs2acc, __ T16B, vtable, vbytes);
4629     __ uaddlv(vs1acc, __ T16B, vbytes);
4630     __ uaddlv(vs2acc, __ T8H, vs2acc);
4631 
4632     // s1 = s1 + vs1acc, s2 = s2 + vs2acc
4633     __ fmovd(temp0, vs1acc);
4634     __ fmovd(temp1, vs2acc);
4635     __ add(s1, s1, temp0);
4636     __ add(s2, s2, temp1);
4637   }
4638 
4639   /**
4640    *  Arguments:
4641    *
4642    *  Input:
4643    *    c_rarg0   - x address
4644    *    c_rarg1   - x length
4645    *    c_rarg2   - y address
4646    *    c_rarg3   - y length
4647    *    c_rarg4   - z address
4648    */
4649   address generate_multiplyToLen() {
4650     __ align(CodeEntryAlignment);
4651     StubCodeMark mark(this, "StubRoutines", "multiplyToLen");
4652 
4653     address start = __ pc();
4654     const Register x     = r0;
4655     const Register xlen  = r1;
4656     const Register y     = r2;
4657     const Register ylen  = r3;
4658     const Register z     = r4;
4659 
4660     const Register tmp0  = r5;
4661     const Register tmp1  = r10;
4662     const Register tmp2  = r11;
4663     const Register tmp3  = r12;
4664     const Register tmp4  = r13;
4665     const Register tmp5  = r14;
4666     const Register tmp6  = r15;
4667     const Register tmp7  = r16;
4668 
4669     BLOCK_COMMENT("Entry:");
4670     __ enter(); // required for proper stackwalking of RuntimeStub frame
4671     __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
4672     __ leave(); // required for proper stackwalking of RuntimeStub frame
4673     __ ret(lr);
4674 
4675     return start;
4676   }
4677 
4678   address generate_squareToLen() {
4679     // squareToLen algorithm for sizes 1..127 described in java code works
4680     // faster than multiply_to_len on some CPUs and slower on others, but
4681     // multiply_to_len shows a bit better overall results
4682     __ align(CodeEntryAlignment);
4683     StubCodeMark mark(this, "StubRoutines", "squareToLen");
4684     address start = __ pc();
4685 
4686     const Register x     = r0;
4687     const Register xlen  = r1;
4688     const Register z     = r2;
4689     const Register y     = r4; // == x
4690     const Register ylen  = r5; // == xlen
4691 
4692     const Register tmp0  = r3;
4693     const Register tmp1  = r10;
4694     const Register tmp2  = r11;
4695     const Register tmp3  = r12;
4696     const Register tmp4  = r13;
4697     const Register tmp5  = r14;
4698     const Register tmp6  = r15;
4699     const Register tmp7  = r16;
4700 
4701     RegSet spilled_regs = RegSet::of(y, ylen);
4702     BLOCK_COMMENT("Entry:");
4703     __ enter();
4704     __ push(spilled_regs, sp);
4705     __ mov(y, x);
4706     __ mov(ylen, xlen);
4707     __ multiply_to_len(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
4708     __ pop(spilled_regs, sp);
4709     __ leave();
4710     __ ret(lr);
4711     return start;
4712   }
4713 
4714   address generate_mulAdd() {
4715     __ align(CodeEntryAlignment);
4716     StubCodeMark mark(this, "StubRoutines", "mulAdd");
4717 
4718     address start = __ pc();
4719 
4720     const Register out     = r0;
4721     const Register in      = r1;
4722     const Register offset  = r2;
4723     const Register len     = r3;
4724     const Register k       = r4;
4725 
4726     BLOCK_COMMENT("Entry:");
4727     __ enter();
4728     __ mul_add(out, in, offset, len, k);
4729     __ leave();
4730     __ ret(lr);
4731 
4732     return start;
4733   }
4734 
4735   // Arguments:
4736   //
4737   // Input:
4738   //   c_rarg0   - newArr address
4739   //   c_rarg1   - oldArr address
4740   //   c_rarg2   - newIdx
4741   //   c_rarg3   - shiftCount
4742   //   c_rarg4   - numIter
4743   //
4744   address generate_bigIntegerRightShift() {
4745     __ align(CodeEntryAlignment);
4746     StubCodeMark mark(this,  "StubRoutines", "bigIntegerRightShiftWorker");
4747     address start = __ pc();
4748 
4749     Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit;
4750 
4751     Register newArr        = c_rarg0;
4752     Register oldArr        = c_rarg1;
4753     Register newIdx        = c_rarg2;
4754     Register shiftCount    = c_rarg3;
4755     Register numIter       = c_rarg4;
4756     Register idx           = numIter;
4757 
4758     Register newArrCur     = rscratch1;
4759     Register shiftRevCount = rscratch2;
4760     Register oldArrCur     = r13;
4761     Register oldArrNext    = r14;
4762 
4763     FloatRegister oldElem0        = v0;
4764     FloatRegister oldElem1        = v1;
4765     FloatRegister newElem         = v2;
4766     FloatRegister shiftVCount     = v3;
4767     FloatRegister shiftVRevCount  = v4;
4768 
4769     __ cbz(idx, Exit);
4770 
4771     __ add(newArr, newArr, newIdx, Assembler::LSL, 2);
4772 
4773     // left shift count
4774     __ movw(shiftRevCount, 32);
4775     __ subw(shiftRevCount, shiftRevCount, shiftCount);
4776 
4777     // numIter too small to allow a 4-words SIMD loop, rolling back
4778     __ cmp(numIter, (u1)4);
4779     __ br(Assembler::LT, ShiftThree);
4780 
4781     __ dup(shiftVCount,    __ T4S, shiftCount);
4782     __ dup(shiftVRevCount, __ T4S, shiftRevCount);
4783     __ negr(shiftVCount,   __ T4S, shiftVCount);
4784 
4785     __ BIND(ShiftSIMDLoop);
4786 
4787     // Calculate the load addresses
4788     __ sub(idx, idx, 4);
4789     __ add(oldArrNext, oldArr, idx, Assembler::LSL, 2);
4790     __ add(newArrCur,  newArr, idx, Assembler::LSL, 2);
4791     __ add(oldArrCur,  oldArrNext, 4);
4792 
4793     // Load 4 words and process
4794     __ ld1(oldElem0,  __ T4S,  Address(oldArrCur));
4795     __ ld1(oldElem1,  __ T4S,  Address(oldArrNext));
4796     __ ushl(oldElem0, __ T4S,  oldElem0, shiftVCount);
4797     __ ushl(oldElem1, __ T4S,  oldElem1, shiftVRevCount);
4798     __ orr(newElem,   __ T16B, oldElem0, oldElem1);
4799     __ st1(newElem,   __ T4S,  Address(newArrCur));
4800 
4801     __ cmp(idx, (u1)4);
4802     __ br(Assembler::LT, ShiftTwoLoop);
4803     __ b(ShiftSIMDLoop);
4804 
4805     __ BIND(ShiftTwoLoop);
4806     __ cbz(idx, Exit);
4807     __ cmp(idx, (u1)1);
4808     __ br(Assembler::EQ, ShiftOne);
4809 
4810     // Calculate the load addresses
4811     __ sub(idx, idx, 2);
4812     __ add(oldArrNext, oldArr, idx, Assembler::LSL, 2);
4813     __ add(newArrCur,  newArr, idx, Assembler::LSL, 2);
4814     __ add(oldArrCur,  oldArrNext, 4);
4815 
4816     // Load 2 words and process
4817     __ ld1(oldElem0,  __ T2S, Address(oldArrCur));
4818     __ ld1(oldElem1,  __ T2S, Address(oldArrNext));
4819     __ ushl(oldElem0, __ T2S, oldElem0, shiftVCount);
4820     __ ushl(oldElem1, __ T2S, oldElem1, shiftVRevCount);
4821     __ orr(newElem,   __ T8B, oldElem0, oldElem1);
4822     __ st1(newElem,   __ T2S, Address(newArrCur));
4823     __ b(ShiftTwoLoop);
4824 
4825     __ BIND(ShiftThree);
4826     __ tbz(idx, 1, ShiftOne);
4827     __ tbz(idx, 0, ShiftTwo);
4828     __ ldrw(r10,  Address(oldArr, 12));
4829     __ ldrw(r11,  Address(oldArr, 8));
4830     __ lsrvw(r10, r10, shiftCount);
4831     __ lslvw(r11, r11, shiftRevCount);
4832     __ orrw(r12,  r10, r11);
4833     __ strw(r12,  Address(newArr, 8));
4834 
4835     __ BIND(ShiftTwo);
4836     __ ldrw(r10,  Address(oldArr, 8));
4837     __ ldrw(r11,  Address(oldArr, 4));
4838     __ lsrvw(r10, r10, shiftCount);
4839     __ lslvw(r11, r11, shiftRevCount);
4840     __ orrw(r12,  r10, r11);
4841     __ strw(r12,  Address(newArr, 4));
4842 
4843     __ BIND(ShiftOne);
4844     __ ldrw(r10,  Address(oldArr, 4));
4845     __ ldrw(r11,  Address(oldArr));
4846     __ lsrvw(r10, r10, shiftCount);
4847     __ lslvw(r11, r11, shiftRevCount);
4848     __ orrw(r12,  r10, r11);
4849     __ strw(r12,  Address(newArr));
4850 
4851     __ BIND(Exit);
4852     __ ret(lr);
4853 
4854     return start;
4855   }
4856 
4857   // Arguments:
4858   //
4859   // Input:
4860   //   c_rarg0   - newArr address
4861   //   c_rarg1   - oldArr address
4862   //   c_rarg2   - newIdx
4863   //   c_rarg3   - shiftCount
4864   //   c_rarg4   - numIter
4865   //
4866   address generate_bigIntegerLeftShift() {
4867     __ align(CodeEntryAlignment);
4868     StubCodeMark mark(this,  "StubRoutines", "bigIntegerLeftShiftWorker");
4869     address start = __ pc();
4870 
4871     Label ShiftSIMDLoop, ShiftTwoLoop, ShiftThree, ShiftTwo, ShiftOne, Exit;
4872 
4873     Register newArr        = c_rarg0;
4874     Register oldArr        = c_rarg1;
4875     Register newIdx        = c_rarg2;
4876     Register shiftCount    = c_rarg3;
4877     Register numIter       = c_rarg4;
4878 
4879     Register shiftRevCount = rscratch1;
4880     Register oldArrNext    = rscratch2;
4881 
4882     FloatRegister oldElem0        = v0;
4883     FloatRegister oldElem1        = v1;
4884     FloatRegister newElem         = v2;
4885     FloatRegister shiftVCount     = v3;
4886     FloatRegister shiftVRevCount  = v4;
4887 
4888     __ cbz(numIter, Exit);
4889 
4890     __ add(oldArrNext, oldArr, 4);
4891     __ add(newArr, newArr, newIdx, Assembler::LSL, 2);
4892 
4893     // right shift count
4894     __ movw(shiftRevCount, 32);
4895     __ subw(shiftRevCount, shiftRevCount, shiftCount);
4896 
4897     // numIter too small to allow a 4-words SIMD loop, rolling back
4898     __ cmp(numIter, (u1)4);
4899     __ br(Assembler::LT, ShiftThree);
4900 
4901     __ dup(shiftVCount,     __ T4S, shiftCount);
4902     __ dup(shiftVRevCount,  __ T4S, shiftRevCount);
4903     __ negr(shiftVRevCount, __ T4S, shiftVRevCount);
4904 
4905     __ BIND(ShiftSIMDLoop);
4906 
4907     // load 4 words and process
4908     __ ld1(oldElem0,  __ T4S,  __ post(oldArr, 16));
4909     __ ld1(oldElem1,  __ T4S,  __ post(oldArrNext, 16));
4910     __ ushl(oldElem0, __ T4S,  oldElem0, shiftVCount);
4911     __ ushl(oldElem1, __ T4S,  oldElem1, shiftVRevCount);
4912     __ orr(newElem,   __ T16B, oldElem0, oldElem1);
4913     __ st1(newElem,   __ T4S,  __ post(newArr, 16));
4914     __ sub(numIter,   numIter, 4);
4915 
4916     __ cmp(numIter, (u1)4);
4917     __ br(Assembler::LT, ShiftTwoLoop);
4918     __ b(ShiftSIMDLoop);
4919 
4920     __ BIND(ShiftTwoLoop);
4921     __ cbz(numIter, Exit);
4922     __ cmp(numIter, (u1)1);
4923     __ br(Assembler::EQ, ShiftOne);
4924 
4925     // load 2 words and process
4926     __ ld1(oldElem0,  __ T2S,  __ post(oldArr, 8));
4927     __ ld1(oldElem1,  __ T2S,  __ post(oldArrNext, 8));
4928     __ ushl(oldElem0, __ T2S,  oldElem0, shiftVCount);
4929     __ ushl(oldElem1, __ T2S,  oldElem1, shiftVRevCount);
4930     __ orr(newElem,   __ T8B,  oldElem0, oldElem1);
4931     __ st1(newElem,   __ T2S,  __ post(newArr, 8));
4932     __ sub(numIter,   numIter, 2);
4933     __ b(ShiftTwoLoop);
4934 
4935     __ BIND(ShiftThree);
4936     __ ldrw(r10,  __ post(oldArr, 4));
4937     __ ldrw(r11,  __ post(oldArrNext, 4));
4938     __ lslvw(r10, r10, shiftCount);
4939     __ lsrvw(r11, r11, shiftRevCount);
4940     __ orrw(r12,  r10, r11);
4941     __ strw(r12,  __ post(newArr, 4));
4942     __ tbz(numIter, 1, Exit);
4943     __ tbz(numIter, 0, ShiftOne);
4944 
4945     __ BIND(ShiftTwo);
4946     __ ldrw(r10,  __ post(oldArr, 4));
4947     __ ldrw(r11,  __ post(oldArrNext, 4));
4948     __ lslvw(r10, r10, shiftCount);
4949     __ lsrvw(r11, r11, shiftRevCount);
4950     __ orrw(r12,  r10, r11);
4951     __ strw(r12,  __ post(newArr, 4));
4952 
4953     __ BIND(ShiftOne);
4954     __ ldrw(r10,  Address(oldArr));
4955     __ ldrw(r11,  Address(oldArrNext));
4956     __ lslvw(r10, r10, shiftCount);
4957     __ lsrvw(r11, r11, shiftRevCount);
4958     __ orrw(r12,  r10, r11);
4959     __ strw(r12,  Address(newArr));
4960 
4961     __ BIND(Exit);
4962     __ ret(lr);
4963 
4964     return start;
4965   }
4966 
4967   address generate_count_positives(address &count_positives_long) {
4968     const u1 large_loop_size = 64;
4969     const uint64_t UPPER_BIT_MASK=0x8080808080808080;
4970     int dcache_line = VM_Version::dcache_line_size();
4971 
4972     Register ary1 = r1, len = r2, result = r0;
4973 
4974     __ align(CodeEntryAlignment);
4975 
4976     StubCodeMark mark(this, "StubRoutines", "count_positives");
4977 
4978     address entry = __ pc();
4979 
4980     __ enter();
4981     // precondition: a copy of len is already in result
4982     // __ mov(result, len);
4983 
4984   Label RET_ADJUST, RET_ADJUST_16, RET_ADJUST_LONG, RET_NO_POP, RET_LEN, ALIGNED, LOOP16, CHECK_16,
4985         LARGE_LOOP, POST_LOOP16, LEN_OVER_15, LEN_OVER_8, POST_LOOP16_LOAD_TAIL;
4986 
4987   __ cmp(len, (u1)15);
4988   __ br(Assembler::GT, LEN_OVER_15);
4989   // The only case when execution falls into this code is when pointer is near
4990   // the end of memory page and we have to avoid reading next page
4991   __ add(ary1, ary1, len);
4992   __ subs(len, len, 8);
4993   __ br(Assembler::GT, LEN_OVER_8);
4994   __ ldr(rscratch2, Address(ary1, -8));
4995   __ sub(rscratch1, zr, len, __ LSL, 3);  // LSL 3 is to get bits from bytes.
4996   __ lsrv(rscratch2, rscratch2, rscratch1);
4997   __ tst(rscratch2, UPPER_BIT_MASK);
4998   __ csel(result, zr, result, Assembler::NE);
4999   __ leave();
5000   __ ret(lr);
5001   __ bind(LEN_OVER_8);
5002   __ ldp(rscratch1, rscratch2, Address(ary1, -16));
5003   __ sub(len, len, 8); // no data dep., then sub can be executed while loading
5004   __ tst(rscratch2, UPPER_BIT_MASK);
5005   __ br(Assembler::NE, RET_NO_POP);
5006   __ sub(rscratch2, zr, len, __ LSL, 3); // LSL 3 is to get bits from bytes
5007   __ lsrv(rscratch1, rscratch1, rscratch2);
5008   __ tst(rscratch1, UPPER_BIT_MASK);
5009   __ bind(RET_NO_POP);
5010   __ csel(result, zr, result, Assembler::NE);
5011   __ leave();
5012   __ ret(lr);
5013 
5014   Register tmp1 = r3, tmp2 = r4, tmp3 = r5, tmp4 = r6, tmp5 = r7, tmp6 = r10;
5015   const RegSet spilled_regs = RegSet::range(tmp1, tmp5) + tmp6;
5016 
5017   count_positives_long = __ pc(); // 2nd entry point
5018 
5019   __ enter();
5020 
5021   __ bind(LEN_OVER_15);
5022     __ push(spilled_regs, sp);
5023     __ andr(rscratch2, ary1, 15); // check pointer for 16-byte alignment
5024     __ cbz(rscratch2, ALIGNED);
5025     __ ldp(tmp6, tmp1, Address(ary1));
5026     __ mov(tmp5, 16);
5027     __ sub(rscratch1, tmp5, rscratch2); // amount of bytes until aligned address
5028     __ add(ary1, ary1, rscratch1);
5029     __ orr(tmp6, tmp6, tmp1);
5030     __ tst(tmp6, UPPER_BIT_MASK);
5031     __ br(Assembler::NE, RET_ADJUST);
5032     __ sub(len, len, rscratch1);
5033 
5034   __ bind(ALIGNED);
5035     __ cmp(len, large_loop_size);
5036     __ br(Assembler::LT, CHECK_16);
5037     // Perform 16-byte load as early return in pre-loop to handle situation
5038     // when initially aligned large array has negative values at starting bytes,
5039     // so LARGE_LOOP would do 4 reads instead of 1 (in worst case), which is
5040     // slower. Cases with negative bytes further ahead won't be affected that
5041     // much. In fact, it'll be faster due to early loads, less instructions and
5042     // less branches in LARGE_LOOP.
5043     __ ldp(tmp6, tmp1, Address(__ post(ary1, 16)));
5044     __ sub(len, len, 16);
5045     __ orr(tmp6, tmp6, tmp1);
5046     __ tst(tmp6, UPPER_BIT_MASK);
5047     __ br(Assembler::NE, RET_ADJUST_16);
5048     __ cmp(len, large_loop_size);
5049     __ br(Assembler::LT, CHECK_16);
5050 
5051     if (SoftwarePrefetchHintDistance >= 0
5052         && SoftwarePrefetchHintDistance >= dcache_line) {
5053       // initial prefetch
5054       __ prfm(Address(ary1, SoftwarePrefetchHintDistance - dcache_line));
5055     }
5056   __ bind(LARGE_LOOP);
5057     if (SoftwarePrefetchHintDistance >= 0) {
5058       __ prfm(Address(ary1, SoftwarePrefetchHintDistance));
5059     }
5060     // Issue load instructions first, since it can save few CPU/MEM cycles, also
5061     // instead of 4 triples of "orr(...), addr(...);cbnz(...);" (for each ldp)
5062     // better generate 7 * orr(...) + 1 andr(...) + 1 cbnz(...) which saves 3
5063     // instructions per cycle and have less branches, but this approach disables
5064     // early return, thus, all 64 bytes are loaded and checked every time.
5065     __ ldp(tmp2, tmp3, Address(ary1));
5066     __ ldp(tmp4, tmp5, Address(ary1, 16));
5067     __ ldp(rscratch1, rscratch2, Address(ary1, 32));
5068     __ ldp(tmp6, tmp1, Address(ary1, 48));
5069     __ add(ary1, ary1, large_loop_size);
5070     __ sub(len, len, large_loop_size);
5071     __ orr(tmp2, tmp2, tmp3);
5072     __ orr(tmp4, tmp4, tmp5);
5073     __ orr(rscratch1, rscratch1, rscratch2);
5074     __ orr(tmp6, tmp6, tmp1);
5075     __ orr(tmp2, tmp2, tmp4);
5076     __ orr(rscratch1, rscratch1, tmp6);
5077     __ orr(tmp2, tmp2, rscratch1);
5078     __ tst(tmp2, UPPER_BIT_MASK);
5079     __ br(Assembler::NE, RET_ADJUST_LONG);
5080     __ cmp(len, large_loop_size);
5081     __ br(Assembler::GE, LARGE_LOOP);
5082 
5083   __ bind(CHECK_16); // small 16-byte load pre-loop
5084     __ cmp(len, (u1)16);
5085     __ br(Assembler::LT, POST_LOOP16);
5086 
5087   __ bind(LOOP16); // small 16-byte load loop
5088     __ ldp(tmp2, tmp3, Address(__ post(ary1, 16)));
5089     __ sub(len, len, 16);
5090     __ orr(tmp2, tmp2, tmp3);
5091     __ tst(tmp2, UPPER_BIT_MASK);
5092     __ br(Assembler::NE, RET_ADJUST_16);
5093     __ cmp(len, (u1)16);
5094     __ br(Assembler::GE, LOOP16); // 16-byte load loop end
5095 
5096   __ bind(POST_LOOP16); // 16-byte aligned, so we can read unconditionally
5097     __ cmp(len, (u1)8);
5098     __ br(Assembler::LE, POST_LOOP16_LOAD_TAIL);
5099     __ ldr(tmp3, Address(__ post(ary1, 8)));
5100     __ tst(tmp3, UPPER_BIT_MASK);
5101     __ br(Assembler::NE, RET_ADJUST);
5102     __ sub(len, len, 8);
5103 
5104   __ bind(POST_LOOP16_LOAD_TAIL);
5105     __ cbz(len, RET_LEN); // Can't shift left by 64 when len==0
5106     __ ldr(tmp1, Address(ary1));
5107     __ mov(tmp2, 64);
5108     __ sub(tmp4, tmp2, len, __ LSL, 3);
5109     __ lslv(tmp1, tmp1, tmp4);
5110     __ tst(tmp1, UPPER_BIT_MASK);
5111     __ br(Assembler::NE, RET_ADJUST);
5112     // Fallthrough
5113 
5114   __ bind(RET_LEN);
5115     __ pop(spilled_regs, sp);
5116     __ leave();
5117     __ ret(lr);
5118 
5119     // difference result - len is the count of guaranteed to be
5120     // positive bytes
5121 
5122   __ bind(RET_ADJUST_LONG);
5123     __ add(len, len, (u1)(large_loop_size - 16));
5124   __ bind(RET_ADJUST_16);
5125     __ add(len, len, 16);
5126   __ bind(RET_ADJUST);
5127     __ pop(spilled_regs, sp);
5128     __ leave();
5129     __ sub(result, result, len);
5130     __ ret(lr);
5131 
5132     return entry;
5133   }
5134 
5135   void generate_large_array_equals_loop_nonsimd(int loopThreshold,
5136         bool usePrefetch, Label &NOT_EQUAL) {
5137     Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
5138         tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11,
5139         tmp7 = r12, tmp8 = r13;
5140     Label LOOP;
5141 
5142     __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize)));
5143     __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize)));
5144     __ bind(LOOP);
5145     if (usePrefetch) {
5146       __ prfm(Address(a1, SoftwarePrefetchHintDistance));
5147       __ prfm(Address(a2, SoftwarePrefetchHintDistance));
5148     }
5149     __ ldp(tmp5, tmp7, Address(__ post(a1, 2 * wordSize)));
5150     __ eor(tmp1, tmp1, tmp2);
5151     __ eor(tmp3, tmp3, tmp4);
5152     __ ldp(tmp6, tmp8, Address(__ post(a2, 2 * wordSize)));
5153     __ orr(tmp1, tmp1, tmp3);
5154     __ cbnz(tmp1, NOT_EQUAL);
5155     __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize)));
5156     __ eor(tmp5, tmp5, tmp6);
5157     __ eor(tmp7, tmp7, tmp8);
5158     __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize)));
5159     __ orr(tmp5, tmp5, tmp7);
5160     __ cbnz(tmp5, NOT_EQUAL);
5161     __ ldp(tmp5, tmp7, Address(__ post(a1, 2 * wordSize)));
5162     __ eor(tmp1, tmp1, tmp2);
5163     __ eor(tmp3, tmp3, tmp4);
5164     __ ldp(tmp6, tmp8, Address(__ post(a2, 2 * wordSize)));
5165     __ orr(tmp1, tmp1, tmp3);
5166     __ cbnz(tmp1, NOT_EQUAL);
5167     __ ldp(tmp1, tmp3, Address(__ post(a1, 2 * wordSize)));
5168     __ eor(tmp5, tmp5, tmp6);
5169     __ sub(cnt1, cnt1, 8 * wordSize);
5170     __ eor(tmp7, tmp7, tmp8);
5171     __ ldp(tmp2, tmp4, Address(__ post(a2, 2 * wordSize)));
5172     // tmp6 is not used. MacroAssembler::subs is used here (rather than
5173     // cmp) because subs allows an unlimited range of immediate operand.
5174     __ subs(tmp6, cnt1, loopThreshold);
5175     __ orr(tmp5, tmp5, tmp7);
5176     __ cbnz(tmp5, NOT_EQUAL);
5177     __ br(__ GE, LOOP);
5178     // post-loop
5179     __ eor(tmp1, tmp1, tmp2);
5180     __ eor(tmp3, tmp3, tmp4);
5181     __ orr(tmp1, tmp1, tmp3);
5182     __ sub(cnt1, cnt1, 2 * wordSize);
5183     __ cbnz(tmp1, NOT_EQUAL);
5184   }
5185 
5186   void generate_large_array_equals_loop_simd(int loopThreshold,
5187         bool usePrefetch, Label &NOT_EQUAL) {
5188     Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
5189         tmp2 = rscratch2;
5190     Label LOOP;
5191 
5192     __ bind(LOOP);
5193     if (usePrefetch) {
5194       __ prfm(Address(a1, SoftwarePrefetchHintDistance));
5195       __ prfm(Address(a2, SoftwarePrefetchHintDistance));
5196     }
5197     __ ld1(v0, v1, v2, v3, __ T2D, Address(__ post(a1, 4 * 2 * wordSize)));
5198     __ sub(cnt1, cnt1, 8 * wordSize);
5199     __ ld1(v4, v5, v6, v7, __ T2D, Address(__ post(a2, 4 * 2 * wordSize)));
5200     __ subs(tmp1, cnt1, loopThreshold);
5201     __ eor(v0, __ T16B, v0, v4);
5202     __ eor(v1, __ T16B, v1, v5);
5203     __ eor(v2, __ T16B, v2, v6);
5204     __ eor(v3, __ T16B, v3, v7);
5205     __ orr(v0, __ T16B, v0, v1);
5206     __ orr(v1, __ T16B, v2, v3);
5207     __ orr(v0, __ T16B, v0, v1);
5208     __ umov(tmp1, v0, __ D, 0);
5209     __ umov(tmp2, v0, __ D, 1);
5210     __ orr(tmp1, tmp1, tmp2);
5211     __ cbnz(tmp1, NOT_EQUAL);
5212     __ br(__ GE, LOOP);
5213   }
5214 
5215   // a1 = r1 - array1 address
5216   // a2 = r2 - array2 address
5217   // result = r0 - return value. Already contains "false"
5218   // cnt1 = r10 - amount of elements left to check, reduced by wordSize
5219   // r3-r5 are reserved temporary registers
5220   // Clobbers: v0-v7 when UseSIMDForArrayEquals, rscratch1, rscratch2
5221   address generate_large_array_equals() {
5222     Register a1 = r1, a2 = r2, result = r0, cnt1 = r10, tmp1 = rscratch1,
5223         tmp2 = rscratch2, tmp3 = r3, tmp4 = r4, tmp5 = r5, tmp6 = r11,
5224         tmp7 = r12, tmp8 = r13;
5225     Label TAIL, NOT_EQUAL, EQUAL, NOT_EQUAL_NO_POP, NO_PREFETCH_LARGE_LOOP,
5226         SMALL_LOOP, POST_LOOP;
5227     const int PRE_LOOP_SIZE = UseSIMDForArrayEquals ? 0 : 16;
5228     // calculate if at least 32 prefetched bytes are used
5229     int prefetchLoopThreshold = SoftwarePrefetchHintDistance + 32;
5230     int nonPrefetchLoopThreshold = (64 + PRE_LOOP_SIZE);
5231     RegSet spilled_regs = RegSet::range(tmp6, tmp8);
5232     assert_different_registers(a1, a2, result, cnt1, tmp1, tmp2, tmp3, tmp4,
5233         tmp5, tmp6, tmp7, tmp8);
5234 
5235     __ align(CodeEntryAlignment);
5236 
5237     StubCodeMark mark(this, "StubRoutines", "large_array_equals");
5238 
5239     address entry = __ pc();
5240     __ enter();
5241     __ sub(cnt1, cnt1, wordSize);  // first 8 bytes were loaded outside of stub
5242     // also advance pointers to use post-increment instead of pre-increment
5243     __ add(a1, a1, wordSize);
5244     __ add(a2, a2, wordSize);
5245     if (AvoidUnalignedAccesses) {
5246       // both implementations (SIMD/nonSIMD) are using relatively large load
5247       // instructions (ld1/ldp), which has huge penalty (up to x2 exec time)
5248       // on some CPUs in case of address is not at least 16-byte aligned.
5249       // Arrays are 8-byte aligned currently, so, we can make additional 8-byte
5250       // load if needed at least for 1st address and make if 16-byte aligned.
5251       Label ALIGNED16;
5252       __ tbz(a1, 3, ALIGNED16);
5253       __ ldr(tmp1, Address(__ post(a1, wordSize)));
5254       __ ldr(tmp2, Address(__ post(a2, wordSize)));
5255       __ sub(cnt1, cnt1, wordSize);
5256       __ eor(tmp1, tmp1, tmp2);
5257       __ cbnz(tmp1, NOT_EQUAL_NO_POP);
5258       __ bind(ALIGNED16);
5259     }
5260     if (UseSIMDForArrayEquals) {
5261       if (SoftwarePrefetchHintDistance >= 0) {
5262         __ subs(tmp1, cnt1, prefetchLoopThreshold);
5263         __ br(__ LE, NO_PREFETCH_LARGE_LOOP);
5264         generate_large_array_equals_loop_simd(prefetchLoopThreshold,
5265             /* prfm = */ true, NOT_EQUAL);
5266         __ subs(zr, cnt1, nonPrefetchLoopThreshold);
5267         __ br(__ LT, TAIL);
5268       }
5269       __ bind(NO_PREFETCH_LARGE_LOOP);
5270       generate_large_array_equals_loop_simd(nonPrefetchLoopThreshold,
5271           /* prfm = */ false, NOT_EQUAL);
5272     } else {
5273       __ push(spilled_regs, sp);
5274       if (SoftwarePrefetchHintDistance >= 0) {
5275         __ subs(tmp1, cnt1, prefetchLoopThreshold);
5276         __ br(__ LE, NO_PREFETCH_LARGE_LOOP);
5277         generate_large_array_equals_loop_nonsimd(prefetchLoopThreshold,
5278             /* prfm = */ true, NOT_EQUAL);
5279         __ subs(zr, cnt1, nonPrefetchLoopThreshold);
5280         __ br(__ LT, TAIL);
5281       }
5282       __ bind(NO_PREFETCH_LARGE_LOOP);
5283       generate_large_array_equals_loop_nonsimd(nonPrefetchLoopThreshold,
5284           /* prfm = */ false, NOT_EQUAL);
5285     }
5286     __ bind(TAIL);
5287       __ cbz(cnt1, EQUAL);
5288       __ subs(cnt1, cnt1, wordSize);
5289       __ br(__ LE, POST_LOOP);
5290     __ bind(SMALL_LOOP);
5291       __ ldr(tmp1, Address(__ post(a1, wordSize)));
5292       __ ldr(tmp2, Address(__ post(a2, wordSize)));
5293       __ subs(cnt1, cnt1, wordSize);
5294       __ eor(tmp1, tmp1, tmp2);
5295       __ cbnz(tmp1, NOT_EQUAL);
5296       __ br(__ GT, SMALL_LOOP);
5297     __ bind(POST_LOOP);
5298       __ ldr(tmp1, Address(a1, cnt1));
5299       __ ldr(tmp2, Address(a2, cnt1));
5300       __ eor(tmp1, tmp1, tmp2);
5301       __ cbnz(tmp1, NOT_EQUAL);
5302     __ bind(EQUAL);
5303       __ mov(result, true);
5304     __ bind(NOT_EQUAL);
5305       if (!UseSIMDForArrayEquals) {
5306         __ pop(spilled_regs, sp);
5307       }
5308     __ bind(NOT_EQUAL_NO_POP);
5309     __ leave();
5310     __ ret(lr);
5311     return entry;
5312   }
5313 
5314   address generate_dsin_dcos(bool isCos) {
5315     __ align(CodeEntryAlignment);
5316     StubCodeMark mark(this, "StubRoutines", isCos ? "libmDcos" : "libmDsin");
5317     address start = __ pc();
5318     __ generate_dsin_dcos(isCos, (address)StubRoutines::aarch64::_npio2_hw,
5319         (address)StubRoutines::aarch64::_two_over_pi,
5320         (address)StubRoutines::aarch64::_pio2,
5321         (address)StubRoutines::aarch64::_dsin_coef,
5322         (address)StubRoutines::aarch64::_dcos_coef);
5323     return start;
5324   }
5325 
5326   // code for comparing 16 characters of strings with Latin1 and Utf16 encoding
5327   void compare_string_16_x_LU(Register tmpL, Register tmpU, Label &DIFF1,
5328       Label &DIFF2) {
5329     Register cnt1 = r2, tmp2 = r11, tmp3 = r12;
5330     FloatRegister vtmp = v1, vtmpZ = v0, vtmp3 = v2;
5331 
5332     __ ldrq(vtmp, Address(__ post(tmp2, 16)));
5333     __ ldr(tmpU, Address(__ post(cnt1, 8)));
5334     __ zip1(vtmp3, __ T16B, vtmp, vtmpZ);
5335     // now we have 32 bytes of characters (converted to U) in vtmp:vtmp3
5336 
5337     __ fmovd(tmpL, vtmp3);
5338     __ eor(rscratch2, tmp3, tmpL);
5339     __ cbnz(rscratch2, DIFF2);
5340 
5341     __ ldr(tmp3, Address(__ post(cnt1, 8)));
5342     __ umov(tmpL, vtmp3, __ D, 1);
5343     __ eor(rscratch2, tmpU, tmpL);
5344     __ cbnz(rscratch2, DIFF1);
5345 
5346     __ zip2(vtmp, __ T16B, vtmp, vtmpZ);
5347     __ ldr(tmpU, Address(__ post(cnt1, 8)));
5348     __ fmovd(tmpL, vtmp);
5349     __ eor(rscratch2, tmp3, tmpL);
5350     __ cbnz(rscratch2, DIFF2);
5351 
5352     __ ldr(tmp3, Address(__ post(cnt1, 8)));
5353     __ umov(tmpL, vtmp, __ D, 1);
5354     __ eor(rscratch2, tmpU, tmpL);
5355     __ cbnz(rscratch2, DIFF1);
5356   }
5357 
5358   // r0  = result
5359   // r1  = str1
5360   // r2  = cnt1
5361   // r3  = str2
5362   // r4  = cnt2
5363   // r10 = tmp1
5364   // r11 = tmp2
5365   address generate_compare_long_string_different_encoding(bool isLU) {
5366     __ align(CodeEntryAlignment);
5367     StubCodeMark mark(this, "StubRoutines", isLU
5368         ? "compare_long_string_different_encoding LU"
5369         : "compare_long_string_different_encoding UL");
5370     address entry = __ pc();
5371     Label SMALL_LOOP, TAIL, TAIL_LOAD_16, LOAD_LAST, DIFF1, DIFF2,
5372         DONE, CALCULATE_DIFFERENCE, LARGE_LOOP_PREFETCH, NO_PREFETCH,
5373         LARGE_LOOP_PREFETCH_REPEAT1, LARGE_LOOP_PREFETCH_REPEAT2;
5374     Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
5375         tmp1 = r10, tmp2 = r11, tmp3 = r12, tmp4 = r14;
5376     FloatRegister vtmpZ = v0, vtmp = v1, vtmp3 = v2;
5377     RegSet spilled_regs = RegSet::of(tmp3, tmp4);
5378 
5379     int prefetchLoopExitCondition = MAX2(64, SoftwarePrefetchHintDistance/2);
5380 
5381     __ eor(vtmpZ, __ T16B, vtmpZ, vtmpZ);
5382     // cnt2 == amount of characters left to compare
5383     // Check already loaded first 4 symbols(vtmp and tmp2(LU)/tmp1(UL))
5384     __ zip1(vtmp, __ T8B, vtmp, vtmpZ);
5385     __ add(str1, str1, isLU ? wordSize/2 : wordSize);
5386     __ add(str2, str2, isLU ? wordSize : wordSize/2);
5387     __ fmovd(isLU ? tmp1 : tmp2, vtmp);
5388     __ subw(cnt2, cnt2, 8); // Already loaded 4 symbols. Last 4 is special case.
5389     __ eor(rscratch2, tmp1, tmp2);
5390     __ mov(rscratch1, tmp2);
5391     __ cbnz(rscratch2, CALCULATE_DIFFERENCE);
5392     Register tmpU = isLU ? rscratch1 : tmp1, // where to keep U for comparison
5393              tmpL = isLU ? tmp1 : rscratch1; // where to keep L for comparison
5394     __ push(spilled_regs, sp);
5395     __ mov(tmp2, isLU ? str1 : str2); // init the pointer to L next load
5396     __ mov(cnt1, isLU ? str2 : str1); // init the pointer to U next load
5397 
5398     __ ldr(tmp3, Address(__ post(cnt1, 8)));
5399 
5400     if (SoftwarePrefetchHintDistance >= 0) {
5401       __ subs(rscratch2, cnt2, prefetchLoopExitCondition);
5402       __ br(__ LT, NO_PREFETCH);
5403       __ bind(LARGE_LOOP_PREFETCH);
5404         __ prfm(Address(tmp2, SoftwarePrefetchHintDistance));
5405         __ mov(tmp4, 2);
5406         __ prfm(Address(cnt1, SoftwarePrefetchHintDistance));
5407         __ bind(LARGE_LOOP_PREFETCH_REPEAT1);
5408           compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2);
5409           __ subs(tmp4, tmp4, 1);
5410           __ br(__ GT, LARGE_LOOP_PREFETCH_REPEAT1);
5411           __ prfm(Address(cnt1, SoftwarePrefetchHintDistance));
5412           __ mov(tmp4, 2);
5413         __ bind(LARGE_LOOP_PREFETCH_REPEAT2);
5414           compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2);
5415           __ subs(tmp4, tmp4, 1);
5416           __ br(__ GT, LARGE_LOOP_PREFETCH_REPEAT2);
5417           __ sub(cnt2, cnt2, 64);
5418           __ subs(rscratch2, cnt2, prefetchLoopExitCondition);
5419           __ br(__ GE, LARGE_LOOP_PREFETCH);
5420     }
5421     __ cbz(cnt2, LOAD_LAST); // no characters left except last load
5422     __ bind(NO_PREFETCH);
5423     __ subs(cnt2, cnt2, 16);
5424     __ br(__ LT, TAIL);
5425     __ align(OptoLoopAlignment);
5426     __ bind(SMALL_LOOP); // smaller loop
5427       __ subs(cnt2, cnt2, 16);
5428       compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2);
5429       __ br(__ GE, SMALL_LOOP);
5430       __ cmn(cnt2, (u1)16);
5431       __ br(__ EQ, LOAD_LAST);
5432     __ bind(TAIL); // 1..15 characters left until last load (last 4 characters)
5433       __ add(cnt1, cnt1, cnt2, __ LSL, 1); // Address of 32 bytes before last 4 characters in UTF-16 string
5434       __ add(tmp2, tmp2, cnt2); // Address of 16 bytes before last 4 characters in Latin1 string
5435       __ ldr(tmp3, Address(cnt1, -8));
5436       compare_string_16_x_LU(tmpL, tmpU, DIFF1, DIFF2); // last 16 characters before last load
5437       __ b(LOAD_LAST);
5438     __ bind(DIFF2);
5439       __ mov(tmpU, tmp3);
5440     __ bind(DIFF1);
5441       __ pop(spilled_regs, sp);
5442       __ b(CALCULATE_DIFFERENCE);
5443     __ bind(LOAD_LAST);
5444       // Last 4 UTF-16 characters are already pre-loaded into tmp3 by compare_string_16_x_LU.
5445       // No need to load it again
5446       __ mov(tmpU, tmp3);
5447       __ pop(spilled_regs, sp);
5448 
5449       // tmp2 points to the address of the last 4 Latin1 characters right now
5450       __ ldrs(vtmp, Address(tmp2));
5451       __ zip1(vtmp, __ T8B, vtmp, vtmpZ);
5452       __ fmovd(tmpL, vtmp);
5453 
5454       __ eor(rscratch2, tmpU, tmpL);
5455       __ cbz(rscratch2, DONE);
5456 
5457     // Find the first different characters in the longwords and
5458     // compute their difference.
5459     __ bind(CALCULATE_DIFFERENCE);
5460       __ rev(rscratch2, rscratch2);
5461       __ clz(rscratch2, rscratch2);
5462       __ andr(rscratch2, rscratch2, -16);
5463       __ lsrv(tmp1, tmp1, rscratch2);
5464       __ uxthw(tmp1, tmp1);
5465       __ lsrv(rscratch1, rscratch1, rscratch2);
5466       __ uxthw(rscratch1, rscratch1);
5467       __ subw(result, tmp1, rscratch1);
5468     __ bind(DONE);
5469       __ ret(lr);
5470     return entry;
5471   }
5472 
5473   // r0 = input (float16)
5474   // v0 = result (float)
5475   // v1 = temporary float register
5476   address generate_float16ToFloat() {
5477     __ align(CodeEntryAlignment);
5478     StubCodeMark mark(this, "StubRoutines", "float16ToFloat");
5479     address entry = __ pc();
5480     BLOCK_COMMENT("Entry:");
5481     __ flt16_to_flt(v0, r0, v1);
5482     __ ret(lr);
5483     return entry;
5484   }
5485 
5486   // v0 = input (float)
5487   // r0 = result (float16)
5488   // v1 = temporary float register
5489   address generate_floatToFloat16() {
5490     __ align(CodeEntryAlignment);
5491     StubCodeMark mark(this, "StubRoutines", "floatToFloat16");
5492     address entry = __ pc();
5493     BLOCK_COMMENT("Entry:");
5494     __ flt_to_flt16(r0, v0, v1);
5495     __ ret(lr);
5496     return entry;
5497   }
5498 
5499   address generate_method_entry_barrier() {
5500     __ align(CodeEntryAlignment);
5501     StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier");
5502 
5503     Label deoptimize_label;
5504 
5505     address start = __ pc();
5506 
5507     BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
5508 
5509     if (bs_asm->nmethod_patching_type() == NMethodPatchingType::conc_instruction_and_data_patch) {
5510       BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
5511       // We can get here despite the nmethod being good, if we have not
5512       // yet applied our cross modification fence (or data fence).
5513       Address thread_epoch_addr(rthread, in_bytes(bs_nm->thread_disarmed_guard_value_offset()) + 4);
5514       __ lea(rscratch2, ExternalAddress(bs_asm->patching_epoch_addr()));
5515       __ ldrw(rscratch2, rscratch2);
5516       __ strw(rscratch2, thread_epoch_addr);
5517       __ isb();
5518       __ membar(__ LoadLoad);
5519     }
5520 
5521     __ set_last_Java_frame(sp, rfp, lr, rscratch1);
5522 
5523     __ enter();
5524     __ add(rscratch2, sp, wordSize);  // rscratch2 points to the saved lr
5525 
5526     __ sub(sp, sp, 4 * wordSize);  // four words for the returned {sp, fp, lr, pc}
5527 
5528     __ push_call_clobbered_registers();
5529 
5530     __ mov(c_rarg0, rscratch2);
5531     __ call_VM_leaf
5532          (CAST_FROM_FN_PTR
5533           (address, BarrierSetNMethod::nmethod_stub_entry_barrier), 1);
5534 
5535     __ reset_last_Java_frame(true);
5536 
5537     __ mov(rscratch1, r0);
5538 
5539     __ pop_call_clobbered_registers();
5540 
5541     __ cbnz(rscratch1, deoptimize_label);
5542 
5543     __ leave();
5544     __ ret(lr);
5545 
5546     __ BIND(deoptimize_label);
5547 
5548     __ ldp(/* new sp */ rscratch1, rfp, Address(sp, 0 * wordSize));
5549     __ ldp(lr, /* new pc*/ rscratch2, Address(sp, 2 * wordSize));
5550 
5551     __ mov(sp, rscratch1);
5552     __ br(rscratch2);
5553 
5554     return start;
5555   }
5556 
5557   // r0  = result
5558   // r1  = str1
5559   // r2  = cnt1
5560   // r3  = str2
5561   // r4  = cnt2
5562   // r10 = tmp1
5563   // r11 = tmp2
5564   address generate_compare_long_string_same_encoding(bool isLL) {
5565     __ align(CodeEntryAlignment);
5566     StubCodeMark mark(this, "StubRoutines", isLL
5567         ? "compare_long_string_same_encoding LL"
5568         : "compare_long_string_same_encoding UU");
5569     address entry = __ pc();
5570     Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
5571         tmp1 = r10, tmp2 = r11, tmp1h = rscratch1, tmp2h = rscratch2;
5572 
5573     Label LARGE_LOOP_PREFETCH, LOOP_COMPARE16, DIFF, LESS16, LESS8, CAL_DIFFERENCE, LENGTH_DIFF;
5574 
5575     // exit from large loop when less than 64 bytes left to read or we're about
5576     // to prefetch memory behind array border
5577     int largeLoopExitCondition = MAX2(64, SoftwarePrefetchHintDistance)/(isLL ? 1 : 2);
5578 
5579     // before jumping to stub, pre-load 8 bytes already, so do comparison directly
5580     __ eor(rscratch2, tmp1, tmp2);
5581     __ cbnz(rscratch2, CAL_DIFFERENCE);
5582 
5583     __ sub(cnt2, cnt2, wordSize/(isLL ? 1 : 2));
5584     // update pointers, because of previous read
5585     __ add(str1, str1, wordSize);
5586     __ add(str2, str2, wordSize);
5587     if (SoftwarePrefetchHintDistance >= 0) {
5588       __ align(OptoLoopAlignment);
5589       __ bind(LARGE_LOOP_PREFETCH);
5590         __ prfm(Address(str1, SoftwarePrefetchHintDistance));
5591         __ prfm(Address(str2, SoftwarePrefetchHintDistance));
5592 
5593         for (int i = 0; i < 4; i++) {
5594           __ ldp(tmp1, tmp1h, Address(str1, i * 16));
5595           __ ldp(tmp2, tmp2h, Address(str2, i * 16));
5596           __ cmp(tmp1, tmp2);
5597           __ ccmp(tmp1h, tmp2h, 0, Assembler::EQ);
5598           __ br(Assembler::NE, DIFF);
5599         }
5600         __ sub(cnt2, cnt2, isLL ? 64 : 32);
5601         __ add(str1, str1, 64);
5602         __ add(str2, str2, 64);
5603         __ subs(rscratch2, cnt2, largeLoopExitCondition);
5604         __ br(Assembler::GE, LARGE_LOOP_PREFETCH);
5605         __ cbz(cnt2, LENGTH_DIFF); // no more chars left?
5606     }
5607 
5608     __ subs(rscratch1, cnt2, isLL ? 16 : 8);
5609     __ br(Assembler::LE, LESS16);
5610     __ align(OptoLoopAlignment);
5611     __ bind(LOOP_COMPARE16);
5612       __ ldp(tmp1, tmp1h, Address(__ post(str1, 16)));
5613       __ ldp(tmp2, tmp2h, Address(__ post(str2, 16)));
5614       __ cmp(tmp1, tmp2);
5615       __ ccmp(tmp1h, tmp2h, 0, Assembler::EQ);
5616       __ br(Assembler::NE, DIFF);
5617       __ sub(cnt2, cnt2, isLL ? 16 : 8);
5618       __ subs(rscratch2, cnt2, isLL ? 16 : 8);
5619       __ br(Assembler::LT, LESS16);
5620 
5621       __ ldp(tmp1, tmp1h, Address(__ post(str1, 16)));
5622       __ ldp(tmp2, tmp2h, Address(__ post(str2, 16)));
5623       __ cmp(tmp1, tmp2);
5624       __ ccmp(tmp1h, tmp2h, 0, Assembler::EQ);
5625       __ br(Assembler::NE, DIFF);
5626       __ sub(cnt2, cnt2, isLL ? 16 : 8);
5627       __ subs(rscratch2, cnt2, isLL ? 16 : 8);
5628       __ br(Assembler::GE, LOOP_COMPARE16);
5629       __ cbz(cnt2, LENGTH_DIFF);
5630 
5631     __ bind(LESS16);
5632       // each 8 compare
5633       __ subs(cnt2, cnt2, isLL ? 8 : 4);
5634       __ br(Assembler::LE, LESS8);
5635       __ ldr(tmp1, Address(__ post(str1, 8)));
5636       __ ldr(tmp2, Address(__ post(str2, 8)));
5637       __ eor(rscratch2, tmp1, tmp2);
5638       __ cbnz(rscratch2, CAL_DIFFERENCE);
5639       __ sub(cnt2, cnt2, isLL ? 8 : 4);
5640 
5641     __ bind(LESS8); // directly load last 8 bytes
5642       if (!isLL) {
5643         __ add(cnt2, cnt2, cnt2);
5644       }
5645       __ ldr(tmp1, Address(str1, cnt2));
5646       __ ldr(tmp2, Address(str2, cnt2));
5647       __ eor(rscratch2, tmp1, tmp2);
5648       __ cbz(rscratch2, LENGTH_DIFF);
5649       __ b(CAL_DIFFERENCE);
5650 
5651     __ bind(DIFF);
5652       __ cmp(tmp1, tmp2);
5653       __ csel(tmp1, tmp1, tmp1h, Assembler::NE);
5654       __ csel(tmp2, tmp2, tmp2h, Assembler::NE);
5655       // reuse rscratch2 register for the result of eor instruction
5656       __ eor(rscratch2, tmp1, tmp2);
5657 
5658     __ bind(CAL_DIFFERENCE);
5659       __ rev(rscratch2, rscratch2);
5660       __ clz(rscratch2, rscratch2);
5661       __ andr(rscratch2, rscratch2, isLL ? -8 : -16);
5662       __ lsrv(tmp1, tmp1, rscratch2);
5663       __ lsrv(tmp2, tmp2, rscratch2);
5664       if (isLL) {
5665         __ uxtbw(tmp1, tmp1);
5666         __ uxtbw(tmp2, tmp2);
5667       } else {
5668         __ uxthw(tmp1, tmp1);
5669         __ uxthw(tmp2, tmp2);
5670       }
5671       __ subw(result, tmp1, tmp2);
5672 
5673     __ bind(LENGTH_DIFF);
5674       __ ret(lr);
5675     return entry;
5676   }
5677 
5678   enum string_compare_mode {
5679     LL,
5680     LU,
5681     UL,
5682     UU,
5683   };
5684 
5685   // The following registers are declared in aarch64.ad
5686   // r0  = result
5687   // r1  = str1
5688   // r2  = cnt1
5689   // r3  = str2
5690   // r4  = cnt2
5691   // r10 = tmp1
5692   // r11 = tmp2
5693   // z0  = ztmp1
5694   // z1  = ztmp2
5695   // p0  = pgtmp1
5696   // p1  = pgtmp2
5697   address generate_compare_long_string_sve(string_compare_mode mode) {
5698     __ align(CodeEntryAlignment);
5699     address entry = __ pc();
5700     Register result = r0, str1 = r1, cnt1 = r2, str2 = r3, cnt2 = r4,
5701              tmp1 = r10, tmp2 = r11;
5702 
5703     Label LOOP, DONE, MISMATCH;
5704     Register vec_len = tmp1;
5705     Register idx = tmp2;
5706     // The minimum of the string lengths has been stored in cnt2.
5707     Register cnt = cnt2;
5708     FloatRegister ztmp1 = z0, ztmp2 = z1;
5709     PRegister pgtmp1 = p0, pgtmp2 = p1;
5710 
5711 #define LOAD_PAIR(ztmp1, ztmp2, pgtmp1, src1, src2, idx)                       \
5712     switch (mode) {                                                            \
5713       case LL:                                                                 \
5714         __ sve_ld1b(ztmp1, __ B, pgtmp1, Address(str1, idx));                  \
5715         __ sve_ld1b(ztmp2, __ B, pgtmp1, Address(str2, idx));                  \
5716         break;                                                                 \
5717       case LU:                                                                 \
5718         __ sve_ld1b(ztmp1, __ H, pgtmp1, Address(str1, idx));                  \
5719         __ sve_ld1h(ztmp2, __ H, pgtmp1, Address(str2, idx, Address::lsl(1))); \
5720         break;                                                                 \
5721       case UL:                                                                 \
5722         __ sve_ld1h(ztmp1, __ H, pgtmp1, Address(str1, idx, Address::lsl(1))); \
5723         __ sve_ld1b(ztmp2, __ H, pgtmp1, Address(str2, idx));                  \
5724         break;                                                                 \
5725       case UU:                                                                 \
5726         __ sve_ld1h(ztmp1, __ H, pgtmp1, Address(str1, idx, Address::lsl(1))); \
5727         __ sve_ld1h(ztmp2, __ H, pgtmp1, Address(str2, idx, Address::lsl(1))); \
5728         break;                                                                 \
5729       default:                                                                 \
5730         ShouldNotReachHere();                                                  \
5731     }
5732 
5733     const char* stubname;
5734     switch (mode) {
5735       case LL: stubname = "compare_long_string_same_encoding LL";      break;
5736       case LU: stubname = "compare_long_string_different_encoding LU"; break;
5737       case UL: stubname = "compare_long_string_different_encoding UL"; break;
5738       case UU: stubname = "compare_long_string_same_encoding UU";      break;
5739       default: ShouldNotReachHere();
5740     }
5741 
5742     StubCodeMark mark(this, "StubRoutines", stubname);
5743 
5744     __ mov(idx, 0);
5745     __ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt);
5746 
5747     if (mode == LL) {
5748       __ sve_cntb(vec_len);
5749     } else {
5750       __ sve_cnth(vec_len);
5751     }
5752 
5753     __ sub(rscratch1, cnt, vec_len);
5754 
5755     __ bind(LOOP);
5756 
5757       // main loop
5758       LOAD_PAIR(ztmp1, ztmp2, pgtmp1, src1, src2, idx);
5759       __ add(idx, idx, vec_len);
5760       // Compare strings.
5761       __ sve_cmp(Assembler::NE, pgtmp2, mode == LL ? __ B : __ H, pgtmp1, ztmp1, ztmp2);
5762       __ br(__ NE, MISMATCH);
5763       __ cmp(idx, rscratch1);
5764       __ br(__ LT, LOOP);
5765 
5766     // post loop, last iteration
5767     __ sve_whilelt(pgtmp1, mode == LL ? __ B : __ H, idx, cnt);
5768 
5769     LOAD_PAIR(ztmp1, ztmp2, pgtmp1, src1, src2, idx);
5770     __ sve_cmp(Assembler::NE, pgtmp2, mode == LL ? __ B : __ H, pgtmp1, ztmp1, ztmp2);
5771     __ br(__ EQ, DONE);
5772 
5773     __ bind(MISMATCH);
5774 
5775     // Crop the vector to find its location.
5776     __ sve_brkb(pgtmp2, pgtmp1, pgtmp2, false /* isMerge */);
5777     // Extract the first different characters of each string.
5778     __ sve_lasta(rscratch1, mode == LL ? __ B : __ H, pgtmp2, ztmp1);
5779     __ sve_lasta(rscratch2, mode == LL ? __ B : __ H, pgtmp2, ztmp2);
5780 
5781     // Compute the difference of the first different characters.
5782     __ sub(result, rscratch1, rscratch2);
5783 
5784     __ bind(DONE);
5785     __ ret(lr);
5786 #undef LOAD_PAIR
5787     return entry;
5788   }
5789 
5790   void generate_compare_long_strings() {
5791     if (UseSVE == 0) {
5792       StubRoutines::aarch64::_compare_long_string_LL
5793           = generate_compare_long_string_same_encoding(true);
5794       StubRoutines::aarch64::_compare_long_string_UU
5795           = generate_compare_long_string_same_encoding(false);
5796       StubRoutines::aarch64::_compare_long_string_LU
5797           = generate_compare_long_string_different_encoding(true);
5798       StubRoutines::aarch64::_compare_long_string_UL
5799           = generate_compare_long_string_different_encoding(false);
5800     } else {
5801       StubRoutines::aarch64::_compare_long_string_LL
5802           = generate_compare_long_string_sve(LL);
5803       StubRoutines::aarch64::_compare_long_string_UU
5804           = generate_compare_long_string_sve(UU);
5805       StubRoutines::aarch64::_compare_long_string_LU
5806           = generate_compare_long_string_sve(LU);
5807       StubRoutines::aarch64::_compare_long_string_UL
5808           = generate_compare_long_string_sve(UL);
5809     }
5810   }
5811 
5812   // R0 = result
5813   // R1 = str2
5814   // R2 = cnt1
5815   // R3 = str1
5816   // R4 = cnt2
5817   // Clobbers: rscratch1, rscratch2, v0, v1, rflags
5818   //
5819   // This generic linear code use few additional ideas, which makes it faster:
5820   // 1) we can safely keep at least 1st register of pattern(since length >= 8)
5821   // in order to skip initial loading(help in systems with 1 ld pipeline)
5822   // 2) we can use "fast" algorithm of finding single character to search for
5823   // first symbol with less branches(1 branch per each loaded register instead
5824   // of branch for each symbol), so, this is where constants like
5825   // 0x0101...01, 0x00010001...0001, 0x7f7f...7f, 0x7fff7fff...7fff comes from
5826   // 3) after loading and analyzing 1st register of source string, it can be
5827   // used to search for every 1st character entry, saving few loads in
5828   // comparison with "simplier-but-slower" implementation
5829   // 4) in order to avoid lots of push/pop operations, code below is heavily
5830   // re-using/re-initializing/compressing register values, which makes code
5831   // larger and a bit less readable, however, most of extra operations are
5832   // issued during loads or branches, so, penalty is minimal
5833   address generate_string_indexof_linear(bool str1_isL, bool str2_isL) {
5834     const char* stubName = str1_isL
5835         ? (str2_isL ? "indexof_linear_ll" : "indexof_linear_ul")
5836         : "indexof_linear_uu";
5837     __ align(CodeEntryAlignment);
5838     StubCodeMark mark(this, "StubRoutines", stubName);
5839     address entry = __ pc();
5840 
5841     int str1_chr_size = str1_isL ? 1 : 2;
5842     int str2_chr_size = str2_isL ? 1 : 2;
5843     int str1_chr_shift = str1_isL ? 0 : 1;
5844     int str2_chr_shift = str2_isL ? 0 : 1;
5845     bool isL = str1_isL && str2_isL;
5846    // parameters
5847     Register result = r0, str2 = r1, cnt1 = r2, str1 = r3, cnt2 = r4;
5848     // temporary registers
5849     Register tmp1 = r20, tmp2 = r21, tmp3 = r22, tmp4 = r23;
5850     RegSet spilled_regs = RegSet::range(tmp1, tmp4);
5851     // redefinitions
5852     Register ch1 = rscratch1, ch2 = rscratch2, first = tmp3;
5853 
5854     __ push(spilled_regs, sp);
5855     Label L_LOOP, L_LOOP_PROCEED, L_SMALL, L_HAS_ZERO,
5856         L_HAS_ZERO_LOOP, L_CMP_LOOP, L_CMP_LOOP_NOMATCH, L_SMALL_PROCEED,
5857         L_SMALL_HAS_ZERO_LOOP, L_SMALL_CMP_LOOP_NOMATCH, L_SMALL_CMP_LOOP,
5858         L_POST_LOOP, L_CMP_LOOP_LAST_CMP, L_HAS_ZERO_LOOP_NOMATCH,
5859         L_SMALL_CMP_LOOP_LAST_CMP, L_SMALL_CMP_LOOP_LAST_CMP2,
5860         L_CMP_LOOP_LAST_CMP2, DONE, NOMATCH;
5861     // Read whole register from str1. It is safe, because length >=8 here
5862     __ ldr(ch1, Address(str1));
5863     // Read whole register from str2. It is safe, because length >=8 here
5864     __ ldr(ch2, Address(str2));
5865     __ sub(cnt2, cnt2, cnt1);
5866     __ andr(first, ch1, str1_isL ? 0xFF : 0xFFFF);
5867     if (str1_isL != str2_isL) {
5868       __ eor(v0, __ T16B, v0, v0);
5869     }
5870     __ mov(tmp1, str2_isL ? 0x0101010101010101 : 0x0001000100010001);
5871     __ mul(first, first, tmp1);
5872     // check if we have less than 1 register to check
5873     __ subs(cnt2, cnt2, wordSize/str2_chr_size - 1);
5874     if (str1_isL != str2_isL) {
5875       __ fmovd(v1, ch1);
5876     }
5877     __ br(__ LE, L_SMALL);
5878     __ eor(ch2, first, ch2);
5879     if (str1_isL != str2_isL) {
5880       __ zip1(v1, __ T16B, v1, v0);
5881     }
5882     __ sub(tmp2, ch2, tmp1);
5883     __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
5884     __ bics(tmp2, tmp2, ch2);
5885     if (str1_isL != str2_isL) {
5886       __ fmovd(ch1, v1);
5887     }
5888     __ br(__ NE, L_HAS_ZERO);
5889     __ subs(cnt2, cnt2, wordSize/str2_chr_size);
5890     __ add(result, result, wordSize/str2_chr_size);
5891     __ add(str2, str2, wordSize);
5892     __ br(__ LT, L_POST_LOOP);
5893     __ BIND(L_LOOP);
5894       __ ldr(ch2, Address(str2));
5895       __ eor(ch2, first, ch2);
5896       __ sub(tmp2, ch2, tmp1);
5897       __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
5898       __ bics(tmp2, tmp2, ch2);
5899       __ br(__ NE, L_HAS_ZERO);
5900     __ BIND(L_LOOP_PROCEED);
5901       __ subs(cnt2, cnt2, wordSize/str2_chr_size);
5902       __ add(str2, str2, wordSize);
5903       __ add(result, result, wordSize/str2_chr_size);
5904       __ br(__ GE, L_LOOP);
5905     __ BIND(L_POST_LOOP);
5906       __ subs(zr, cnt2, -wordSize/str2_chr_size); // no extra characters to check
5907       __ br(__ LE, NOMATCH);
5908       __ ldr(ch2, Address(str2));
5909       __ sub(cnt2, zr, cnt2, __ LSL, LogBitsPerByte + str2_chr_shift);
5910       __ eor(ch2, first, ch2);
5911       __ sub(tmp2, ch2, tmp1);
5912       __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
5913       __ mov(tmp4, -1); // all bits set
5914       __ b(L_SMALL_PROCEED);
5915     __ align(OptoLoopAlignment);
5916     __ BIND(L_SMALL);
5917       __ sub(cnt2, zr, cnt2, __ LSL, LogBitsPerByte + str2_chr_shift);
5918       __ eor(ch2, first, ch2);
5919       if (str1_isL != str2_isL) {
5920         __ zip1(v1, __ T16B, v1, v0);
5921       }
5922       __ sub(tmp2, ch2, tmp1);
5923       __ mov(tmp4, -1); // all bits set
5924       __ orr(ch2, ch2, str2_isL ? 0x7f7f7f7f7f7f7f7f : 0x7fff7fff7fff7fff);
5925       if (str1_isL != str2_isL) {
5926         __ fmovd(ch1, v1); // move converted 4 symbols
5927       }
5928     __ BIND(L_SMALL_PROCEED);
5929       __ lsrv(tmp4, tmp4, cnt2); // mask. zeroes on useless bits.
5930       __ bic(tmp2, tmp2, ch2);
5931       __ ands(tmp2, tmp2, tmp4); // clear useless bits and check
5932       __ rbit(tmp2, tmp2);
5933       __ br(__ EQ, NOMATCH);
5934     __ BIND(L_SMALL_HAS_ZERO_LOOP);
5935       __ clz(tmp4, tmp2); // potentially long. Up to 4 cycles on some cpu's
5936       __ cmp(cnt1, u1(wordSize/str2_chr_size));
5937       __ br(__ LE, L_SMALL_CMP_LOOP_LAST_CMP2);
5938       if (str2_isL) { // LL
5939         __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte); // address of "index"
5940         __ ldr(ch2, Address(str2)); // read whole register of str2. Safe.
5941         __ lslv(tmp2, tmp2, tmp4); // shift off leading zeroes from match info
5942         __ add(result, result, tmp4, __ LSR, LogBitsPerByte);
5943         __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
5944       } else {
5945         __ mov(ch2, 0xE); // all bits in byte set except last one
5946         __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
5947         __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
5948         __ lslv(tmp2, tmp2, tmp4);
5949         __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
5950         __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
5951         __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
5952         __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
5953       }
5954       __ cmp(ch1, ch2);
5955       __ mov(tmp4, wordSize/str2_chr_size);
5956       __ br(__ NE, L_SMALL_CMP_LOOP_NOMATCH);
5957     __ BIND(L_SMALL_CMP_LOOP);
5958       str1_isL ? __ ldrb(first, Address(str1, tmp4, Address::lsl(str1_chr_shift)))
5959                : __ ldrh(first, Address(str1, tmp4, Address::lsl(str1_chr_shift)));
5960       str2_isL ? __ ldrb(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)))
5961                : __ ldrh(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)));
5962       __ add(tmp4, tmp4, 1);
5963       __ cmp(tmp4, cnt1);
5964       __ br(__ GE, L_SMALL_CMP_LOOP_LAST_CMP);
5965       __ cmp(first, ch2);
5966       __ br(__ EQ, L_SMALL_CMP_LOOP);
5967     __ BIND(L_SMALL_CMP_LOOP_NOMATCH);
5968       __ cbz(tmp2, NOMATCH); // no more matches. exit
5969       __ clz(tmp4, tmp2);
5970       __ add(result, result, 1); // advance index
5971       __ add(str2, str2, str2_chr_size); // advance pointer
5972       __ b(L_SMALL_HAS_ZERO_LOOP);
5973     __ align(OptoLoopAlignment);
5974     __ BIND(L_SMALL_CMP_LOOP_LAST_CMP);
5975       __ cmp(first, ch2);
5976       __ br(__ NE, L_SMALL_CMP_LOOP_NOMATCH);
5977       __ b(DONE);
5978     __ align(OptoLoopAlignment);
5979     __ BIND(L_SMALL_CMP_LOOP_LAST_CMP2);
5980       if (str2_isL) { // LL
5981         __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte); // address of "index"
5982         __ ldr(ch2, Address(str2)); // read whole register of str2. Safe.
5983         __ lslv(tmp2, tmp2, tmp4); // shift off leading zeroes from match info
5984         __ add(result, result, tmp4, __ LSR, LogBitsPerByte);
5985         __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
5986       } else {
5987         __ mov(ch2, 0xE); // all bits in byte set except last one
5988         __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
5989         __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
5990         __ lslv(tmp2, tmp2, tmp4);
5991         __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
5992         __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
5993         __ lsl(tmp2, tmp2, 1); // shift off leading "1" from match info
5994         __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
5995       }
5996       __ cmp(ch1, ch2);
5997       __ br(__ NE, L_SMALL_CMP_LOOP_NOMATCH);
5998       __ b(DONE);
5999     __ align(OptoLoopAlignment);
6000     __ BIND(L_HAS_ZERO);
6001       __ rbit(tmp2, tmp2);
6002       __ clz(tmp4, tmp2); // potentially long. Up to 4 cycles on some CPU's
6003       // Now, perform compression of counters(cnt2 and cnt1) into one register.
6004       // It's fine because both counters are 32bit and are not changed in this
6005       // loop. Just restore it on exit. So, cnt1 can be re-used in this loop.
6006       __ orr(cnt2, cnt2, cnt1, __ LSL, BitsPerByte * wordSize / 2);
6007       __ sub(result, result, 1);
6008     __ BIND(L_HAS_ZERO_LOOP);
6009       __ mov(cnt1, wordSize/str2_chr_size);
6010       __ cmp(cnt1, cnt2, __ LSR, BitsPerByte * wordSize / 2);
6011       __ br(__ GE, L_CMP_LOOP_LAST_CMP2); // case of 8 bytes only to compare
6012       if (str2_isL) {
6013         __ lsr(ch2, tmp4, LogBitsPerByte + str2_chr_shift); // char index
6014         __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
6015         __ lslv(tmp2, tmp2, tmp4);
6016         __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
6017         __ add(tmp4, tmp4, 1);
6018         __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
6019         __ lsl(tmp2, tmp2, 1);
6020         __ mov(tmp4, wordSize/str2_chr_size);
6021       } else {
6022         __ mov(ch2, 0xE);
6023         __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
6024         __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
6025         __ lslv(tmp2, tmp2, tmp4);
6026         __ add(tmp4, tmp4, 1);
6027         __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
6028         __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte);
6029         __ lsl(tmp2, tmp2, 1);
6030         __ mov(tmp4, wordSize/str2_chr_size);
6031         __ sub(str2, str2, str2_chr_size);
6032       }
6033       __ cmp(ch1, ch2);
6034       __ mov(tmp4, wordSize/str2_chr_size);
6035       __ br(__ NE, L_CMP_LOOP_NOMATCH);
6036     __ BIND(L_CMP_LOOP);
6037       str1_isL ? __ ldrb(cnt1, Address(str1, tmp4, Address::lsl(str1_chr_shift)))
6038                : __ ldrh(cnt1, Address(str1, tmp4, Address::lsl(str1_chr_shift)));
6039       str2_isL ? __ ldrb(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)))
6040                : __ ldrh(ch2, Address(str2, tmp4, Address::lsl(str2_chr_shift)));
6041       __ add(tmp4, tmp4, 1);
6042       __ cmp(tmp4, cnt2, __ LSR, BitsPerByte * wordSize / 2);
6043       __ br(__ GE, L_CMP_LOOP_LAST_CMP);
6044       __ cmp(cnt1, ch2);
6045       __ br(__ EQ, L_CMP_LOOP);
6046     __ BIND(L_CMP_LOOP_NOMATCH);
6047       // here we're not matched
6048       __ cbz(tmp2, L_HAS_ZERO_LOOP_NOMATCH); // no more matches. Proceed to main loop
6049       __ clz(tmp4, tmp2);
6050       __ add(str2, str2, str2_chr_size); // advance pointer
6051       __ b(L_HAS_ZERO_LOOP);
6052     __ align(OptoLoopAlignment);
6053     __ BIND(L_CMP_LOOP_LAST_CMP);
6054       __ cmp(cnt1, ch2);
6055       __ br(__ NE, L_CMP_LOOP_NOMATCH);
6056       __ b(DONE);
6057     __ align(OptoLoopAlignment);
6058     __ BIND(L_CMP_LOOP_LAST_CMP2);
6059       if (str2_isL) {
6060         __ lsr(ch2, tmp4, LogBitsPerByte + str2_chr_shift); // char index
6061         __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
6062         __ lslv(tmp2, tmp2, tmp4);
6063         __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
6064         __ add(tmp4, tmp4, 1);
6065         __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
6066         __ lsl(tmp2, tmp2, 1);
6067       } else {
6068         __ mov(ch2, 0xE);
6069         __ andr(ch2, ch2, tmp4, __ LSR, LogBitsPerByte); // byte shift amount
6070         __ ldr(ch2, Address(str2, ch2)); // read whole register of str2. Safe.
6071         __ lslv(tmp2, tmp2, tmp4);
6072         __ add(tmp4, tmp4, 1);
6073         __ add(result, result, tmp4, __ LSR, LogBitsPerByte + str2_chr_shift);
6074         __ add(str2, str2, tmp4, __ LSR, LogBitsPerByte);
6075         __ lsl(tmp2, tmp2, 1);
6076         __ sub(str2, str2, str2_chr_size);
6077       }
6078       __ cmp(ch1, ch2);
6079       __ br(__ NE, L_CMP_LOOP_NOMATCH);
6080       __ b(DONE);
6081     __ align(OptoLoopAlignment);
6082     __ BIND(L_HAS_ZERO_LOOP_NOMATCH);
6083       // 1) Restore "result" index. Index was wordSize/str2_chr_size * N until
6084       // L_HAS_ZERO block. Byte octet was analyzed in L_HAS_ZERO_LOOP,
6085       // so, result was increased at max by wordSize/str2_chr_size - 1, so,
6086       // respective high bit wasn't changed. L_LOOP_PROCEED will increase
6087       // result by analyzed characters value, so, we can just reset lower bits
6088       // in result here. Clear 2 lower bits for UU/UL and 3 bits for LL
6089       // 2) restore cnt1 and cnt2 values from "compressed" cnt2
6090       // 3) advance str2 value to represent next str2 octet. result & 7/3 is
6091       // index of last analyzed substring inside current octet. So, str2 in at
6092       // respective start address. We need to advance it to next octet
6093       __ andr(tmp2, result, wordSize/str2_chr_size - 1); // symbols analyzed
6094       __ lsr(cnt1, cnt2, BitsPerByte * wordSize / 2);
6095       __ bfm(result, zr, 0, 2 - str2_chr_shift);
6096       __ sub(str2, str2, tmp2, __ LSL, str2_chr_shift); // restore str2
6097       __ movw(cnt2, cnt2);
6098       __ b(L_LOOP_PROCEED);
6099     __ align(OptoLoopAlignment);
6100     __ BIND(NOMATCH);
6101       __ mov(result, -1);
6102     __ BIND(DONE);
6103       __ pop(spilled_regs, sp);
6104       __ ret(lr);
6105     return entry;
6106   }
6107 
6108   void generate_string_indexof_stubs() {
6109     StubRoutines::aarch64::_string_indexof_linear_ll = generate_string_indexof_linear(true, true);
6110     StubRoutines::aarch64::_string_indexof_linear_uu = generate_string_indexof_linear(false, false);
6111     StubRoutines::aarch64::_string_indexof_linear_ul = generate_string_indexof_linear(true, false);
6112   }
6113 
6114   void inflate_and_store_2_fp_registers(bool generatePrfm,
6115       FloatRegister src1, FloatRegister src2) {
6116     Register dst = r1;
6117     __ zip1(v1, __ T16B, src1, v0);
6118     __ zip2(v2, __ T16B, src1, v0);
6119     if (generatePrfm) {
6120       __ prfm(Address(dst, SoftwarePrefetchHintDistance), PSTL1STRM);
6121     }
6122     __ zip1(v3, __ T16B, src2, v0);
6123     __ zip2(v4, __ T16B, src2, v0);
6124     __ st1(v1, v2, v3, v4, __ T16B, Address(__ post(dst, 64)));
6125   }
6126 
6127   // R0 = src
6128   // R1 = dst
6129   // R2 = len
6130   // R3 = len >> 3
6131   // V0 = 0
6132   // v1 = loaded 8 bytes
6133   // Clobbers: r0, r1, r3, rscratch1, rflags, v0-v6
6134   address generate_large_byte_array_inflate() {
6135     __ align(CodeEntryAlignment);
6136     StubCodeMark mark(this, "StubRoutines", "large_byte_array_inflate");
6137     address entry = __ pc();
6138     Label LOOP, LOOP_START, LOOP_PRFM, LOOP_PRFM_START, DONE;
6139     Register src = r0, dst = r1, len = r2, octetCounter = r3;
6140     const int large_loop_threshold = MAX2(64, SoftwarePrefetchHintDistance)/8 + 4;
6141 
6142     // do one more 8-byte read to have address 16-byte aligned in most cases
6143     // also use single store instruction
6144     __ ldrd(v2, __ post(src, 8));
6145     __ sub(octetCounter, octetCounter, 2);
6146     __ zip1(v1, __ T16B, v1, v0);
6147     __ zip1(v2, __ T16B, v2, v0);
6148     __ st1(v1, v2, __ T16B, __ post(dst, 32));
6149     __ ld1(v3, v4, v5, v6, __ T16B, Address(__ post(src, 64)));
6150     __ subs(rscratch1, octetCounter, large_loop_threshold);
6151     __ br(__ LE, LOOP_START);
6152     __ b(LOOP_PRFM_START);
6153     __ bind(LOOP_PRFM);
6154       __ ld1(v3, v4, v5, v6, __ T16B, Address(__ post(src, 64)));
6155     __ bind(LOOP_PRFM_START);
6156       __ prfm(Address(src, SoftwarePrefetchHintDistance));
6157       __ sub(octetCounter, octetCounter, 8);
6158       __ subs(rscratch1, octetCounter, large_loop_threshold);
6159       inflate_and_store_2_fp_registers(true, v3, v4);
6160       inflate_and_store_2_fp_registers(true, v5, v6);
6161       __ br(__ GT, LOOP_PRFM);
6162       __ cmp(octetCounter, (u1)8);
6163       __ br(__ LT, DONE);
6164     __ bind(LOOP);
6165       __ ld1(v3, v4, v5, v6, __ T16B, Address(__ post(src, 64)));
6166       __ bind(LOOP_START);
6167       __ sub(octetCounter, octetCounter, 8);
6168       __ cmp(octetCounter, (u1)8);
6169       inflate_and_store_2_fp_registers(false, v3, v4);
6170       inflate_and_store_2_fp_registers(false, v5, v6);
6171       __ br(__ GE, LOOP);
6172     __ bind(DONE);
6173       __ ret(lr);
6174     return entry;
6175   }
6176 
6177   /**
6178    *  Arguments:
6179    *
6180    *  Input:
6181    *  c_rarg0   - current state address
6182    *  c_rarg1   - H key address
6183    *  c_rarg2   - data address
6184    *  c_rarg3   - number of blocks
6185    *
6186    *  Output:
6187    *  Updated state at c_rarg0
6188    */
6189   address generate_ghash_processBlocks() {
6190     // Bafflingly, GCM uses little-endian for the byte order, but
6191     // big-endian for the bit order.  For example, the polynomial 1 is
6192     // represented as the 16-byte string 80 00 00 00 | 12 bytes of 00.
6193     //
6194     // So, we must either reverse the bytes in each word and do
6195     // everything big-endian or reverse the bits in each byte and do
6196     // it little-endian.  On AArch64 it's more idiomatic to reverse
6197     // the bits in each byte (we have an instruction, RBIT, to do
6198     // that) and keep the data in little-endian bit order through the
6199     // calculation, bit-reversing the inputs and outputs.
6200 
6201     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
6202     __ align(wordSize * 2);
6203     address p = __ pc();
6204     __ emit_int64(0x87);  // The low-order bits of the field
6205                           // polynomial (i.e. p = z^7+z^2+z+1)
6206                           // repeated in the low and high parts of a
6207                           // 128-bit vector
6208     __ emit_int64(0x87);
6209 
6210     __ align(CodeEntryAlignment);
6211     address start = __ pc();
6212 
6213     Register state   = c_rarg0;
6214     Register subkeyH = c_rarg1;
6215     Register data    = c_rarg2;
6216     Register blocks  = c_rarg3;
6217 
6218     FloatRegister vzr = v30;
6219     __ eor(vzr, __ T16B, vzr, vzr); // zero register
6220 
6221     __ ldrq(v24, p);    // The field polynomial
6222 
6223     __ ldrq(v0, Address(state));
6224     __ ldrq(v1, Address(subkeyH));
6225 
6226     __ rev64(v0, __ T16B, v0);          // Bit-reverse words in state and subkeyH
6227     __ rbit(v0, __ T16B, v0);
6228     __ rev64(v1, __ T16B, v1);
6229     __ rbit(v1, __ T16B, v1);
6230 
6231     __ ext(v4, __ T16B, v1, v1, 0x08); // long-swap subkeyH into v1
6232     __ eor(v4, __ T16B, v4, v1);       // xor subkeyH into subkeyL (Karatsuba: (A1+A0))
6233 
6234     {
6235       Label L_ghash_loop;
6236       __ bind(L_ghash_loop);
6237 
6238       __ ldrq(v2, Address(__ post(data, 0x10))); // Load the data, bit
6239                                                  // reversing each byte
6240       __ rbit(v2, __ T16B, v2);
6241       __ eor(v2, __ T16B, v0, v2);   // bit-swapped data ^ bit-swapped state
6242 
6243       // Multiply state in v2 by subkey in v1
6244       __ ghash_multiply(/*result_lo*/v5, /*result_hi*/v7,
6245                         /*a*/v1, /*b*/v2, /*a1_xor_a0*/v4,
6246                         /*temps*/v6, v3, /*reuse/clobber b*/v2);
6247       // Reduce v7:v5 by the field polynomial
6248       __ ghash_reduce(/*result*/v0, /*lo*/v5, /*hi*/v7, /*p*/v24, vzr, /*temp*/v3);
6249 
6250       __ sub(blocks, blocks, 1);
6251       __ cbnz(blocks, L_ghash_loop);
6252     }
6253 
6254     // The bit-reversed result is at this point in v0
6255     __ rev64(v0, __ T16B, v0);
6256     __ rbit(v0, __ T16B, v0);
6257 
6258     __ st1(v0, __ T16B, state);
6259     __ ret(lr);
6260 
6261     return start;
6262   }
6263 
6264   address generate_ghash_processBlocks_wide() {
6265     address small = generate_ghash_processBlocks();
6266 
6267     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks_wide");
6268     __ align(wordSize * 2);
6269     address p = __ pc();
6270     __ emit_int64(0x87);  // The low-order bits of the field
6271                           // polynomial (i.e. p = z^7+z^2+z+1)
6272                           // repeated in the low and high parts of a
6273                           // 128-bit vector
6274     __ emit_int64(0x87);
6275 
6276     __ align(CodeEntryAlignment);
6277     address start = __ pc();
6278 
6279     Register state   = c_rarg0;
6280     Register subkeyH = c_rarg1;
6281     Register data    = c_rarg2;
6282     Register blocks  = c_rarg3;
6283 
6284     const int unroll = 4;
6285 
6286     __ cmp(blocks, (unsigned char)(unroll * 2));
6287     __ br(__ LT, small);
6288 
6289     if (unroll > 1) {
6290     // Save state before entering routine
6291       __ sub(sp, sp, 4 * 16);
6292       __ st1(v12, v13, v14, v15, __ T16B, Address(sp));
6293       __ sub(sp, sp, 4 * 16);
6294       __ st1(v8, v9, v10, v11, __ T16B, Address(sp));
6295     }
6296 
6297     __ ghash_processBlocks_wide(p, state, subkeyH, data, blocks, unroll);
6298 
6299     if (unroll > 1) {
6300       // And restore state
6301       __ ld1(v8, v9, v10, v11, __ T16B, __ post(sp, 4 * 16));
6302       __ ld1(v12, v13, v14, v15, __ T16B, __ post(sp, 4 * 16));
6303     }
6304 
6305     __ cmp(blocks, (unsigned char)0);
6306     __ br(__ GT, small);
6307 
6308     __ ret(lr);
6309 
6310     return start;
6311   }
6312 
6313   void generate_base64_encode_simdround(Register src, Register dst,
6314         FloatRegister codec, u8 size) {
6315 
6316     FloatRegister in0  = v4,  in1  = v5,  in2  = v6;
6317     FloatRegister out0 = v16, out1 = v17, out2 = v18, out3 = v19;
6318     FloatRegister ind0 = v20, ind1 = v21, ind2 = v22, ind3 = v23;
6319 
6320     Assembler::SIMD_Arrangement arrangement = size == 16 ? __ T16B : __ T8B;
6321 
6322     __ ld3(in0, in1, in2, arrangement, __ post(src, 3 * size));
6323 
6324     __ ushr(ind0, arrangement, in0,  2);
6325 
6326     __ ushr(ind1, arrangement, in1,  2);
6327     __ shl(in0,   arrangement, in0,  6);
6328     __ orr(ind1,  arrangement, ind1, in0);
6329     __ ushr(ind1, arrangement, ind1, 2);
6330 
6331     __ ushr(ind2, arrangement, in2,  4);
6332     __ shl(in1,   arrangement, in1,  4);
6333     __ orr(ind2,  arrangement, in1,  ind2);
6334     __ ushr(ind2, arrangement, ind2, 2);
6335 
6336     __ shl(ind3,  arrangement, in2,  2);
6337     __ ushr(ind3, arrangement, ind3, 2);
6338 
6339     __ tbl(out0,  arrangement, codec,  4, ind0);
6340     __ tbl(out1,  arrangement, codec,  4, ind1);
6341     __ tbl(out2,  arrangement, codec,  4, ind2);
6342     __ tbl(out3,  arrangement, codec,  4, ind3);
6343 
6344     __ st4(out0,  out1, out2, out3, arrangement, __ post(dst, 4 * size));
6345   }
6346 
6347    /**
6348    *  Arguments:
6349    *
6350    *  Input:
6351    *  c_rarg0   - src_start
6352    *  c_rarg1   - src_offset
6353    *  c_rarg2   - src_length
6354    *  c_rarg3   - dest_start
6355    *  c_rarg4   - dest_offset
6356    *  c_rarg5   - isURL
6357    *
6358    */
6359   address generate_base64_encodeBlock() {
6360 
6361     static const char toBase64[64] = {
6362       'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
6363       'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
6364       'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
6365       'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
6366       '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
6367     };
6368 
6369     static const char toBase64URL[64] = {
6370       'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
6371       'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
6372       'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
6373       'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
6374       '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_'
6375     };
6376 
6377     __ align(CodeEntryAlignment);
6378     StubCodeMark mark(this, "StubRoutines", "encodeBlock");
6379     address start = __ pc();
6380 
6381     Register src   = c_rarg0;  // source array
6382     Register soff  = c_rarg1;  // source start offset
6383     Register send  = c_rarg2;  // source end offset
6384     Register dst   = c_rarg3;  // dest array
6385     Register doff  = c_rarg4;  // position for writing to dest array
6386     Register isURL = c_rarg5;  // Base64 or URL character set
6387 
6388     // c_rarg6 and c_rarg7 are free to use as temps
6389     Register codec  = c_rarg6;
6390     Register length = c_rarg7;
6391 
6392     Label ProcessData, Process48B, Process24B, Process3B, SIMDExit, Exit;
6393 
6394     __ add(src, src, soff);
6395     __ add(dst, dst, doff);
6396     __ sub(length, send, soff);
6397 
6398     // load the codec base address
6399     __ lea(codec, ExternalAddress((address) toBase64));
6400     __ cbz(isURL, ProcessData);
6401     __ lea(codec, ExternalAddress((address) toBase64URL));
6402 
6403     __ BIND(ProcessData);
6404 
6405     // too short to formup a SIMD loop, roll back
6406     __ cmp(length, (u1)24);
6407     __ br(Assembler::LT, Process3B);
6408 
6409     __ ld1(v0, v1, v2, v3, __ T16B, Address(codec));
6410 
6411     __ BIND(Process48B);
6412     __ cmp(length, (u1)48);
6413     __ br(Assembler::LT, Process24B);
6414     generate_base64_encode_simdround(src, dst, v0, 16);
6415     __ sub(length, length, 48);
6416     __ b(Process48B);
6417 
6418     __ BIND(Process24B);
6419     __ cmp(length, (u1)24);
6420     __ br(Assembler::LT, SIMDExit);
6421     generate_base64_encode_simdround(src, dst, v0, 8);
6422     __ sub(length, length, 24);
6423 
6424     __ BIND(SIMDExit);
6425     __ cbz(length, Exit);
6426 
6427     __ BIND(Process3B);
6428     //  3 src bytes, 24 bits
6429     __ ldrb(r10, __ post(src, 1));
6430     __ ldrb(r11, __ post(src, 1));
6431     __ ldrb(r12, __ post(src, 1));
6432     __ orrw(r11, r11, r10, Assembler::LSL, 8);
6433     __ orrw(r12, r12, r11, Assembler::LSL, 8);
6434     // codec index
6435     __ ubfmw(r15, r12, 18, 23);
6436     __ ubfmw(r14, r12, 12, 17);
6437     __ ubfmw(r13, r12, 6,  11);
6438     __ andw(r12,  r12, 63);
6439     // get the code based on the codec
6440     __ ldrb(r15, Address(codec, r15, Address::uxtw(0)));
6441     __ ldrb(r14, Address(codec, r14, Address::uxtw(0)));
6442     __ ldrb(r13, Address(codec, r13, Address::uxtw(0)));
6443     __ ldrb(r12, Address(codec, r12, Address::uxtw(0)));
6444     __ strb(r15, __ post(dst, 1));
6445     __ strb(r14, __ post(dst, 1));
6446     __ strb(r13, __ post(dst, 1));
6447     __ strb(r12, __ post(dst, 1));
6448     __ sub(length, length, 3);
6449     __ cbnz(length, Process3B);
6450 
6451     __ BIND(Exit);
6452     __ ret(lr);
6453 
6454     return start;
6455   }
6456 
6457   void generate_base64_decode_simdround(Register src, Register dst,
6458         FloatRegister codecL, FloatRegister codecH, int size, Label& Exit) {
6459 
6460     FloatRegister in0  = v16, in1  = v17,  in2 = v18,  in3 = v19;
6461     FloatRegister out0 = v20, out1 = v21, out2 = v22;
6462 
6463     FloatRegister decL0 = v23, decL1 = v24, decL2 = v25, decL3 = v26;
6464     FloatRegister decH0 = v28, decH1 = v29, decH2 = v30, decH3 = v31;
6465 
6466     Label NoIllegalData, ErrorInLowerHalf, StoreLegalData;
6467 
6468     Assembler::SIMD_Arrangement arrangement = size == 16 ? __ T16B : __ T8B;
6469 
6470     __ ld4(in0, in1, in2, in3, arrangement, __ post(src, 4 * size));
6471 
6472     // we need unsigned saturating subtract, to make sure all input values
6473     // in range [0, 63] will have 0U value in the higher half lookup
6474     __ uqsubv(decH0, __ T16B, in0, v27);
6475     __ uqsubv(decH1, __ T16B, in1, v27);
6476     __ uqsubv(decH2, __ T16B, in2, v27);
6477     __ uqsubv(decH3, __ T16B, in3, v27);
6478 
6479     // lower half lookup
6480     __ tbl(decL0, arrangement, codecL, 4, in0);
6481     __ tbl(decL1, arrangement, codecL, 4, in1);
6482     __ tbl(decL2, arrangement, codecL, 4, in2);
6483     __ tbl(decL3, arrangement, codecL, 4, in3);
6484 
6485     // higher half lookup
6486     __ tbx(decH0, arrangement, codecH, 4, decH0);
6487     __ tbx(decH1, arrangement, codecH, 4, decH1);
6488     __ tbx(decH2, arrangement, codecH, 4, decH2);
6489     __ tbx(decH3, arrangement, codecH, 4, decH3);
6490 
6491     // combine lower and higher
6492     __ orr(decL0, arrangement, decL0, decH0);
6493     __ orr(decL1, arrangement, decL1, decH1);
6494     __ orr(decL2, arrangement, decL2, decH2);
6495     __ orr(decL3, arrangement, decL3, decH3);
6496 
6497     // check illegal inputs, value larger than 63 (maximum of 6 bits)
6498     __ cm(Assembler::HI, decH0, arrangement, decL0, v27);
6499     __ cm(Assembler::HI, decH1, arrangement, decL1, v27);
6500     __ cm(Assembler::HI, decH2, arrangement, decL2, v27);
6501     __ cm(Assembler::HI, decH3, arrangement, decL3, v27);
6502     __ orr(in0, arrangement, decH0, decH1);
6503     __ orr(in1, arrangement, decH2, decH3);
6504     __ orr(in2, arrangement, in0,   in1);
6505     __ umaxv(in3, arrangement, in2);
6506     __ umov(rscratch2, in3, __ B, 0);
6507 
6508     // get the data to output
6509     __ shl(out0,  arrangement, decL0, 2);
6510     __ ushr(out1, arrangement, decL1, 4);
6511     __ orr(out0,  arrangement, out0,  out1);
6512     __ shl(out1,  arrangement, decL1, 4);
6513     __ ushr(out2, arrangement, decL2, 2);
6514     __ orr(out1,  arrangement, out1,  out2);
6515     __ shl(out2,  arrangement, decL2, 6);
6516     __ orr(out2,  arrangement, out2,  decL3);
6517 
6518     __ cbz(rscratch2, NoIllegalData);
6519 
6520     // handle illegal input
6521     __ umov(r10, in2, __ D, 0);
6522     if (size == 16) {
6523       __ cbnz(r10, ErrorInLowerHalf);
6524 
6525       // illegal input is in higher half, store the lower half now.
6526       __ st3(out0, out1, out2, __ T8B, __ post(dst, 24));
6527 
6528       __ umov(r10, in2,  __ D, 1);
6529       __ umov(r11, out0, __ D, 1);
6530       __ umov(r12, out1, __ D, 1);
6531       __ umov(r13, out2, __ D, 1);
6532       __ b(StoreLegalData);
6533 
6534       __ BIND(ErrorInLowerHalf);
6535     }
6536     __ umov(r11, out0, __ D, 0);
6537     __ umov(r12, out1, __ D, 0);
6538     __ umov(r13, out2, __ D, 0);
6539 
6540     __ BIND(StoreLegalData);
6541     __ tbnz(r10, 5, Exit); // 0xff indicates illegal input
6542     __ strb(r11, __ post(dst, 1));
6543     __ strb(r12, __ post(dst, 1));
6544     __ strb(r13, __ post(dst, 1));
6545     __ lsr(r10, r10, 8);
6546     __ lsr(r11, r11, 8);
6547     __ lsr(r12, r12, 8);
6548     __ lsr(r13, r13, 8);
6549     __ b(StoreLegalData);
6550 
6551     __ BIND(NoIllegalData);
6552     __ st3(out0, out1, out2, arrangement, __ post(dst, 3 * size));
6553   }
6554 
6555 
6556    /**
6557    *  Arguments:
6558    *
6559    *  Input:
6560    *  c_rarg0   - src_start
6561    *  c_rarg1   - src_offset
6562    *  c_rarg2   - src_length
6563    *  c_rarg3   - dest_start
6564    *  c_rarg4   - dest_offset
6565    *  c_rarg5   - isURL
6566    *  c_rarg6   - isMIME
6567    *
6568    */
6569   address generate_base64_decodeBlock() {
6570 
6571     // The SIMD part of this Base64 decode intrinsic is based on the algorithm outlined
6572     // on http://0x80.pl/articles/base64-simd-neon.html#encoding-quadwords, in section
6573     // titled "Base64 decoding".
6574 
6575     // Non-SIMD lookup tables are mostly dumped from fromBase64 array used in java.util.Base64,
6576     // except the trailing character '=' is also treated illegal value in this intrinsic. That
6577     // is java.util.Base64.fromBase64['='] = -2, while fromBase(URL)64ForNoSIMD['='] = 255 here.
6578     static const uint8_t fromBase64ForNoSIMD[256] = {
6579       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6580       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6581       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,  62u, 255u, 255u, 255u,  63u,
6582        52u,  53u,  54u,  55u,  56u,  57u,  58u,  59u,  60u,  61u, 255u, 255u, 255u, 255u, 255u, 255u,
6583       255u,   0u,   1u,   2u,   3u,   4u,   5u,   6u,   7u,   8u,   9u,  10u,  11u,  12u,  13u,  14u,
6584        15u,  16u,  17u,  18u,  19u,  20u,  21u,  22u,  23u,  24u,  25u, 255u, 255u, 255u, 255u, 255u,
6585       255u,  26u,  27u,  28u,  29u,  30u,  31u,  32u,  33u,  34u,  35u,  36u,  37u,  38u,  39u,  40u,
6586        41u,  42u,  43u,  44u,  45u,  46u,  47u,  48u,  49u,  50u,  51u, 255u, 255u, 255u, 255u, 255u,
6587       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6588       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6589       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6590       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6591       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6592       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6593       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6594       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6595     };
6596 
6597     static const uint8_t fromBase64URLForNoSIMD[256] = {
6598       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6599       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6600       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,  62u, 255u, 255u,
6601        52u,  53u,  54u,  55u,  56u,  57u,  58u,  59u,  60u,  61u, 255u, 255u, 255u, 255u, 255u, 255u,
6602       255u,   0u,   1u,   2u,   3u,   4u,   5u,   6u,   7u,   8u,   9u,  10u,  11u,  12u,  13u,  14u,
6603        15u,  16u,  17u,  18u,  19u,  20u,  21u,  22u,  23u,  24u,  25u, 255u, 255u, 255u, 255u,  63u,
6604       255u,  26u,  27u,  28u,  29u,  30u,  31u,  32u,  33u,  34u,  35u,  36u,  37u,  38u,  39u,  40u,
6605        41u,  42u,  43u,  44u,  45u,  46u,  47u,  48u,  49u,  50u,  51u, 255u, 255u, 255u, 255u, 255u,
6606       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6607       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6608       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6609       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6610       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6611       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6612       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6613       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6614     };
6615 
6616     // A legal value of base64 code is in range [0, 127].  We need two lookups
6617     // with tbl/tbx and combine them to get the decode data. The 1st table vector
6618     // lookup use tbl, out of range indices are set to 0 in destination. The 2nd
6619     // table vector lookup use tbx, out of range indices are unchanged in
6620     // destination. Input [64..126] is mapped to index [65, 127] in second lookup.
6621     // The value of index 64 is set to 0, so that we know that we already get the
6622     // decoded data with the 1st lookup.
6623     static const uint8_t fromBase64ForSIMD[128] = {
6624       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6625       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6626       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,  62u, 255u, 255u, 255u,  63u,
6627        52u,  53u,  54u,  55u,  56u,  57u,  58u,  59u,  60u,  61u, 255u, 255u, 255u, 255u, 255u, 255u,
6628         0u, 255u,   0u,   1u,   2u,   3u,   4u,   5u,   6u,   7u,   8u,   9u,  10u,  11u,  12u,  13u,
6629        14u,  15u,  16u,  17u,  18u,  19u,  20u,  21u,  22u,  23u,  24u,  25u, 255u, 255u, 255u, 255u,
6630       255u, 255u,  26u,  27u,  28u,  29u,  30u,  31u,  32u,  33u,  34u,  35u,  36u,  37u,  38u,  39u,
6631        40u,  41u,  42u,  43u,  44u,  45u,  46u,  47u,  48u,  49u,  50u,  51u, 255u, 255u, 255u, 255u,
6632     };
6633 
6634     static const uint8_t fromBase64URLForSIMD[128] = {
6635       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6636       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,
6637       255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u, 255u,  62u, 255u, 255u,
6638        52u,  53u,  54u,  55u,  56u,  57u,  58u,  59u,  60u,  61u, 255u, 255u, 255u, 255u, 255u, 255u,
6639         0u, 255u,   0u,   1u,   2u,   3u,   4u,   5u,   6u,   7u,   8u,   9u,  10u,  11u,  12u,  13u,
6640        14u,  15u,  16u,  17u,  18u,  19u,  20u,  21u,  22u,  23u,  24u,  25u, 255u, 255u, 255u, 255u,
6641        63u, 255u,  26u,  27u,  28u,  29u,  30u,  31u,  32u,  33u,  34u,  35u,  36u,  37u,  38u,  39u,
6642        40u,  41u,  42u,  43u,  44u,  45u,  46u,  47u,  48u,  49u,  50u,  51u, 255u, 255u, 255u, 255u,
6643     };
6644 
6645     __ align(CodeEntryAlignment);
6646     StubCodeMark mark(this, "StubRoutines", "decodeBlock");
6647     address start = __ pc();
6648 
6649     Register src    = c_rarg0;  // source array
6650     Register soff   = c_rarg1;  // source start offset
6651     Register send   = c_rarg2;  // source end offset
6652     Register dst    = c_rarg3;  // dest array
6653     Register doff   = c_rarg4;  // position for writing to dest array
6654     Register isURL  = c_rarg5;  // Base64 or URL character set
6655     Register isMIME = c_rarg6;  // Decoding MIME block - unused in this implementation
6656 
6657     Register length = send;    // reuse send as length of source data to process
6658 
6659     Register simd_codec   = c_rarg6;
6660     Register nosimd_codec = c_rarg7;
6661 
6662     Label ProcessData, Process64B, Process32B, Process4B, SIMDEnter, SIMDExit, Exit;
6663 
6664     __ enter();
6665 
6666     __ add(src, src, soff);
6667     __ add(dst, dst, doff);
6668 
6669     __ mov(doff, dst);
6670 
6671     __ sub(length, send, soff);
6672     __ bfm(length, zr, 0, 1);
6673 
6674     __ lea(nosimd_codec, ExternalAddress((address) fromBase64ForNoSIMD));
6675     __ cbz(isURL, ProcessData);
6676     __ lea(nosimd_codec, ExternalAddress((address) fromBase64URLForNoSIMD));
6677 
6678     __ BIND(ProcessData);
6679     __ mov(rscratch1, length);
6680     __ cmp(length, (u1)144); // 144 = 80 + 64
6681     __ br(Assembler::LT, Process4B);
6682 
6683     // In the MIME case, the line length cannot be more than 76
6684     // bytes (see RFC 2045). This is too short a block for SIMD
6685     // to be worthwhile, so we use non-SIMD here.
6686     __ movw(rscratch1, 79);
6687 
6688     __ BIND(Process4B);
6689     __ ldrw(r14, __ post(src, 4));
6690     __ ubfxw(r10, r14, 0,  8);
6691     __ ubfxw(r11, r14, 8,  8);
6692     __ ubfxw(r12, r14, 16, 8);
6693     __ ubfxw(r13, r14, 24, 8);
6694     // get the de-code
6695     __ ldrb(r10, Address(nosimd_codec, r10, Address::uxtw(0)));
6696     __ ldrb(r11, Address(nosimd_codec, r11, Address::uxtw(0)));
6697     __ ldrb(r12, Address(nosimd_codec, r12, Address::uxtw(0)));
6698     __ ldrb(r13, Address(nosimd_codec, r13, Address::uxtw(0)));
6699     // error detection, 255u indicates an illegal input
6700     __ orrw(r14, r10, r11);
6701     __ orrw(r15, r12, r13);
6702     __ orrw(r14, r14, r15);
6703     __ tbnz(r14, 7, Exit);
6704     // recover the data
6705     __ lslw(r14, r10, 10);
6706     __ bfiw(r14, r11, 4, 6);
6707     __ bfmw(r14, r12, 2, 5);
6708     __ rev16w(r14, r14);
6709     __ bfiw(r13, r12, 6, 2);
6710     __ strh(r14, __ post(dst, 2));
6711     __ strb(r13, __ post(dst, 1));
6712     // non-simd loop
6713     __ subsw(rscratch1, rscratch1, 4);
6714     __ br(Assembler::GT, Process4B);
6715 
6716     // if exiting from PreProcess80B, rscratch1 == -1;
6717     // otherwise, rscratch1 == 0.
6718     __ cbzw(rscratch1, Exit);
6719     __ sub(length, length, 80);
6720 
6721     __ lea(simd_codec, ExternalAddress((address) fromBase64ForSIMD));
6722     __ cbz(isURL, SIMDEnter);
6723     __ lea(simd_codec, ExternalAddress((address) fromBase64URLForSIMD));
6724 
6725     __ BIND(SIMDEnter);
6726     __ ld1(v0, v1, v2, v3, __ T16B, __ post(simd_codec, 64));
6727     __ ld1(v4, v5, v6, v7, __ T16B, Address(simd_codec));
6728     __ mov(rscratch1, 63);
6729     __ dup(v27, __ T16B, rscratch1);
6730 
6731     __ BIND(Process64B);
6732     __ cmp(length, (u1)64);
6733     __ br(Assembler::LT, Process32B);
6734     generate_base64_decode_simdround(src, dst, v0, v4, 16, Exit);
6735     __ sub(length, length, 64);
6736     __ b(Process64B);
6737 
6738     __ BIND(Process32B);
6739     __ cmp(length, (u1)32);
6740     __ br(Assembler::LT, SIMDExit);
6741     generate_base64_decode_simdround(src, dst, v0, v4, 8, Exit);
6742     __ sub(length, length, 32);
6743     __ b(Process32B);
6744 
6745     __ BIND(SIMDExit);
6746     __ cbz(length, Exit);
6747     __ movw(rscratch1, length);
6748     __ b(Process4B);
6749 
6750     __ BIND(Exit);
6751     __ sub(c_rarg0, dst, doff);
6752 
6753     __ leave();
6754     __ ret(lr);
6755 
6756     return start;
6757   }
6758 
6759   // Support for spin waits.
6760   address generate_spin_wait() {
6761     __ align(CodeEntryAlignment);
6762     StubCodeMark mark(this, "StubRoutines", "spin_wait");
6763     address start = __ pc();
6764 
6765     __ spin_wait();
6766     __ ret(lr);
6767 
6768     return start;
6769   }
6770 
6771   address generate_lookup_secondary_supers_table_stub(u1 super_klass_index) {
6772     StubCodeMark mark(this, "StubRoutines", "lookup_secondary_supers_table");
6773 
6774     address start = __ pc();
6775     const Register
6776       r_super_klass  = r0,
6777       r_array_base   = r1,
6778       r_array_length = r2,
6779       r_array_index  = r3,
6780       r_sub_klass    = r4,
6781       r_bitmap       = rscratch2,
6782       result         = r5;
6783     const FloatRegister
6784       vtemp          = v0;
6785 
6786     Label L_success;
6787     __ enter();
6788     __ lookup_secondary_supers_table(r_sub_klass, r_super_klass,
6789                                      r_array_base, r_array_length, r_array_index,
6790                                      vtemp, result, super_klass_index,
6791                                      /*stub_is_near*/true);
6792     __ leave();
6793     __ ret(lr);
6794 
6795     return start;
6796   }
6797 
6798   // Slow path implementation for UseSecondarySupersTable.
6799   address generate_lookup_secondary_supers_table_slow_path_stub() {
6800     StubCodeMark mark(this, "StubRoutines", "lookup_secondary_supers_table_slow_path");
6801 
6802     address start = __ pc();
6803     const Register
6804       r_super_klass  = r0,        // argument
6805       r_array_base   = r1,        // argument
6806       temp1          = r2,        // temp
6807       r_array_index  = r3,        // argument
6808       r_bitmap       = rscratch2, // argument
6809       result         = r5;        // argument
6810 
6811     __ lookup_secondary_supers_table_slow_path(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, result);
6812     __ ret(lr);
6813 
6814     return start;
6815   }
6816 
6817 #if defined (LINUX) && !defined (__ARM_FEATURE_ATOMICS)
6818 
6819   // ARMv8.1 LSE versions of the atomic stubs used by Atomic::PlatformXX.
6820   //
6821   // If LSE is in use, generate LSE versions of all the stubs. The
6822   // non-LSE versions are in atomic_aarch64.S.
6823 
6824   // class AtomicStubMark records the entry point of a stub and the
6825   // stub pointer which will point to it. The stub pointer is set to
6826   // the entry point when ~AtomicStubMark() is called, which must be
6827   // after ICache::invalidate_range. This ensures safe publication of
6828   // the generated code.
6829   class AtomicStubMark {
6830     address _entry_point;
6831     aarch64_atomic_stub_t *_stub;
6832     MacroAssembler *_masm;
6833   public:
6834     AtomicStubMark(MacroAssembler *masm, aarch64_atomic_stub_t *stub) {
6835       _masm = masm;
6836       __ align(32);
6837       _entry_point = __ pc();
6838       _stub = stub;
6839     }
6840     ~AtomicStubMark() {
6841       *_stub = (aarch64_atomic_stub_t)_entry_point;
6842     }
6843   };
6844 
6845   // NB: For memory_order_conservative we need a trailing membar after
6846   // LSE atomic operations but not a leading membar.
6847   //
6848   // We don't need a leading membar because a clause in the Arm ARM
6849   // says:
6850   //
6851   //   Barrier-ordered-before
6852   //
6853   //   Barrier instructions order prior Memory effects before subsequent
6854   //   Memory effects generated by the same Observer. A read or a write
6855   //   RW1 is Barrier-ordered-before a read or a write RW 2 from the same
6856   //   Observer if and only if RW1 appears in program order before RW 2
6857   //   and [ ... ] at least one of RW 1 and RW 2 is generated by an atomic
6858   //   instruction with both Acquire and Release semantics.
6859   //
6860   // All the atomic instructions {ldaddal, swapal, casal} have Acquire
6861   // and Release semantics, therefore we don't need a leading
6862   // barrier. However, there is no corresponding Barrier-ordered-after
6863   // relationship, therefore we need a trailing membar to prevent a
6864   // later store or load from being reordered with the store in an
6865   // atomic instruction.
6866   //
6867   // This was checked by using the herd7 consistency model simulator
6868   // (http://diy.inria.fr/) with this test case:
6869   //
6870   // AArch64 LseCas
6871   // { 0:X1=x; 0:X2=y; 1:X1=x; 1:X2=y; }
6872   // P0 | P1;
6873   // LDR W4, [X2] | MOV W3, #0;
6874   // DMB LD       | MOV W4, #1;
6875   // LDR W3, [X1] | CASAL W3, W4, [X1];
6876   //              | DMB ISH;
6877   //              | STR W4, [X2];
6878   // exists
6879   // (0:X3=0 /\ 0:X4=1)
6880   //
6881   // If X3 == 0 && X4 == 1, the store to y in P1 has been reordered
6882   // with the store to x in P1. Without the DMB in P1 this may happen.
6883   //
6884   // At the time of writing we don't know of any AArch64 hardware that
6885   // reorders stores in this way, but the Reference Manual permits it.
6886 
6887   void gen_cas_entry(Assembler::operand_size size,
6888                      atomic_memory_order order) {
6889     Register prev = r3, ptr = c_rarg0, compare_val = c_rarg1,
6890       exchange_val = c_rarg2;
6891     bool acquire, release;
6892     switch (order) {
6893       case memory_order_relaxed:
6894         acquire = false;
6895         release = false;
6896         break;
6897       case memory_order_release:
6898         acquire = false;
6899         release = true;
6900         break;
6901       default:
6902         acquire = true;
6903         release = true;
6904         break;
6905     }
6906     __ mov(prev, compare_val);
6907     __ lse_cas(prev, exchange_val, ptr, size, acquire, release, /*not_pair*/true);
6908     if (order == memory_order_conservative) {
6909       __ membar(Assembler::StoreStore|Assembler::StoreLoad);
6910     }
6911     if (size == Assembler::xword) {
6912       __ mov(r0, prev);
6913     } else {
6914       __ movw(r0, prev);
6915     }
6916     __ ret(lr);
6917   }
6918 
6919   void gen_ldadd_entry(Assembler::operand_size size, atomic_memory_order order) {
6920     Register prev = r2, addr = c_rarg0, incr = c_rarg1;
6921     // If not relaxed, then default to conservative.  Relaxed is the only
6922     // case we use enough to be worth specializing.
6923     if (order == memory_order_relaxed) {
6924       __ ldadd(size, incr, prev, addr);
6925     } else {
6926       __ ldaddal(size, incr, prev, addr);
6927       __ membar(Assembler::StoreStore|Assembler::StoreLoad);
6928     }
6929     if (size == Assembler::xword) {
6930       __ mov(r0, prev);
6931     } else {
6932       __ movw(r0, prev);
6933     }
6934     __ ret(lr);
6935   }
6936 
6937   void gen_swpal_entry(Assembler::operand_size size) {
6938     Register prev = r2, addr = c_rarg0, incr = c_rarg1;
6939     __ swpal(size, incr, prev, addr);
6940     __ membar(Assembler::StoreStore|Assembler::StoreLoad);
6941     if (size == Assembler::xword) {
6942       __ mov(r0, prev);
6943     } else {
6944       __ movw(r0, prev);
6945     }
6946     __ ret(lr);
6947   }
6948 
6949   void generate_atomic_entry_points() {
6950     if (! UseLSE) {
6951       return;
6952     }
6953 
6954     __ align(CodeEntryAlignment);
6955     StubCodeMark mark(this, "StubRoutines", "atomic entry points");
6956     address first_entry = __ pc();
6957 
6958     // ADD, memory_order_conservative
6959     AtomicStubMark mark_fetch_add_4(_masm, &aarch64_atomic_fetch_add_4_impl);
6960     gen_ldadd_entry(Assembler::word, memory_order_conservative);
6961     AtomicStubMark mark_fetch_add_8(_masm, &aarch64_atomic_fetch_add_8_impl);
6962     gen_ldadd_entry(Assembler::xword, memory_order_conservative);
6963 
6964     // ADD, memory_order_relaxed
6965     AtomicStubMark mark_fetch_add_4_relaxed
6966       (_masm, &aarch64_atomic_fetch_add_4_relaxed_impl);
6967     gen_ldadd_entry(MacroAssembler::word, memory_order_relaxed);
6968     AtomicStubMark mark_fetch_add_8_relaxed
6969       (_masm, &aarch64_atomic_fetch_add_8_relaxed_impl);
6970     gen_ldadd_entry(MacroAssembler::xword, memory_order_relaxed);
6971 
6972     // XCHG, memory_order_conservative
6973     AtomicStubMark mark_xchg_4(_masm, &aarch64_atomic_xchg_4_impl);
6974     gen_swpal_entry(Assembler::word);
6975     AtomicStubMark mark_xchg_8_impl(_masm, &aarch64_atomic_xchg_8_impl);
6976     gen_swpal_entry(Assembler::xword);
6977 
6978     // CAS, memory_order_conservative
6979     AtomicStubMark mark_cmpxchg_1(_masm, &aarch64_atomic_cmpxchg_1_impl);
6980     gen_cas_entry(MacroAssembler::byte, memory_order_conservative);
6981     AtomicStubMark mark_cmpxchg_4(_masm, &aarch64_atomic_cmpxchg_4_impl);
6982     gen_cas_entry(MacroAssembler::word, memory_order_conservative);
6983     AtomicStubMark mark_cmpxchg_8(_masm, &aarch64_atomic_cmpxchg_8_impl);
6984     gen_cas_entry(MacroAssembler::xword, memory_order_conservative);
6985 
6986     // CAS, memory_order_relaxed
6987     AtomicStubMark mark_cmpxchg_1_relaxed
6988       (_masm, &aarch64_atomic_cmpxchg_1_relaxed_impl);
6989     gen_cas_entry(MacroAssembler::byte, memory_order_relaxed);
6990     AtomicStubMark mark_cmpxchg_4_relaxed
6991       (_masm, &aarch64_atomic_cmpxchg_4_relaxed_impl);
6992     gen_cas_entry(MacroAssembler::word, memory_order_relaxed);
6993     AtomicStubMark mark_cmpxchg_8_relaxed
6994       (_masm, &aarch64_atomic_cmpxchg_8_relaxed_impl);
6995     gen_cas_entry(MacroAssembler::xword, memory_order_relaxed);
6996 
6997     AtomicStubMark mark_cmpxchg_4_release
6998       (_masm, &aarch64_atomic_cmpxchg_4_release_impl);
6999     gen_cas_entry(MacroAssembler::word, memory_order_release);
7000     AtomicStubMark mark_cmpxchg_8_release
7001       (_masm, &aarch64_atomic_cmpxchg_8_release_impl);
7002     gen_cas_entry(MacroAssembler::xword, memory_order_release);
7003 
7004     AtomicStubMark mark_cmpxchg_4_seq_cst
7005       (_masm, &aarch64_atomic_cmpxchg_4_seq_cst_impl);
7006     gen_cas_entry(MacroAssembler::word, memory_order_seq_cst);
7007     AtomicStubMark mark_cmpxchg_8_seq_cst
7008       (_masm, &aarch64_atomic_cmpxchg_8_seq_cst_impl);
7009     gen_cas_entry(MacroAssembler::xword, memory_order_seq_cst);
7010 
7011     ICache::invalidate_range(first_entry, __ pc() - first_entry);
7012   }
7013 #endif // LINUX
7014 
7015   address generate_cont_thaw(Continuation::thaw_kind kind) {
7016     bool return_barrier = Continuation::is_thaw_return_barrier(kind);
7017     bool return_barrier_exception = Continuation::is_thaw_return_barrier_exception(kind);
7018 
7019     address start = __ pc();
7020 
7021     if (return_barrier) {
7022       __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
7023       __ mov(sp, rscratch1);
7024     }
7025     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
7026 
7027     if (return_barrier) {
7028       // preserve possible return value from a method returning to the return barrier
7029       __ fmovd(rscratch1, v0);
7030       __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
7031     }
7032 
7033     __ movw(c_rarg1, (return_barrier ? 1 : 0));
7034     __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), rthread, c_rarg1);
7035     __ mov(rscratch2, r0); // r0 contains the size of the frames to thaw, 0 if overflow or no more frames
7036 
7037     if (return_barrier) {
7038       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
7039       __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
7040       __ fmovd(v0, rscratch1);
7041     }
7042     assert_asm(_masm, (__ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset())), __ cmp(sp, rscratch1)), Assembler::EQ, "incorrect sp");
7043 
7044 
7045     Label thaw_success;
7046     // rscratch2 contains the size of the frames to thaw, 0 if overflow or no more frames
7047     __ cbnz(rscratch2, thaw_success);
7048     __ lea(rscratch1, RuntimeAddress(SharedRuntime::throw_StackOverflowError_entry()));
7049     __ br(rscratch1);
7050     __ bind(thaw_success);
7051 
7052     // make room for the thawed frames
7053     __ sub(rscratch1, sp, rscratch2);
7054     __ andr(rscratch1, rscratch1, -16); // align
7055     __ mov(sp, rscratch1);
7056 
7057     if (return_barrier) {
7058       // save original return value -- again
7059       __ fmovd(rscratch1, v0);
7060       __ stp(rscratch1, r0, Address(__ pre(sp, -2 * wordSize)));
7061     }
7062 
7063     // If we want, we can templatize thaw by kind, and have three different entries
7064     __ movw(c_rarg1, (uint32_t)kind);
7065 
7066     __ call_VM_leaf(Continuation::thaw_entry(), rthread, c_rarg1);
7067     __ mov(rscratch2, r0); // r0 is the sp of the yielding frame
7068 
7069     if (return_barrier) {
7070       // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
7071       __ ldp(rscratch1, r0, Address(__ post(sp, 2 * wordSize)));
7072       __ fmovd(v0, rscratch1);
7073     } else {
7074       __ mov(r0, zr); // return 0 (success) from doYield
7075     }
7076 
7077     // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
7078     __ sub(sp, rscratch2, 2*wordSize); // now pointing to rfp spill
7079     __ mov(rfp, sp);
7080 
7081     if (return_barrier_exception) {
7082       __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
7083       __ authenticate_return_address(c_rarg1);
7084       __ verify_oop(r0);
7085       // save return value containing the exception oop in callee-saved R19
7086       __ mov(r19, r0);
7087 
7088       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
7089 
7090       // Reinitialize the ptrue predicate register, in case the external runtime call clobbers ptrue reg, as we may return to SVE compiled code.
7091       // __ reinitialize_ptrue();
7092 
7093       // see OptoRuntime::generate_exception_blob: r0 -- exception oop, r3 -- exception pc
7094 
7095       __ mov(r1, r0); // the exception handler
7096       __ mov(r0, r19); // restore return value containing the exception oop
7097       __ verify_oop(r0);
7098 
7099       __ leave();
7100       __ mov(r3, lr);
7101       __ br(r1); // the exception handler
7102     } else {
7103       // We're "returning" into the topmost thawed frame; see Thaw::push_return_frame
7104       __ leave();
7105       __ ret(lr);
7106     }
7107 
7108     return start;
7109   }
7110 
7111   address generate_cont_thaw() {
7112     if (!Continuations::enabled()) return nullptr;
7113 
7114     StubCodeMark mark(this, "StubRoutines", "Cont thaw");
7115     address start = __ pc();
7116     generate_cont_thaw(Continuation::thaw_top);
7117     return start;
7118   }
7119 
7120   address generate_cont_returnBarrier() {
7121     if (!Continuations::enabled()) return nullptr;
7122 
7123     // TODO: will probably need multiple return barriers depending on return type
7124     StubCodeMark mark(this, "StubRoutines", "cont return barrier");
7125     address start = __ pc();
7126 
7127     generate_cont_thaw(Continuation::thaw_return_barrier);
7128 
7129     return start;
7130   }
7131 
7132   address generate_cont_returnBarrier_exception() {
7133     if (!Continuations::enabled()) return nullptr;
7134 
7135     StubCodeMark mark(this, "StubRoutines", "cont return barrier exception handler");
7136     address start = __ pc();
7137 
7138     generate_cont_thaw(Continuation::thaw_return_barrier_exception);
7139 
7140     return start;
7141   }
7142 
7143   // In sun.security.util.math.intpoly.IntegerPolynomial1305, integers
7144   // are represented as long[5], with BITS_PER_LIMB = 26.
7145   // Pack five 26-bit limbs into three 64-bit registers.
7146   void pack_26(Register dest0, Register dest1, Register dest2, Register src) {
7147     __ ldp(dest0, rscratch1, Address(src, 0));     // 26 bits
7148     __ add(dest0, dest0, rscratch1, Assembler::LSL, 26);  // 26 bits
7149     __ ldp(rscratch1, rscratch2, Address(src, 2 * sizeof (jlong)));
7150     __ add(dest0, dest0, rscratch1, Assembler::LSL, 52);  // 12 bits
7151 
7152     __ add(dest1, zr, rscratch1, Assembler::LSR, 12);     // 14 bits
7153     __ add(dest1, dest1, rscratch2, Assembler::LSL, 14);  // 26 bits
7154     __ ldr(rscratch1, Address(src, 4 * sizeof (jlong)));
7155     __ add(dest1, dest1, rscratch1, Assembler::LSL, 40);  // 24 bits
7156 
7157     if (dest2->is_valid()) {
7158       __ add(dest2, zr, rscratch1, Assembler::LSR, 24);     // 2 bits
7159     } else {
7160 #ifdef ASSERT
7161       Label OK;
7162       __ cmp(zr, rscratch1, Assembler::LSR, 24);     // 2 bits
7163       __ br(__ EQ, OK);
7164       __ stop("high bits of Poly1305 integer should be zero");
7165       __ should_not_reach_here();
7166       __ bind(OK);
7167 #endif
7168     }
7169   }
7170 
7171   // As above, but return only a 128-bit integer, packed into two
7172   // 64-bit registers.
7173   void pack_26(Register dest0, Register dest1, Register src) {
7174     pack_26(dest0, dest1, noreg, src);
7175   }
7176 
7177   // Multiply and multiply-accumulate unsigned 64-bit registers.
7178   void wide_mul(Register prod_lo, Register prod_hi, Register n, Register m) {
7179     __ mul(prod_lo, n, m);
7180     __ umulh(prod_hi, n, m);
7181   }
7182   void wide_madd(Register sum_lo, Register sum_hi, Register n, Register m) {
7183     wide_mul(rscratch1, rscratch2, n, m);
7184     __ adds(sum_lo, sum_lo, rscratch1);
7185     __ adc(sum_hi, sum_hi, rscratch2);
7186   }
7187 
7188   // Poly1305, RFC 7539
7189 
7190   // See https://loup-vaillant.fr/tutorials/poly1305-design for a
7191   // description of the tricks used to simplify and accelerate this
7192   // computation.
7193 
7194   address generate_poly1305_processBlocks() {
7195     __ align(CodeEntryAlignment);
7196     StubCodeMark mark(this, "StubRoutines", "poly1305_processBlocks");
7197     address start = __ pc();
7198     Label here;
7199     __ enter();
7200     RegSet callee_saved = RegSet::range(r19, r28);
7201     __ push(callee_saved, sp);
7202 
7203     RegSetIterator<Register> regs = (RegSet::range(c_rarg0, r28) - r18_tls - rscratch1 - rscratch2).begin();
7204 
7205     // Arguments
7206     const Register input_start = *regs, length = *++regs, acc_start = *++regs, r_start = *++regs;
7207 
7208     // R_n is the 128-bit randomly-generated key, packed into two
7209     // registers.  The caller passes this key to us as long[5], with
7210     // BITS_PER_LIMB = 26.
7211     const Register R_0 = *++regs, R_1 = *++regs;
7212     pack_26(R_0, R_1, r_start);
7213 
7214     // RR_n is (R_n >> 2) * 5
7215     const Register RR_0 = *++regs, RR_1 = *++regs;
7216     __ lsr(RR_0, R_0, 2);
7217     __ add(RR_0, RR_0, RR_0, Assembler::LSL, 2);
7218     __ lsr(RR_1, R_1, 2);
7219     __ add(RR_1, RR_1, RR_1, Assembler::LSL, 2);
7220 
7221     // U_n is the current checksum
7222     const Register U_0 = *++regs, U_1 = *++regs, U_2 = *++regs;
7223     pack_26(U_0, U_1, U_2, acc_start);
7224 
7225     static constexpr int BLOCK_LENGTH = 16;
7226     Label DONE, LOOP;
7227 
7228     __ cmp(length, checked_cast<u1>(BLOCK_LENGTH));
7229     __ br(Assembler::LT, DONE); {
7230       __ bind(LOOP);
7231 
7232       // S_n is to be the sum of U_n and the next block of data
7233       const Register S_0 = *++regs, S_1 = *++regs, S_2 = *++regs;
7234       __ ldp(S_0, S_1, __ post(input_start, 2 * wordSize));
7235       __ adds(S_0, U_0, S_0);
7236       __ adcs(S_1, U_1, S_1);
7237       __ adc(S_2, U_2, zr);
7238       __ add(S_2, S_2, 1);
7239 
7240       const Register U_0HI = *++regs, U_1HI = *++regs;
7241 
7242       // NB: this logic depends on some of the special properties of
7243       // Poly1305 keys. In particular, because we know that the top
7244       // four bits of R_0 and R_1 are zero, we can add together
7245       // partial products without any risk of needing to propagate a
7246       // carry out.
7247       wide_mul(U_0, U_0HI, S_0, R_0);  wide_madd(U_0, U_0HI, S_1, RR_1); wide_madd(U_0, U_0HI, S_2, RR_0);
7248       wide_mul(U_1, U_1HI, S_0, R_1);  wide_madd(U_1, U_1HI, S_1, R_0);  wide_madd(U_1, U_1HI, S_2, RR_1);
7249       __ andr(U_2, R_0, 3);
7250       __ mul(U_2, S_2, U_2);
7251 
7252       // Recycle registers S_0, S_1, S_2
7253       regs = (regs.remaining() + S_0 + S_1 + S_2).begin();
7254 
7255       // Partial reduction mod 2**130 - 5
7256       __ adds(U_1, U_0HI, U_1);
7257       __ adc(U_2, U_1HI, U_2);
7258       // Sum now in U_2:U_1:U_0.
7259       // Dead: U_0HI, U_1HI.
7260       regs = (regs.remaining() + U_0HI + U_1HI).begin();
7261 
7262       // U_2:U_1:U_0 += (U_2 >> 2) * 5 in two steps
7263 
7264       // First, U_2:U_1:U_0 += (U_2 >> 2)
7265       __ lsr(rscratch1, U_2, 2);
7266       __ andr(U_2, U_2, (u8)3);
7267       __ adds(U_0, U_0, rscratch1);
7268       __ adcs(U_1, U_1, zr);
7269       __ adc(U_2, U_2, zr);
7270       // Second, U_2:U_1:U_0 += (U_2 >> 2) << 2
7271       __ adds(U_0, U_0, rscratch1, Assembler::LSL, 2);
7272       __ adcs(U_1, U_1, zr);
7273       __ adc(U_2, U_2, zr);
7274 
7275       __ sub(length, length, checked_cast<u1>(BLOCK_LENGTH));
7276       __ cmp(length, checked_cast<u1>(BLOCK_LENGTH));
7277       __ br(~ Assembler::LT, LOOP);
7278     }
7279 
7280     // Further reduce modulo 2^130 - 5
7281     __ lsr(rscratch1, U_2, 2);
7282     __ add(rscratch1, rscratch1, rscratch1, Assembler::LSL, 2); // rscratch1 = U_2 * 5
7283     __ adds(U_0, U_0, rscratch1); // U_0 += U_2 * 5
7284     __ adcs(U_1, U_1, zr);
7285     __ andr(U_2, U_2, (u1)3);
7286     __ adc(U_2, U_2, zr);
7287 
7288     // Unpack the sum into five 26-bit limbs and write to memory.
7289     __ ubfiz(rscratch1, U_0, 0, 26);
7290     __ ubfx(rscratch2, U_0, 26, 26);
7291     __ stp(rscratch1, rscratch2, Address(acc_start));
7292     __ ubfx(rscratch1, U_0, 52, 12);
7293     __ bfi(rscratch1, U_1, 12, 14);
7294     __ ubfx(rscratch2, U_1, 14, 26);
7295     __ stp(rscratch1, rscratch2, Address(acc_start, 2 * sizeof (jlong)));
7296     __ ubfx(rscratch1, U_1, 40, 24);
7297     __ bfi(rscratch1, U_2, 24, 3);
7298     __ str(rscratch1, Address(acc_start, 4 * sizeof (jlong)));
7299 
7300     __ bind(DONE);
7301     __ pop(callee_saved, sp);
7302     __ leave();
7303     __ ret(lr);
7304 
7305     return start;
7306   }
7307 
7308   // exception handler for upcall stubs
7309   address generate_upcall_stub_exception_handler() {
7310     StubCodeMark mark(this, "StubRoutines", "upcall stub exception handler");
7311     address start = __ pc();
7312 
7313     // Native caller has no idea how to handle exceptions,
7314     // so we just crash here. Up to callee to catch exceptions.
7315     __ verify_oop(r0);
7316     __ movptr(rscratch1, CAST_FROM_FN_PTR(uint64_t, UpcallLinker::handle_uncaught_exception));
7317     __ blr(rscratch1);
7318     __ should_not_reach_here();
7319 
7320     return start;
7321   }
7322 
7323   // load Method* target of MethodHandle
7324   // j_rarg0 = jobject receiver
7325   // rmethod = result
7326   address generate_upcall_stub_load_target() {
7327     StubCodeMark mark(this, "StubRoutines", "upcall_stub_load_target");
7328     address start = __ pc();
7329 
7330     __ resolve_global_jobject(j_rarg0, rscratch1, rscratch2);
7331       // Load target method from receiver
7332     __ load_heap_oop(rmethod, Address(j_rarg0, java_lang_invoke_MethodHandle::form_offset()), rscratch1, rscratch2);
7333     __ load_heap_oop(rmethod, Address(rmethod, java_lang_invoke_LambdaForm::vmentry_offset()), rscratch1, rscratch2);
7334     __ load_heap_oop(rmethod, Address(rmethod, java_lang_invoke_MemberName::method_offset()), rscratch1, rscratch2);
7335     __ access_load_at(T_ADDRESS, IN_HEAP, rmethod,
7336                       Address(rmethod, java_lang_invoke_ResolvedMethodName::vmtarget_offset()),
7337                       noreg, noreg);
7338     __ str(rmethod, Address(rthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
7339 
7340     __ ret(lr);
7341 
7342     return start;
7343   }
7344 
7345 #undef __
7346 #define __ masm->
7347 
7348   class MontgomeryMultiplyGenerator : public MacroAssembler {
7349 
7350     Register Pa_base, Pb_base, Pn_base, Pm_base, inv, Rlen, Ra, Rb, Rm, Rn,
7351       Pa, Pb, Pn, Pm, Rhi_ab, Rlo_ab, Rhi_mn, Rlo_mn, t0, t1, t2, Ri, Rj;
7352 
7353     RegSet _toSave;
7354     bool _squaring;
7355 
7356   public:
7357     MontgomeryMultiplyGenerator (Assembler *as, bool squaring)
7358       : MacroAssembler(as->code()), _squaring(squaring) {
7359 
7360       // Register allocation
7361 
7362       RegSetIterator<Register> regs = (RegSet::range(r0, r26) - r18_tls).begin();
7363       Pa_base = *regs;       // Argument registers
7364       if (squaring)
7365         Pb_base = Pa_base;
7366       else
7367         Pb_base = *++regs;
7368       Pn_base = *++regs;
7369       Rlen= *++regs;
7370       inv = *++regs;
7371       Pm_base = *++regs;
7372 
7373                           // Working registers:
7374       Ra =  *++regs;        // The current digit of a, b, n, and m.
7375       Rb =  *++regs;
7376       Rm =  *++regs;
7377       Rn =  *++regs;
7378 
7379       Pa =  *++regs;        // Pointers to the current/next digit of a, b, n, and m.
7380       Pb =  *++regs;
7381       Pm =  *++regs;
7382       Pn =  *++regs;
7383 
7384       t0 =  *++regs;        // Three registers which form a
7385       t1 =  *++regs;        // triple-precision accumuator.
7386       t2 =  *++regs;
7387 
7388       Ri =  *++regs;        // Inner and outer loop indexes.
7389       Rj =  *++regs;
7390 
7391       Rhi_ab = *++regs;     // Product registers: low and high parts
7392       Rlo_ab = *++regs;     // of a*b and m*n.
7393       Rhi_mn = *++regs;
7394       Rlo_mn = *++regs;
7395 
7396       // r19 and up are callee-saved.
7397       _toSave = RegSet::range(r19, *regs) + Pm_base;
7398     }
7399 
7400   private:
7401     void save_regs() {
7402       push(_toSave, sp);
7403     }
7404 
7405     void restore_regs() {
7406       pop(_toSave, sp);
7407     }
7408 
7409     template <typename T>
7410     void unroll_2(Register count, T block) {
7411       Label loop, end, odd;
7412       tbnz(count, 0, odd);
7413       cbz(count, end);
7414       align(16);
7415       bind(loop);
7416       (this->*block)();
7417       bind(odd);
7418       (this->*block)();
7419       subs(count, count, 2);
7420       br(Assembler::GT, loop);
7421       bind(end);
7422     }
7423 
7424     template <typename T>
7425     void unroll_2(Register count, T block, Register d, Register s, Register tmp) {
7426       Label loop, end, odd;
7427       tbnz(count, 0, odd);
7428       cbz(count, end);
7429       align(16);
7430       bind(loop);
7431       (this->*block)(d, s, tmp);
7432       bind(odd);
7433       (this->*block)(d, s, tmp);
7434       subs(count, count, 2);
7435       br(Assembler::GT, loop);
7436       bind(end);
7437     }
7438 
7439     void pre1(RegisterOrConstant i) {
7440       block_comment("pre1");
7441       // Pa = Pa_base;
7442       // Pb = Pb_base + i;
7443       // Pm = Pm_base;
7444       // Pn = Pn_base + i;
7445       // Ra = *Pa;
7446       // Rb = *Pb;
7447       // Rm = *Pm;
7448       // Rn = *Pn;
7449       ldr(Ra, Address(Pa_base));
7450       ldr(Rb, Address(Pb_base, i, Address::uxtw(LogBytesPerWord)));
7451       ldr(Rm, Address(Pm_base));
7452       ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
7453       lea(Pa, Address(Pa_base));
7454       lea(Pb, Address(Pb_base, i, Address::uxtw(LogBytesPerWord)));
7455       lea(Pm, Address(Pm_base));
7456       lea(Pn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
7457 
7458       // Zero the m*n result.
7459       mov(Rhi_mn, zr);
7460       mov(Rlo_mn, zr);
7461     }
7462 
7463     // The core multiply-accumulate step of a Montgomery
7464     // multiplication.  The idea is to schedule operations as a
7465     // pipeline so that instructions with long latencies (loads and
7466     // multiplies) have time to complete before their results are
7467     // used.  This most benefits in-order implementations of the
7468     // architecture but out-of-order ones also benefit.
7469     void step() {
7470       block_comment("step");
7471       // MACC(Ra, Rb, t0, t1, t2);
7472       // Ra = *++Pa;
7473       // Rb = *--Pb;
7474       umulh(Rhi_ab, Ra, Rb);
7475       mul(Rlo_ab, Ra, Rb);
7476       ldr(Ra, pre(Pa, wordSize));
7477       ldr(Rb, pre(Pb, -wordSize));
7478       acc(Rhi_mn, Rlo_mn, t0, t1, t2); // The pending m*n from the
7479                                        // previous iteration.
7480       // MACC(Rm, Rn, t0, t1, t2);
7481       // Rm = *++Pm;
7482       // Rn = *--Pn;
7483       umulh(Rhi_mn, Rm, Rn);
7484       mul(Rlo_mn, Rm, Rn);
7485       ldr(Rm, pre(Pm, wordSize));
7486       ldr(Rn, pre(Pn, -wordSize));
7487       acc(Rhi_ab, Rlo_ab, t0, t1, t2);
7488     }
7489 
7490     void post1() {
7491       block_comment("post1");
7492 
7493       // MACC(Ra, Rb, t0, t1, t2);
7494       // Ra = *++Pa;
7495       // Rb = *--Pb;
7496       umulh(Rhi_ab, Ra, Rb);
7497       mul(Rlo_ab, Ra, Rb);
7498       acc(Rhi_mn, Rlo_mn, t0, t1, t2);  // The pending m*n
7499       acc(Rhi_ab, Rlo_ab, t0, t1, t2);
7500 
7501       // *Pm = Rm = t0 * inv;
7502       mul(Rm, t0, inv);
7503       str(Rm, Address(Pm));
7504 
7505       // MACC(Rm, Rn, t0, t1, t2);
7506       // t0 = t1; t1 = t2; t2 = 0;
7507       umulh(Rhi_mn, Rm, Rn);
7508 
7509 #ifndef PRODUCT
7510       // assert(m[i] * n[0] + t0 == 0, "broken Montgomery multiply");
7511       {
7512         mul(Rlo_mn, Rm, Rn);
7513         add(Rlo_mn, t0, Rlo_mn);
7514         Label ok;
7515         cbz(Rlo_mn, ok); {
7516           stop("broken Montgomery multiply");
7517         } bind(ok);
7518       }
7519 #endif
7520       // We have very carefully set things up so that
7521       // m[i]*n[0] + t0 == 0 (mod b), so we don't have to calculate
7522       // the lower half of Rm * Rn because we know the result already:
7523       // it must be -t0.  t0 + (-t0) must generate a carry iff
7524       // t0 != 0.  So, rather than do a mul and an adds we just set
7525       // the carry flag iff t0 is nonzero.
7526       //
7527       // mul(Rlo_mn, Rm, Rn);
7528       // adds(zr, t0, Rlo_mn);
7529       subs(zr, t0, 1); // Set carry iff t0 is nonzero
7530       adcs(t0, t1, Rhi_mn);
7531       adc(t1, t2, zr);
7532       mov(t2, zr);
7533     }
7534 
7535     void pre2(RegisterOrConstant i, RegisterOrConstant len) {
7536       block_comment("pre2");
7537       // Pa = Pa_base + i-len;
7538       // Pb = Pb_base + len;
7539       // Pm = Pm_base + i-len;
7540       // Pn = Pn_base + len;
7541 
7542       if (i.is_register()) {
7543         sub(Rj, i.as_register(), len);
7544       } else {
7545         mov(Rj, i.as_constant());
7546         sub(Rj, Rj, len);
7547       }
7548       // Rj == i-len
7549 
7550       lea(Pa, Address(Pa_base, Rj, Address::uxtw(LogBytesPerWord)));
7551       lea(Pb, Address(Pb_base, len, Address::uxtw(LogBytesPerWord)));
7552       lea(Pm, Address(Pm_base, Rj, Address::uxtw(LogBytesPerWord)));
7553       lea(Pn, Address(Pn_base, len, Address::uxtw(LogBytesPerWord)));
7554 
7555       // Ra = *++Pa;
7556       // Rb = *--Pb;
7557       // Rm = *++Pm;
7558       // Rn = *--Pn;
7559       ldr(Ra, pre(Pa, wordSize));
7560       ldr(Rb, pre(Pb, -wordSize));
7561       ldr(Rm, pre(Pm, wordSize));
7562       ldr(Rn, pre(Pn, -wordSize));
7563 
7564       mov(Rhi_mn, zr);
7565       mov(Rlo_mn, zr);
7566     }
7567 
7568     void post2(RegisterOrConstant i, RegisterOrConstant len) {
7569       block_comment("post2");
7570       if (i.is_constant()) {
7571         mov(Rj, i.as_constant()-len.as_constant());
7572       } else {
7573         sub(Rj, i.as_register(), len);
7574       }
7575 
7576       adds(t0, t0, Rlo_mn); // The pending m*n, low part
7577 
7578       // As soon as we know the least significant digit of our result,
7579       // store it.
7580       // Pm_base[i-len] = t0;
7581       str(t0, Address(Pm_base, Rj, Address::uxtw(LogBytesPerWord)));
7582 
7583       // t0 = t1; t1 = t2; t2 = 0;
7584       adcs(t0, t1, Rhi_mn); // The pending m*n, high part
7585       adc(t1, t2, zr);
7586       mov(t2, zr);
7587     }
7588 
7589     // A carry in t0 after Montgomery multiplication means that we
7590     // should subtract multiples of n from our result in m.  We'll
7591     // keep doing that until there is no carry.
7592     void normalize(RegisterOrConstant len) {
7593       block_comment("normalize");
7594       // while (t0)
7595       //   t0 = sub(Pm_base, Pn_base, t0, len);
7596       Label loop, post, again;
7597       Register cnt = t1, i = t2; // Re-use registers; we're done with them now
7598       cbz(t0, post); {
7599         bind(again); {
7600           mov(i, zr);
7601           mov(cnt, len);
7602           ldr(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
7603           ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
7604           subs(zr, zr, zr); // set carry flag, i.e. no borrow
7605           align(16);
7606           bind(loop); {
7607             sbcs(Rm, Rm, Rn);
7608             str(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
7609             add(i, i, 1);
7610             ldr(Rm, Address(Pm_base, i, Address::uxtw(LogBytesPerWord)));
7611             ldr(Rn, Address(Pn_base, i, Address::uxtw(LogBytesPerWord)));
7612             sub(cnt, cnt, 1);
7613           } cbnz(cnt, loop);
7614           sbc(t0, t0, zr);
7615         } cbnz(t0, again);
7616       } bind(post);
7617     }
7618 
7619     // Move memory at s to d, reversing words.
7620     //    Increments d to end of copied memory
7621     //    Destroys tmp1, tmp2
7622     //    Preserves len
7623     //    Leaves s pointing to the address which was in d at start
7624     void reverse(Register d, Register s, Register len, Register tmp1, Register tmp2) {
7625       assert(tmp1->encoding() < r19->encoding(), "register corruption");
7626       assert(tmp2->encoding() < r19->encoding(), "register corruption");
7627 
7628       lea(s, Address(s, len, Address::uxtw(LogBytesPerWord)));
7629       mov(tmp1, len);
7630       unroll_2(tmp1, &MontgomeryMultiplyGenerator::reverse1, d, s, tmp2);
7631       sub(s, d, len, ext::uxtw, LogBytesPerWord);
7632     }
7633     // where
7634     void reverse1(Register d, Register s, Register tmp) {
7635       ldr(tmp, pre(s, -wordSize));
7636       ror(tmp, tmp, 32);
7637       str(tmp, post(d, wordSize));
7638     }
7639 
7640     void step_squaring() {
7641       // An extra ACC
7642       step();
7643       acc(Rhi_ab, Rlo_ab, t0, t1, t2);
7644     }
7645 
7646     void last_squaring(RegisterOrConstant i) {
7647       Label dont;
7648       // if ((i & 1) == 0) {
7649       tbnz(i.as_register(), 0, dont); {
7650         // MACC(Ra, Rb, t0, t1, t2);
7651         // Ra = *++Pa;
7652         // Rb = *--Pb;
7653         umulh(Rhi_ab, Ra, Rb);
7654         mul(Rlo_ab, Ra, Rb);
7655         acc(Rhi_ab, Rlo_ab, t0, t1, t2);
7656       } bind(dont);
7657     }
7658 
7659     void extra_step_squaring() {
7660       acc(Rhi_mn, Rlo_mn, t0, t1, t2);  // The pending m*n
7661 
7662       // MACC(Rm, Rn, t0, t1, t2);
7663       // Rm = *++Pm;
7664       // Rn = *--Pn;
7665       umulh(Rhi_mn, Rm, Rn);
7666       mul(Rlo_mn, Rm, Rn);
7667       ldr(Rm, pre(Pm, wordSize));
7668       ldr(Rn, pre(Pn, -wordSize));
7669     }
7670 
7671     void post1_squaring() {
7672       acc(Rhi_mn, Rlo_mn, t0, t1, t2);  // The pending m*n
7673 
7674       // *Pm = Rm = t0 * inv;
7675       mul(Rm, t0, inv);
7676       str(Rm, Address(Pm));
7677 
7678       // MACC(Rm, Rn, t0, t1, t2);
7679       // t0 = t1; t1 = t2; t2 = 0;
7680       umulh(Rhi_mn, Rm, Rn);
7681 
7682 #ifndef PRODUCT
7683       // assert(m[i] * n[0] + t0 == 0, "broken Montgomery multiply");
7684       {
7685         mul(Rlo_mn, Rm, Rn);
7686         add(Rlo_mn, t0, Rlo_mn);
7687         Label ok;
7688         cbz(Rlo_mn, ok); {
7689           stop("broken Montgomery multiply");
7690         } bind(ok);
7691       }
7692 #endif
7693       // We have very carefully set things up so that
7694       // m[i]*n[0] + t0 == 0 (mod b), so we don't have to calculate
7695       // the lower half of Rm * Rn because we know the result already:
7696       // it must be -t0.  t0 + (-t0) must generate a carry iff
7697       // t0 != 0.  So, rather than do a mul and an adds we just set
7698       // the carry flag iff t0 is nonzero.
7699       //
7700       // mul(Rlo_mn, Rm, Rn);
7701       // adds(zr, t0, Rlo_mn);
7702       subs(zr, t0, 1); // Set carry iff t0 is nonzero
7703       adcs(t0, t1, Rhi_mn);
7704       adc(t1, t2, zr);
7705       mov(t2, zr);
7706     }
7707 
7708     void acc(Register Rhi, Register Rlo,
7709              Register t0, Register t1, Register t2) {
7710       adds(t0, t0, Rlo);
7711       adcs(t1, t1, Rhi);
7712       adc(t2, t2, zr);
7713     }
7714 
7715   public:
7716     /**
7717      * Fast Montgomery multiplication.  The derivation of the
7718      * algorithm is in A Cryptographic Library for the Motorola
7719      * DSP56000, Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237.
7720      *
7721      * Arguments:
7722      *
7723      * Inputs for multiplication:
7724      *   c_rarg0   - int array elements a
7725      *   c_rarg1   - int array elements b
7726      *   c_rarg2   - int array elements n (the modulus)
7727      *   c_rarg3   - int length
7728      *   c_rarg4   - int inv
7729      *   c_rarg5   - int array elements m (the result)
7730      *
7731      * Inputs for squaring:
7732      *   c_rarg0   - int array elements a
7733      *   c_rarg1   - int array elements n (the modulus)
7734      *   c_rarg2   - int length
7735      *   c_rarg3   - int inv
7736      *   c_rarg4   - int array elements m (the result)
7737      *
7738      */
7739     address generate_multiply() {
7740       Label argh, nothing;
7741       bind(argh);
7742       stop("MontgomeryMultiply total_allocation must be <= 8192");
7743 
7744       align(CodeEntryAlignment);
7745       address entry = pc();
7746 
7747       cbzw(Rlen, nothing);
7748 
7749       enter();
7750 
7751       // Make room.
7752       cmpw(Rlen, 512);
7753       br(Assembler::HI, argh);
7754       sub(Ra, sp, Rlen, ext::uxtw, exact_log2(4 * sizeof (jint)));
7755       andr(sp, Ra, -2 * wordSize);
7756 
7757       lsrw(Rlen, Rlen, 1);  // length in longwords = len/2
7758 
7759       {
7760         // Copy input args, reversing as we go.  We use Ra as a
7761         // temporary variable.
7762         reverse(Ra, Pa_base, Rlen, t0, t1);
7763         if (!_squaring)
7764           reverse(Ra, Pb_base, Rlen, t0, t1);
7765         reverse(Ra, Pn_base, Rlen, t0, t1);
7766       }
7767 
7768       // Push all call-saved registers and also Pm_base which we'll need
7769       // at the end.
7770       save_regs();
7771 
7772 #ifndef PRODUCT
7773       // assert(inv * n[0] == -1UL, "broken inverse in Montgomery multiply");
7774       {
7775         ldr(Rn, Address(Pn_base, 0));
7776         mul(Rlo_mn, Rn, inv);
7777         subs(zr, Rlo_mn, -1);
7778         Label ok;
7779         br(EQ, ok); {
7780           stop("broken inverse in Montgomery multiply");
7781         } bind(ok);
7782       }
7783 #endif
7784 
7785       mov(Pm_base, Ra);
7786 
7787       mov(t0, zr);
7788       mov(t1, zr);
7789       mov(t2, zr);
7790 
7791       block_comment("for (int i = 0; i < len; i++) {");
7792       mov(Ri, zr); {
7793         Label loop, end;
7794         cmpw(Ri, Rlen);
7795         br(Assembler::GE, end);
7796 
7797         bind(loop);
7798         pre1(Ri);
7799 
7800         block_comment("  for (j = i; j; j--) {"); {
7801           movw(Rj, Ri);
7802           unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
7803         } block_comment("  } // j");
7804 
7805         post1();
7806         addw(Ri, Ri, 1);
7807         cmpw(Ri, Rlen);
7808         br(Assembler::LT, loop);
7809         bind(end);
7810         block_comment("} // i");
7811       }
7812 
7813       block_comment("for (int i = len; i < 2*len; i++) {");
7814       mov(Ri, Rlen); {
7815         Label loop, end;
7816         cmpw(Ri, Rlen, Assembler::LSL, 1);
7817         br(Assembler::GE, end);
7818 
7819         bind(loop);
7820         pre2(Ri, Rlen);
7821 
7822         block_comment("  for (j = len*2-i-1; j; j--) {"); {
7823           lslw(Rj, Rlen, 1);
7824           subw(Rj, Rj, Ri);
7825           subw(Rj, Rj, 1);
7826           unroll_2(Rj, &MontgomeryMultiplyGenerator::step);
7827         } block_comment("  } // j");
7828 
7829         post2(Ri, Rlen);
7830         addw(Ri, Ri, 1);
7831         cmpw(Ri, Rlen, Assembler::LSL, 1);
7832         br(Assembler::LT, loop);
7833         bind(end);
7834       }
7835       block_comment("} // i");
7836 
7837       normalize(Rlen);
7838 
7839       mov(Ra, Pm_base);  // Save Pm_base in Ra
7840       restore_regs();  // Restore caller's Pm_base
7841 
7842       // Copy our result into caller's Pm_base
7843       reverse(Pm_base, Ra, Rlen, t0, t1);
7844 
7845       leave();
7846       bind(nothing);
7847       ret(lr);
7848 
7849       return entry;
7850     }
7851     // In C, approximately:
7852 
7853     // void
7854     // montgomery_multiply(julong Pa_base[], julong Pb_base[],
7855     //                     julong Pn_base[], julong Pm_base[],
7856     //                     julong inv, int len) {
7857     //   julong t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
7858     //   julong *Pa, *Pb, *Pn, *Pm;
7859     //   julong Ra, Rb, Rn, Rm;
7860 
7861     //   int i;
7862 
7863     //   assert(inv * Pn_base[0] == -1UL, "broken inverse in Montgomery multiply");
7864 
7865     //   for (i = 0; i < len; i++) {
7866     //     int j;
7867 
7868     //     Pa = Pa_base;
7869     //     Pb = Pb_base + i;
7870     //     Pm = Pm_base;
7871     //     Pn = Pn_base + i;
7872 
7873     //     Ra = *Pa;
7874     //     Rb = *Pb;
7875     //     Rm = *Pm;
7876     //     Rn = *Pn;
7877 
7878     //     int iters = i;
7879     //     for (j = 0; iters--; j++) {
7880     //       assert(Ra == Pa_base[j] && Rb == Pb_base[i-j], "must be");
7881     //       MACC(Ra, Rb, t0, t1, t2);
7882     //       Ra = *++Pa;
7883     //       Rb = *--Pb;
7884     //       assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
7885     //       MACC(Rm, Rn, t0, t1, t2);
7886     //       Rm = *++Pm;
7887     //       Rn = *--Pn;
7888     //     }
7889 
7890     //     assert(Ra == Pa_base[i] && Rb == Pb_base[0], "must be");
7891     //     MACC(Ra, Rb, t0, t1, t2);
7892     //     *Pm = Rm = t0 * inv;
7893     //     assert(Rm == Pm_base[i] && Rn == Pn_base[0], "must be");
7894     //     MACC(Rm, Rn, t0, t1, t2);
7895 
7896     //     assert(t0 == 0, "broken Montgomery multiply");
7897 
7898     //     t0 = t1; t1 = t2; t2 = 0;
7899     //   }
7900 
7901     //   for (i = len; i < 2*len; i++) {
7902     //     int j;
7903 
7904     //     Pa = Pa_base + i-len;
7905     //     Pb = Pb_base + len;
7906     //     Pm = Pm_base + i-len;
7907     //     Pn = Pn_base + len;
7908 
7909     //     Ra = *++Pa;
7910     //     Rb = *--Pb;
7911     //     Rm = *++Pm;
7912     //     Rn = *--Pn;
7913 
7914     //     int iters = len*2-i-1;
7915     //     for (j = i-len+1; iters--; j++) {
7916     //       assert(Ra == Pa_base[j] && Rb == Pb_base[i-j], "must be");
7917     //       MACC(Ra, Rb, t0, t1, t2);
7918     //       Ra = *++Pa;
7919     //       Rb = *--Pb;
7920     //       assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
7921     //       MACC(Rm, Rn, t0, t1, t2);
7922     //       Rm = *++Pm;
7923     //       Rn = *--Pn;
7924     //     }
7925 
7926     //     Pm_base[i-len] = t0;
7927     //     t0 = t1; t1 = t2; t2 = 0;
7928     //   }
7929 
7930     //   while (t0)
7931     //     t0 = sub(Pm_base, Pn_base, t0, len);
7932     // }
7933 
7934     /**
7935      * Fast Montgomery squaring.  This uses asymptotically 25% fewer
7936      * multiplies than Montgomery multiplication so it should be up to
7937      * 25% faster.  However, its loop control is more complex and it
7938      * may actually run slower on some machines.
7939      *
7940      * Arguments:
7941      *
7942      * Inputs:
7943      *   c_rarg0   - int array elements a
7944      *   c_rarg1   - int array elements n (the modulus)
7945      *   c_rarg2   - int length
7946      *   c_rarg3   - int inv
7947      *   c_rarg4   - int array elements m (the result)
7948      *
7949      */
7950     address generate_square() {
7951       Label argh;
7952       bind(argh);
7953       stop("MontgomeryMultiply total_allocation must be <= 8192");
7954 
7955       align(CodeEntryAlignment);
7956       address entry = pc();
7957 
7958       enter();
7959 
7960       // Make room.
7961       cmpw(Rlen, 512);
7962       br(Assembler::HI, argh);
7963       sub(Ra, sp, Rlen, ext::uxtw, exact_log2(4 * sizeof (jint)));
7964       andr(sp, Ra, -2 * wordSize);
7965 
7966       lsrw(Rlen, Rlen, 1);  // length in longwords = len/2
7967 
7968       {
7969         // Copy input args, reversing as we go.  We use Ra as a
7970         // temporary variable.
7971         reverse(Ra, Pa_base, Rlen, t0, t1);
7972         reverse(Ra, Pn_base, Rlen, t0, t1);
7973       }
7974 
7975       // Push all call-saved registers and also Pm_base which we'll need
7976       // at the end.
7977       save_regs();
7978 
7979       mov(Pm_base, Ra);
7980 
7981       mov(t0, zr);
7982       mov(t1, zr);
7983       mov(t2, zr);
7984 
7985       block_comment("for (int i = 0; i < len; i++) {");
7986       mov(Ri, zr); {
7987         Label loop, end;
7988         bind(loop);
7989         cmp(Ri, Rlen);
7990         br(Assembler::GE, end);
7991 
7992         pre1(Ri);
7993 
7994         block_comment("for (j = (i+1)/2; j; j--) {"); {
7995           add(Rj, Ri, 1);
7996           lsr(Rj, Rj, 1);
7997           unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
7998         } block_comment("  } // j");
7999 
8000         last_squaring(Ri);
8001 
8002         block_comment("  for (j = i/2; j; j--) {"); {
8003           lsr(Rj, Ri, 1);
8004           unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
8005         } block_comment("  } // j");
8006 
8007         post1_squaring();
8008         add(Ri, Ri, 1);
8009         cmp(Ri, Rlen);
8010         br(Assembler::LT, loop);
8011 
8012         bind(end);
8013         block_comment("} // i");
8014       }
8015 
8016       block_comment("for (int i = len; i < 2*len; i++) {");
8017       mov(Ri, Rlen); {
8018         Label loop, end;
8019         bind(loop);
8020         cmp(Ri, Rlen, Assembler::LSL, 1);
8021         br(Assembler::GE, end);
8022 
8023         pre2(Ri, Rlen);
8024 
8025         block_comment("  for (j = (2*len-i-1)/2; j; j--) {"); {
8026           lsl(Rj, Rlen, 1);
8027           sub(Rj, Rj, Ri);
8028           sub(Rj, Rj, 1);
8029           lsr(Rj, Rj, 1);
8030           unroll_2(Rj, &MontgomeryMultiplyGenerator::step_squaring);
8031         } block_comment("  } // j");
8032 
8033         last_squaring(Ri);
8034 
8035         block_comment("  for (j = (2*len-i)/2; j; j--) {"); {
8036           lsl(Rj, Rlen, 1);
8037           sub(Rj, Rj, Ri);
8038           lsr(Rj, Rj, 1);
8039           unroll_2(Rj, &MontgomeryMultiplyGenerator::extra_step_squaring);
8040         } block_comment("  } // j");
8041 
8042         post2(Ri, Rlen);
8043         add(Ri, Ri, 1);
8044         cmp(Ri, Rlen, Assembler::LSL, 1);
8045 
8046         br(Assembler::LT, loop);
8047         bind(end);
8048         block_comment("} // i");
8049       }
8050 
8051       normalize(Rlen);
8052 
8053       mov(Ra, Pm_base);  // Save Pm_base in Ra
8054       restore_regs();  // Restore caller's Pm_base
8055 
8056       // Copy our result into caller's Pm_base
8057       reverse(Pm_base, Ra, Rlen, t0, t1);
8058 
8059       leave();
8060       ret(lr);
8061 
8062       return entry;
8063     }
8064     // In C, approximately:
8065 
8066     // void
8067     // montgomery_square(julong Pa_base[], julong Pn_base[],
8068     //                   julong Pm_base[], julong inv, int len) {
8069     //   julong t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
8070     //   julong *Pa, *Pb, *Pn, *Pm;
8071     //   julong Ra, Rb, Rn, Rm;
8072 
8073     //   int i;
8074 
8075     //   assert(inv * Pn_base[0] == -1UL, "broken inverse in Montgomery multiply");
8076 
8077     //   for (i = 0; i < len; i++) {
8078     //     int j;
8079 
8080     //     Pa = Pa_base;
8081     //     Pb = Pa_base + i;
8082     //     Pm = Pm_base;
8083     //     Pn = Pn_base + i;
8084 
8085     //     Ra = *Pa;
8086     //     Rb = *Pb;
8087     //     Rm = *Pm;
8088     //     Rn = *Pn;
8089 
8090     //     int iters = (i+1)/2;
8091     //     for (j = 0; iters--; j++) {
8092     //       assert(Ra == Pa_base[j] && Rb == Pa_base[i-j], "must be");
8093     //       MACC2(Ra, Rb, t0, t1, t2);
8094     //       Ra = *++Pa;
8095     //       Rb = *--Pb;
8096     //       assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
8097     //       MACC(Rm, Rn, t0, t1, t2);
8098     //       Rm = *++Pm;
8099     //       Rn = *--Pn;
8100     //     }
8101     //     if ((i & 1) == 0) {
8102     //       assert(Ra == Pa_base[j], "must be");
8103     //       MACC(Ra, Ra, t0, t1, t2);
8104     //     }
8105     //     iters = i/2;
8106     //     assert(iters == i-j, "must be");
8107     //     for (; iters--; j++) {
8108     //       assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
8109     //       MACC(Rm, Rn, t0, t1, t2);
8110     //       Rm = *++Pm;
8111     //       Rn = *--Pn;
8112     //     }
8113 
8114     //     *Pm = Rm = t0 * inv;
8115     //     assert(Rm == Pm_base[i] && Rn == Pn_base[0], "must be");
8116     //     MACC(Rm, Rn, t0, t1, t2);
8117 
8118     //     assert(t0 == 0, "broken Montgomery multiply");
8119 
8120     //     t0 = t1; t1 = t2; t2 = 0;
8121     //   }
8122 
8123     //   for (i = len; i < 2*len; i++) {
8124     //     int start = i-len+1;
8125     //     int end = start + (len - start)/2;
8126     //     int j;
8127 
8128     //     Pa = Pa_base + i-len;
8129     //     Pb = Pa_base + len;
8130     //     Pm = Pm_base + i-len;
8131     //     Pn = Pn_base + len;
8132 
8133     //     Ra = *++Pa;
8134     //     Rb = *--Pb;
8135     //     Rm = *++Pm;
8136     //     Rn = *--Pn;
8137 
8138     //     int iters = (2*len-i-1)/2;
8139     //     assert(iters == end-start, "must be");
8140     //     for (j = start; iters--; j++) {
8141     //       assert(Ra == Pa_base[j] && Rb == Pa_base[i-j], "must be");
8142     //       MACC2(Ra, Rb, t0, t1, t2);
8143     //       Ra = *++Pa;
8144     //       Rb = *--Pb;
8145     //       assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
8146     //       MACC(Rm, Rn, t0, t1, t2);
8147     //       Rm = *++Pm;
8148     //       Rn = *--Pn;
8149     //     }
8150     //     if ((i & 1) == 0) {
8151     //       assert(Ra == Pa_base[j], "must be");
8152     //       MACC(Ra, Ra, t0, t1, t2);
8153     //     }
8154     //     iters =  (2*len-i)/2;
8155     //     assert(iters == len-j, "must be");
8156     //     for (; iters--; j++) {
8157     //       assert(Rm == Pm_base[j] && Rn == Pn_base[i-j], "must be");
8158     //       MACC(Rm, Rn, t0, t1, t2);
8159     //       Rm = *++Pm;
8160     //       Rn = *--Pn;
8161     //     }
8162     //     Pm_base[i-len] = t0;
8163     //     t0 = t1; t1 = t2; t2 = 0;
8164     //   }
8165 
8166     //   while (t0)
8167     //     t0 = sub(Pm_base, Pn_base, t0, len);
8168     // }
8169   };
8170 
8171 
8172   // Initialization
8173   void generate_initial_stubs() {
8174     // Generate initial stubs and initializes the entry points
8175 
8176     // entry points that exist in all platforms Note: This is code
8177     // that could be shared among different platforms - however the
8178     // benefit seems to be smaller than the disadvantage of having a
8179     // much more complicated generator structure. See also comment in
8180     // stubRoutines.hpp.
8181 
8182     StubRoutines::_forward_exception_entry = generate_forward_exception();
8183 
8184     StubRoutines::_call_stub_entry =
8185       generate_call_stub(StubRoutines::_call_stub_return_address);
8186 
8187     // is referenced by megamorphic call
8188     StubRoutines::_catch_exception_entry = generate_catch_exception();
8189 
8190     // Initialize table for copy memory (arraycopy) check.
8191     if (UnsafeMemoryAccess::_table == nullptr) {
8192       UnsafeMemoryAccess::create_table(8 + 4); // 8 for copyMemory; 4 for setMemory
8193     }
8194 
8195     if (UseCRC32Intrinsics) {
8196       // set table address before stub generation which use it
8197       StubRoutines::_crc_table_adr = (address)StubRoutines::aarch64::_crc_table;
8198       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
8199     }
8200 
8201     if (UseCRC32CIntrinsics) {
8202       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C();
8203     }
8204 
8205     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
8206       StubRoutines::_dsin = generate_dsin_dcos(/* isCos = */ false);
8207     }
8208 
8209     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
8210       StubRoutines::_dcos = generate_dsin_dcos(/* isCos = */ true);
8211     }
8212 
8213     if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_float16ToFloat) &&
8214         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_floatToFloat16)) {
8215       StubRoutines::_hf2f = generate_float16ToFloat();
8216       StubRoutines::_f2hf = generate_floatToFloat16();
8217     }
8218   }
8219 
8220   void generate_continuation_stubs() {
8221     // Continuation stubs:
8222     StubRoutines::_cont_thaw          = generate_cont_thaw();
8223     StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
8224     StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
8225   }
8226 
8227   void generate_final_stubs() {
8228     // support for verify_oop (must happen after universe_init)
8229     if (VerifyOops) {
8230       StubRoutines::_verify_oop_subroutine_entry   = generate_verify_oop();
8231     }
8232 
8233     // arraycopy stubs used by compilers
8234     generate_arraycopy_stubs();
8235 
8236     BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
8237     if (bs_nm != nullptr) {
8238       StubRoutines::_method_entry_barrier = generate_method_entry_barrier();
8239     }
8240 
8241     StubRoutines::aarch64::_spin_wait = generate_spin_wait();
8242 
8243     if (UsePoly1305Intrinsics) {
8244       StubRoutines::_poly1305_processBlocks = generate_poly1305_processBlocks();
8245     }
8246 
8247 #if defined (LINUX) && !defined (__ARM_FEATURE_ATOMICS)
8248 
8249     generate_atomic_entry_points();
8250 
8251 #endif // LINUX
8252 
8253 #ifdef COMPILER2
8254     if (UseSecondarySupersTable) {
8255       StubRoutines::_lookup_secondary_supers_table_slow_path_stub = generate_lookup_secondary_supers_table_slow_path_stub();
8256       if (! InlineSecondarySupersTest) {
8257         for (int slot = 0; slot < Klass::SECONDARY_SUPERS_TABLE_SIZE; slot++) {
8258           StubRoutines::_lookup_secondary_supers_table_stubs[slot]
8259             = generate_lookup_secondary_supers_table_stub(slot);
8260         }
8261       }
8262     }
8263 #endif
8264 
8265     StubRoutines::_upcall_stub_exception_handler = generate_upcall_stub_exception_handler();
8266     StubRoutines::_upcall_stub_load_target = generate_upcall_stub_load_target();
8267 
8268     StubRoutines::aarch64::set_completed(); // Inidicate that arraycopy and zero_blocks stubs are generated
8269   }
8270 
8271   void generate_compiler_stubs() {
8272 #if COMPILER2_OR_JVMCI
8273 
8274     if (UseSVE == 0) {
8275       StubRoutines::aarch64::_vector_iota_indices = generate_iota_indices("iota_indices");
8276     }
8277 
8278     // array equals stub for large arrays.
8279     if (!UseSimpleArrayEquals) {
8280       StubRoutines::aarch64::_large_array_equals = generate_large_array_equals();
8281     }
8282 
8283     // byte_array_inflate stub for large arrays.
8284     StubRoutines::aarch64::_large_byte_array_inflate = generate_large_byte_array_inflate();
8285 
8286     // countPositives stub for large arrays.
8287     StubRoutines::aarch64::_count_positives = generate_count_positives(StubRoutines::aarch64::_count_positives_long);
8288 
8289     generate_compare_long_strings();
8290 
8291     generate_string_indexof_stubs();
8292 
8293 #ifdef COMPILER2
8294     if (UseMultiplyToLenIntrinsic) {
8295       StubRoutines::_multiplyToLen = generate_multiplyToLen();
8296     }
8297 
8298     if (UseSquareToLenIntrinsic) {
8299       StubRoutines::_squareToLen = generate_squareToLen();
8300     }
8301 
8302     if (UseMulAddIntrinsic) {
8303       StubRoutines::_mulAdd = generate_mulAdd();
8304     }
8305 
8306     if (UseSIMDForBigIntegerShiftIntrinsics) {
8307       StubRoutines::_bigIntegerRightShiftWorker = generate_bigIntegerRightShift();
8308       StubRoutines::_bigIntegerLeftShiftWorker  = generate_bigIntegerLeftShift();
8309     }
8310 
8311     if (UseMontgomeryMultiplyIntrinsic) {
8312       StubCodeMark mark(this, "StubRoutines", "montgomeryMultiply");
8313       MontgomeryMultiplyGenerator g(_masm, /*squaring*/false);
8314       StubRoutines::_montgomeryMultiply = g.generate_multiply();
8315     }
8316 
8317     if (UseMontgomerySquareIntrinsic) {
8318       StubCodeMark mark(this, "StubRoutines", "montgomerySquare");
8319       MontgomeryMultiplyGenerator g(_masm, /*squaring*/true);
8320       // We use generate_multiply() rather than generate_square()
8321       // because it's faster for the sizes of modulus we care about.
8322       StubRoutines::_montgomerySquare = g.generate_multiply();
8323     }
8324 #endif // COMPILER2
8325 
8326     if (UseChaCha20Intrinsics) {
8327       StubRoutines::_chacha20Block = generate_chacha20Block_blockpar();
8328     }
8329 
8330     if (UseBASE64Intrinsics) {
8331         StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock();
8332         StubRoutines::_base64_decodeBlock = generate_base64_decodeBlock();
8333     }
8334 
8335     // data cache line writeback
8336     StubRoutines::_data_cache_writeback = generate_data_cache_writeback();
8337     StubRoutines::_data_cache_writeback_sync = generate_data_cache_writeback_sync();
8338 
8339     if (UseAESIntrinsics) {
8340       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
8341       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
8342       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
8343       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt();
8344       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt();
8345     }
8346     if (UseGHASHIntrinsics) {
8347       // StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
8348       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks_wide();
8349     }
8350     if (UseAESIntrinsics && UseGHASHIntrinsics) {
8351       StubRoutines::_galoisCounterMode_AESCrypt = generate_galoisCounterMode_AESCrypt();
8352     }
8353 
8354     if (UseMD5Intrinsics) {
8355       StubRoutines::_md5_implCompress      = generate_md5_implCompress(false,    "md5_implCompress");
8356       StubRoutines::_md5_implCompressMB    = generate_md5_implCompress(true,     "md5_implCompressMB");
8357     }
8358     if (UseSHA1Intrinsics) {
8359       StubRoutines::_sha1_implCompress     = generate_sha1_implCompress(false,   "sha1_implCompress");
8360       StubRoutines::_sha1_implCompressMB   = generate_sha1_implCompress(true,    "sha1_implCompressMB");
8361     }
8362     if (UseSHA256Intrinsics) {
8363       StubRoutines::_sha256_implCompress   = generate_sha256_implCompress(false, "sha256_implCompress");
8364       StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true,  "sha256_implCompressMB");
8365     }
8366     if (UseSHA512Intrinsics) {
8367       StubRoutines::_sha512_implCompress   = generate_sha512_implCompress(false, "sha512_implCompress");
8368       StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true,  "sha512_implCompressMB");
8369     }
8370     if (UseSHA3Intrinsics) {
8371       StubRoutines::_sha3_implCompress     = generate_sha3_implCompress(false,   "sha3_implCompress");
8372       StubRoutines::_sha3_implCompressMB   = generate_sha3_implCompress(true,    "sha3_implCompressMB");
8373     }
8374 
8375     // generate Adler32 intrinsics code
8376     if (UseAdler32Intrinsics) {
8377       StubRoutines::_updateBytesAdler32 = generate_updateBytesAdler32();
8378     }
8379 #endif // COMPILER2_OR_JVMCI
8380   }
8381 
8382  public:
8383   StubGenerator(CodeBuffer* code, StubsKind kind) : StubCodeGenerator(code) {
8384     switch(kind) {
8385     case Initial_stubs:
8386       generate_initial_stubs();
8387       break;
8388      case Continuation_stubs:
8389       generate_continuation_stubs();
8390       break;
8391     case Compiler_stubs:
8392       generate_compiler_stubs();
8393       break;
8394     case Final_stubs:
8395       generate_final_stubs();
8396       break;
8397     default:
8398       fatal("unexpected stubs kind: %d", kind);
8399       break;
8400     };
8401   }
8402 }; // end class declaration
8403 
8404 void StubGenerator_generate(CodeBuffer* code, StubCodeGenerator::StubsKind kind) {
8405   StubGenerator g(code, kind);
8406 }
8407 
8408 
8409 #if defined (LINUX)
8410 
8411 // Define pointers to atomic stubs and initialize them to point to the
8412 // code in atomic_aarch64.S.
8413 
8414 #define DEFAULT_ATOMIC_OP(OPNAME, SIZE, RELAXED)                                \
8415   extern "C" uint64_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl \
8416     (volatile void *ptr, uint64_t arg1, uint64_t arg2);                 \
8417   aarch64_atomic_stub_t aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _impl \
8418     = aarch64_atomic_ ## OPNAME ## _ ## SIZE ## RELAXED ## _default_impl;
8419 
8420 DEFAULT_ATOMIC_OP(fetch_add, 4, )
8421 DEFAULT_ATOMIC_OP(fetch_add, 8, )
8422 DEFAULT_ATOMIC_OP(fetch_add, 4, _relaxed)
8423 DEFAULT_ATOMIC_OP(fetch_add, 8, _relaxed)
8424 DEFAULT_ATOMIC_OP(xchg, 4, )
8425 DEFAULT_ATOMIC_OP(xchg, 8, )
8426 DEFAULT_ATOMIC_OP(cmpxchg, 1, )
8427 DEFAULT_ATOMIC_OP(cmpxchg, 4, )
8428 DEFAULT_ATOMIC_OP(cmpxchg, 8, )
8429 DEFAULT_ATOMIC_OP(cmpxchg, 1, _relaxed)
8430 DEFAULT_ATOMIC_OP(cmpxchg, 4, _relaxed)
8431 DEFAULT_ATOMIC_OP(cmpxchg, 8, _relaxed)
8432 DEFAULT_ATOMIC_OP(cmpxchg, 4, _release)
8433 DEFAULT_ATOMIC_OP(cmpxchg, 8, _release)
8434 DEFAULT_ATOMIC_OP(cmpxchg, 4, _seq_cst)
8435 DEFAULT_ATOMIC_OP(cmpxchg, 8, _seq_cst)
8436 
8437 #undef DEFAULT_ATOMIC_OP
8438 
8439 #endif // LINUX