1 /*
   2  * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "ci/ciUtilities.hpp"
  29 #include "gc/shared/barrierSet.hpp"
  30 #include "gc/shared/barrierSetAssembler.hpp"
  31 #include "gc/shared/barrierSetNMethod.hpp"
  32 #include "interpreter/interpreter.hpp"
  33 #include "memory/universe.hpp"
  34 #include "nativeInst_x86.hpp"
  35 #include "oops/instanceOop.hpp"
  36 #include "oops/method.hpp"
  37 #include "oops/objArrayKlass.hpp"
  38 #include "oops/oop.inline.hpp"
  39 #include "prims/methodHandles.hpp"
  40 #include "runtime/continuation.hpp"
  41 #include "runtime/frame.inline.hpp"
  42 #include "runtime/handles.inline.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "runtime/stubCodeGenerator.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "runtime/thread.inline.hpp"
  47 #ifdef COMPILER2
  48 #include "opto/runtime.hpp"
  49 #endif
  50 #if INCLUDE_ZGC
  51 #include "gc/z/zThreadLocalData.hpp"
  52 #endif
  53 
  54 // Declaration and definition of StubGenerator (no .hpp file).
  55 // For a more detailed description of the stub routine structure
  56 // see the comment in stubRoutines.hpp
  57 
  58 #define __ _masm->
  59 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  60 #define a__ ((Assembler*)_masm)->
  61 
  62 #ifdef PRODUCT
  63 #define BLOCK_COMMENT(str) /* nothing */
  64 #else
  65 #define BLOCK_COMMENT(str) __ block_comment(str)
  66 #endif
  67 
  68 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  69 const int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
  70 
  71 // Stub Code definitions
  72 
  73 class StubGenerator: public StubCodeGenerator {
  74  private:
  75 
  76 #ifdef PRODUCT
  77 #define inc_counter_np(counter) ((void)0)
  78 #else
  79   void inc_counter_np_(int& counter) {
  80     // This can destroy rscratch1 if counter is far from the code cache
  81     __ incrementl(ExternalAddress((address)&counter));
  82   }
  83 #define inc_counter_np(counter) \
  84   BLOCK_COMMENT("inc_counter " #counter); \
  85   inc_counter_np_(counter);
  86 #endif
  87 
  88   // Call stubs are used to call Java from C
  89   //
  90   // Linux Arguments:
  91   //    c_rarg0:   call wrapper address                   address
  92   //    c_rarg1:   result                                 address
  93   //    c_rarg2:   result type                            BasicType
  94   //    c_rarg3:   method                                 Method*
  95   //    c_rarg4:   (interpreter) entry point              address
  96   //    c_rarg5:   parameters                             intptr_t*
  97   //    16(rbp): parameter size (in words)              int
  98   //    24(rbp): thread                                 Thread*
  99   //
 100   //     [ return_from_Java     ] <--- rsp
 101   //     [ argument word n      ]
 102   //      ...
 103   // -12 [ argument word 1      ]
 104   // -11 [ saved r15            ] <--- rsp_after_call
 105   // -10 [ saved r14            ]
 106   //  -9 [ saved r13            ]
 107   //  -8 [ saved r12            ]
 108   //  -7 [ saved rbx            ]
 109   //  -6 [ call wrapper         ]
 110   //  -5 [ result               ]
 111   //  -4 [ result type          ]
 112   //  -3 [ method               ]
 113   //  -2 [ entry point          ]
 114   //  -1 [ parameters           ]
 115   //   0 [ saved rbp            ] <--- rbp
 116   //   1 [ return address       ]
 117   //   2 [ parameter size       ]
 118   //   3 [ thread               ]
 119   //
 120   // Windows Arguments:
 121   //    c_rarg0:   call wrapper address                   address
 122   //    c_rarg1:   result                                 address
 123   //    c_rarg2:   result type                            BasicType
 124   //    c_rarg3:   method                                 Method*
 125   //    48(rbp): (interpreter) entry point              address
 126   //    56(rbp): parameters                             intptr_t*
 127   //    64(rbp): parameter size (in words)              int
 128   //    72(rbp): thread                                 Thread*
 129   //
 130   //     [ return_from_Java     ] <--- rsp
 131   //     [ argument word n      ]
 132   //      ...
 133   // -60 [ argument word 1      ]
 134   // -59 [ saved xmm31          ] <--- rsp after_call
 135   //     [ saved xmm16-xmm30    ] (EVEX enabled, else the space is blank)
 136   // -27 [ saved xmm15          ]
 137   //     [ saved xmm7-xmm14     ]
 138   //  -9 [ saved xmm6           ] (each xmm register takes 2 slots)
 139   //  -7 [ saved r15            ]
 140   //  -6 [ saved r14            ]
 141   //  -5 [ saved r13            ]
 142   //  -4 [ saved r12            ]
 143   //  -3 [ saved rdi            ]
 144   //  -2 [ saved rsi            ]
 145   //  -1 [ saved rbx            ]
 146   //   0 [ saved rbp            ] <--- rbp
 147   //   1 [ return address       ]
 148   //   2 [ call wrapper         ]
 149   //   3 [ result               ]
 150   //   4 [ result type          ]
 151   //   5 [ method               ]
 152   //   6 [ entry point          ]
 153   //   7 [ parameters           ]
 154   //   8 [ parameter size       ]
 155   //   9 [ thread               ]
 156   //
 157   //    Windows reserves the callers stack space for arguments 1-4.
 158   //    We spill c_rarg0-c_rarg3 to this space.
 159 
 160   // Call stub stack layout word offsets from rbp
 161   enum call_stub_layout {
 162 #ifdef _WIN64
 163     xmm_save_first     = 6,  // save from xmm6
 164     xmm_save_last      = 31, // to xmm31
 165     xmm_save_base      = -9,
 166     rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27
 167     r15_off            = -7,
 168     r14_off            = -6,
 169     r13_off            = -5,
 170     r12_off            = -4,
 171     rdi_off            = -3,
 172     rsi_off            = -2,
 173     rbx_off            = -1,
 174     rbp_off            =  0,
 175     retaddr_off        =  1,
 176     call_wrapper_off   =  2,
 177     result_off         =  3,
 178     result_type_off    =  4,
 179     method_off         =  5,
 180     entry_point_off    =  6,
 181     parameters_off     =  7,
 182     parameter_size_off =  8,
 183     thread_off         =  9
 184 #else
 185     rsp_after_call_off = -12,
 186     mxcsr_off          = rsp_after_call_off,
 187     r15_off            = -11,
 188     r14_off            = -10,
 189     r13_off            = -9,
 190     r12_off            = -8,
 191     rbx_off            = -7,
 192     call_wrapper_off   = -6,
 193     result_off         = -5,
 194     result_type_off    = -4,
 195     method_off         = -3,
 196     entry_point_off    = -2,
 197     parameters_off     = -1,
 198     rbp_off            =  0,
 199     retaddr_off        =  1,
 200     parameter_size_off =  2,
 201     thread_off         =  3
 202 #endif
 203   };
 204 
 205 #ifdef _WIN64
 206   Address xmm_save(int reg) {
 207     assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range");
 208     return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize);
 209   }
 210 #endif
 211 
 212   address generate_call_stub(address& return_address) {
 213     assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&
 214            (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
 215            "adjust this code");
 216     StubCodeMark mark(this, "StubRoutines", "call_stub");
 217     address start = __ pc();
 218 
 219     // same as in generate_catch_exception()!
 220     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 221 
 222     const Address call_wrapper  (rbp, call_wrapper_off   * wordSize);
 223     const Address result        (rbp, result_off         * wordSize);
 224     const Address result_type   (rbp, result_type_off    * wordSize);
 225     const Address method        (rbp, method_off         * wordSize);
 226     const Address entry_point   (rbp, entry_point_off    * wordSize);
 227     const Address parameters    (rbp, parameters_off     * wordSize);
 228     const Address parameter_size(rbp, parameter_size_off * wordSize);
 229 
 230     // same as in generate_catch_exception()!
 231     const Address thread        (rbp, thread_off         * wordSize);
 232 
 233     const Address r15_save(rbp, r15_off * wordSize);
 234     const Address r14_save(rbp, r14_off * wordSize);
 235     const Address r13_save(rbp, r13_off * wordSize);
 236     const Address r12_save(rbp, r12_off * wordSize);
 237     const Address rbx_save(rbp, rbx_off * wordSize);
 238 
 239     // stub code
 240     __ enter();
 241     __ subptr(rsp, -rsp_after_call_off * wordSize);
 242 
 243     // save register parameters
 244 #ifndef _WIN64
 245     __ movptr(parameters,   c_rarg5); // parameters
 246     __ movptr(entry_point,  c_rarg4); // entry_point
 247 #endif
 248 
 249     __ movptr(method,       c_rarg3); // method
 250     __ movl(result_type,  c_rarg2);   // result type
 251     __ movptr(result,       c_rarg1); // result
 252     __ movptr(call_wrapper, c_rarg0); // call wrapper
 253 
 254     // save regs belonging to calling function
 255     __ movptr(rbx_save, rbx);
 256     __ movptr(r12_save, r12);
 257     __ movptr(r13_save, r13);
 258     __ movptr(r14_save, r14);
 259     __ movptr(r15_save, r15);
 260 
 261 #ifdef _WIN64
 262     int last_reg = 15;
 263     if (UseAVX > 2) {
 264       last_reg = 31;
 265     }
 266     if (VM_Version::supports_evex()) {
 267       for (int i = xmm_save_first; i <= last_reg; i++) {
 268         __ vextractf32x4(xmm_save(i), as_XMMRegister(i), 0);
 269       }
 270     } else {
 271       for (int i = xmm_save_first; i <= last_reg; i++) {
 272         __ movdqu(xmm_save(i), as_XMMRegister(i));
 273       }
 274     }
 275 
 276     const Address rdi_save(rbp, rdi_off * wordSize);
 277     const Address rsi_save(rbp, rsi_off * wordSize);
 278 
 279     __ movptr(rsi_save, rsi);
 280     __ movptr(rdi_save, rdi);
 281 #else
 282     const Address mxcsr_save(rbp, mxcsr_off * wordSize);
 283     {
 284       Label skip_ldmx;
 285       __ stmxcsr(mxcsr_save);
 286       __ movl(rax, mxcsr_save);
 287       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 288       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 289       __ cmp32(rax, mxcsr_std);
 290       __ jcc(Assembler::equal, skip_ldmx);
 291       __ ldmxcsr(mxcsr_std);
 292       __ bind(skip_ldmx);
 293     }
 294 #endif
 295 
 296     // Load up thread register
 297     __ movptr(r15_thread, thread);
 298     __ reinit_heapbase();
 299 
 300 #ifdef ASSERT
 301     // make sure we have no pending exceptions
 302     {
 303       Label L;
 304       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 305       __ jcc(Assembler::equal, L);
 306       __ stop("StubRoutines::call_stub: entered with pending exception");
 307       __ bind(L);
 308     }
 309 #endif
 310 
 311     // pass parameters if any
 312     BLOCK_COMMENT("pass parameters if any");
 313     Label parameters_done;
 314     __ movl(c_rarg3, parameter_size);
 315     __ testl(c_rarg3, c_rarg3);
 316     __ jcc(Assembler::zero, parameters_done);
 317 
 318     Label loop;
 319     __ movptr(c_rarg2, parameters);       // parameter pointer
 320     __ movl(c_rarg1, c_rarg3);            // parameter counter is in c_rarg1
 321     __ BIND(loop);
 322     __ movptr(rax, Address(c_rarg2, 0));// get parameter
 323     __ addptr(c_rarg2, wordSize);       // advance to next parameter
 324     __ decrementl(c_rarg1);             // decrement counter
 325     __ push(rax);                       // pass parameter
 326     __ jcc(Assembler::notZero, loop);
 327 
 328     // call Java function
 329     __ BIND(parameters_done);
 330     __ movptr(rbx, method);             // get Method*
 331     __ movptr(c_rarg1, entry_point);    // get entry_point
 332     __ mov(r13, rsp);                   // set sender sp
 333     BLOCK_COMMENT("call Java function");
 334     __ call(c_rarg1);
 335 
 336     BLOCK_COMMENT("call_stub_return_address:");
 337     return_address = __ pc();
 338 
 339     // store result depending on type (everything that is not
 340     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 341     __ movptr(c_rarg0, result);
 342     Label is_long, is_float, is_double, exit;
 343     __ movl(c_rarg1, result_type);
 344     __ cmpl(c_rarg1, T_OBJECT);
 345     __ jcc(Assembler::equal, is_long);
 346     __ cmpl(c_rarg1, T_LONG);
 347     __ jcc(Assembler::equal, is_long);
 348     __ cmpl(c_rarg1, T_FLOAT);
 349     __ jcc(Assembler::equal, is_float);
 350     __ cmpl(c_rarg1, T_DOUBLE);
 351     __ jcc(Assembler::equal, is_double);
 352 
 353     // handle T_INT case
 354     __ movl(Address(c_rarg0, 0), rax);
 355 
 356     __ BIND(exit);
 357 
 358     // pop parameters
 359     __ lea(rsp, rsp_after_call);
 360 
 361 #ifdef ASSERT
 362     // verify that threads correspond
 363     {
 364      Label L1, L2, L3;
 365       __ cmpptr(r15_thread, thread);
 366       __ jcc(Assembler::equal, L1);
 367       __ stop("StubRoutines::call_stub: r15_thread is corrupted");
 368       __ bind(L1);
 369       __ get_thread(rbx);
 370       __ cmpptr(r15_thread, thread);
 371       __ jcc(Assembler::equal, L2);
 372       __ stop("StubRoutines::call_stub: r15_thread is modified by call");
 373       __ bind(L2);
 374       __ cmpptr(r15_thread, rbx);
 375       __ jcc(Assembler::equal, L3);
 376       __ stop("StubRoutines::call_stub: threads must correspond");
 377       __ bind(L3);
 378     }
 379 #endif
 380 
 381     // restore regs belonging to calling function
 382 #ifdef _WIN64
 383     // emit the restores for xmm regs
 384     if (VM_Version::supports_evex()) {
 385       for (int i = xmm_save_first; i <= last_reg; i++) {
 386         __ vinsertf32x4(as_XMMRegister(i), as_XMMRegister(i), xmm_save(i), 0);
 387       }
 388     } else {
 389       for (int i = xmm_save_first; i <= last_reg; i++) {
 390         __ movdqu(as_XMMRegister(i), xmm_save(i));
 391       }
 392     }
 393 #endif
 394     __ movptr(r15, r15_save);
 395     __ movptr(r14, r14_save);
 396     __ movptr(r13, r13_save);
 397     __ movptr(r12, r12_save);
 398     __ movptr(rbx, rbx_save);
 399 
 400 #ifdef _WIN64
 401     __ movptr(rdi, rdi_save);
 402     __ movptr(rsi, rsi_save);
 403 #else
 404     __ ldmxcsr(mxcsr_save);
 405 #endif
 406 
 407     // restore rsp
 408     __ addptr(rsp, -rsp_after_call_off * wordSize);
 409 
 410     // return
 411     __ vzeroupper();
 412     __ pop(rbp);
 413     __ ret(0);
 414 
 415     // handle return types different from T_INT
 416     __ BIND(is_long);
 417     __ movq(Address(c_rarg0, 0), rax);
 418     __ jmp(exit);
 419 
 420     __ BIND(is_float);
 421     __ movflt(Address(c_rarg0, 0), xmm0);
 422     __ jmp(exit);
 423 
 424     __ BIND(is_double);
 425     __ movdbl(Address(c_rarg0, 0), xmm0);
 426     __ jmp(exit);
 427 
 428     return start;
 429   }
 430 
 431   // Return point for a Java call if there's an exception thrown in
 432   // Java code.  The exception is caught and transformed into a
 433   // pending exception stored in JavaThread that can be tested from
 434   // within the VM.
 435   //
 436   // Note: Usually the parameters are removed by the callee. In case
 437   // of an exception crossing an activation frame boundary, that is
 438   // not the case if the callee is compiled code => need to setup the
 439   // rsp.
 440   //
 441   // rax: exception oop
 442 
 443   address generate_catch_exception() {
 444     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 445     address start = __ pc();
 446 
 447     // same as in generate_call_stub():
 448     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 449     const Address thread        (rbp, thread_off         * wordSize);
 450 
 451 #ifdef ASSERT
 452     // verify that threads correspond
 453     {
 454       Label L1, L2, L3;
 455       __ cmpptr(r15_thread, thread);
 456       __ jcc(Assembler::equal, L1);
 457       __ stop("StubRoutines::catch_exception: r15_thread is corrupted");
 458       __ bind(L1);
 459       __ get_thread(rbx);
 460       __ cmpptr(r15_thread, thread);
 461       __ jcc(Assembler::equal, L2);
 462       __ stop("StubRoutines::catch_exception: r15_thread is modified by call");
 463       __ bind(L2);
 464       __ cmpptr(r15_thread, rbx);
 465       __ jcc(Assembler::equal, L3);
 466       __ stop("StubRoutines::catch_exception: threads must correspond");
 467       __ bind(L3);
 468     }
 469 #endif
 470 
 471     // set pending exception
 472     __ verify_oop(rax);
 473 
 474     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
 475     __ lea(rscratch1, ExternalAddress((address)__FILE__));
 476     __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
 477     __ movl(Address(r15_thread, Thread::exception_line_offset()), (int)  __LINE__);
 478 
 479     // complete return to VM
 480     assert(StubRoutines::_call_stub_return_address != NULL,
 481            "_call_stub_return_address must have been generated before");
 482     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 483 
 484     return start;
 485   }
 486 
 487   // Continuation point for runtime calls returning with a pending
 488   // exception.  The pending exception check happened in the runtime
 489   // or native call stub.  The pending exception in Thread is
 490   // converted into a Java-level exception.
 491   //
 492   // Contract with Java-level exception handlers:
 493   // rax: exception
 494   // rdx: throwing pc
 495   //
 496   // NOTE: At entry of this stub, exception-pc must be on stack !!
 497 
 498   address generate_forward_exception() {
 499     StubCodeMark mark(this, "StubRoutines", "forward exception");
 500     address start = __ pc();
 501 
 502     // Upon entry, the sp points to the return address returning into
 503     // Java (interpreted or compiled) code; i.e., the return address
 504     // becomes the throwing pc.
 505     //
 506     // Arguments pushed before the runtime call are still on the stack
 507     // but the exception handler will reset the stack pointer ->
 508     // ignore them.  A potential result in registers can be ignored as
 509     // well.
 510 
 511 #ifdef ASSERT
 512     // make sure this code is only executed if there is a pending exception
 513     {
 514       Label L;
 515       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL);
 516       __ jcc(Assembler::notEqual, L);
 517       __ stop("StubRoutines::forward exception: no pending exception (1)");
 518       __ bind(L);
 519     }
 520 #endif
 521 
 522     // compute exception handler into rbx
 523     __ movptr(c_rarg0, Address(rsp, 0));
 524     BLOCK_COMMENT("call exception_handler_for_return_address");
 525     __ call_VM_leaf(CAST_FROM_FN_PTR(address,
 526                          SharedRuntime::exception_handler_for_return_address),
 527                     r15_thread, c_rarg0);
 528     __ mov(rbx, rax);
 529 
 530     // setup rax & rdx, remove return address & clear pending exception
 531     __ pop(rdx);
 532     __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
 533     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 534 
 535 #ifdef ASSERT
 536     // make sure exception is set
 537     {
 538       Label L;
 539       __ testptr(rax, rax);
 540       __ jcc(Assembler::notEqual, L);
 541       __ stop("StubRoutines::forward exception: no pending exception (2)");
 542       __ bind(L);
 543     }
 544 #endif
 545 
 546     // continue at exception handler (return address removed)
 547     // rax: exception
 548     // rbx: exception handler
 549     // rdx: throwing pc
 550     __ verify_oop(rax);
 551     __ jmp(rbx);
 552 
 553     return start;
 554   }
 555 
 556   // Support for jint atomic::xchg(jint exchange_value, volatile jint* dest)
 557   //
 558   // Arguments :
 559   //    c_rarg0: exchange_value
 560   //    c_rarg0: dest
 561   //
 562   // Result:
 563   //    *dest <- ex, return (orig *dest)
 564   address generate_atomic_xchg() {
 565     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 566     address start = __ pc();
 567 
 568     __ movl(rax, c_rarg0); // Copy to eax we need a return value anyhow
 569     __ xchgl(rax, Address(c_rarg1, 0)); // automatic LOCK
 570     __ ret(0);
 571 
 572     return start;
 573   }
 574 
 575   // Support for intptr_t atomic::xchg_long(jlong exchange_value, volatile jlong* dest)
 576   //
 577   // Arguments :
 578   //    c_rarg0: exchange_value
 579   //    c_rarg1: dest
 580   //
 581   // Result:
 582   //    *dest <- ex, return (orig *dest)
 583   address generate_atomic_xchg_long() {
 584     StubCodeMark mark(this, "StubRoutines", "atomic_xchg_long");
 585     address start = __ pc();
 586 
 587     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 588     __ xchgptr(rax, Address(c_rarg1, 0)); // automatic LOCK
 589     __ ret(0);
 590 
 591     return start;
 592   }
 593 
 594   // Support for jint atomic::atomic_cmpxchg(jint exchange_value, volatile jint* dest,
 595   //                                         jint compare_value)
 596   //
 597   // Arguments :
 598   //    c_rarg0: exchange_value
 599   //    c_rarg1: dest
 600   //    c_rarg2: compare_value
 601   //
 602   // Result:
 603   //    if ( compare_value == *dest ) {
 604   //       *dest = exchange_value
 605   //       return compare_value;
 606   //    else
 607   //       return *dest;
 608   address generate_atomic_cmpxchg() {
 609     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");
 610     address start = __ pc();
 611 
 612     __ movl(rax, c_rarg2);
 613     __ lock();
 614     __ cmpxchgl(c_rarg0, Address(c_rarg1, 0));
 615     __ ret(0);
 616 
 617     return start;
 618   }
 619 
 620   // Support for int8_t atomic::atomic_cmpxchg(int8_t exchange_value, volatile int8_t* dest,
 621   //                                           int8_t compare_value)
 622   //
 623   // Arguments :
 624   //    c_rarg0: exchange_value
 625   //    c_rarg1: dest
 626   //    c_rarg2: compare_value
 627   //
 628   // Result:
 629   //    if ( compare_value == *dest ) {
 630   //       *dest = exchange_value
 631   //       return compare_value;
 632   //    else
 633   //       return *dest;
 634   address generate_atomic_cmpxchg_byte() {
 635     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_byte");
 636     address start = __ pc();
 637 
 638     __ movsbq(rax, c_rarg2);
 639     __ lock();
 640     __ cmpxchgb(c_rarg0, Address(c_rarg1, 0));
 641     __ ret(0);
 642 
 643     return start;
 644   }
 645 
 646   // Support for int64_t atomic::atomic_cmpxchg(int64_t exchange_value,
 647   //                                            volatile int64_t* dest,
 648   //                                            int64_t compare_value)
 649   // Arguments :
 650   //    c_rarg0: exchange_value
 651   //    c_rarg1: dest
 652   //    c_rarg2: compare_value
 653   //
 654   // Result:
 655   //    if ( compare_value == *dest ) {
 656   //       *dest = exchange_value
 657   //       return compare_value;
 658   //    else
 659   //       return *dest;
 660   address generate_atomic_cmpxchg_long() {
 661     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");
 662     address start = __ pc();
 663 
 664     __ movq(rax, c_rarg2);
 665     __ lock();
 666     __ cmpxchgq(c_rarg0, Address(c_rarg1, 0));
 667     __ ret(0);
 668 
 669     return start;
 670   }
 671 
 672   // Support for jint atomic::add(jint add_value, volatile jint* dest)
 673   //
 674   // Arguments :
 675   //    c_rarg0: add_value
 676   //    c_rarg1: dest
 677   //
 678   // Result:
 679   //    *dest += add_value
 680   //    return *dest;
 681   address generate_atomic_add() {
 682     StubCodeMark mark(this, "StubRoutines", "atomic_add");
 683     address start = __ pc();
 684 
 685     __ movl(rax, c_rarg0);
 686     __ lock();
 687     __ xaddl(Address(c_rarg1, 0), c_rarg0);
 688     __ addl(rax, c_rarg0);
 689     __ ret(0);
 690 
 691     return start;
 692   }
 693 
 694   // Support for intptr_t atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest)
 695   //
 696   // Arguments :
 697   //    c_rarg0: add_value
 698   //    c_rarg1: dest
 699   //
 700   // Result:
 701   //    *dest += add_value
 702   //    return *dest;
 703   address generate_atomic_add_long() {
 704     StubCodeMark mark(this, "StubRoutines", "atomic_add_long");
 705     address start = __ pc();
 706 
 707     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 708     __ lock();
 709     __ xaddptr(Address(c_rarg1, 0), c_rarg0);
 710     __ addptr(rax, c_rarg0);
 711     __ ret(0);
 712 
 713     return start;
 714   }
 715 
 716   // Support for intptr_t OrderAccess::fence()
 717   //
 718   // Arguments :
 719   //
 720   // Result:
 721   address generate_orderaccess_fence() {
 722     StubCodeMark mark(this, "StubRoutines", "orderaccess_fence");
 723     address start = __ pc();
 724     __ membar(Assembler::StoreLoad);
 725     __ ret(0);
 726 
 727     return start;
 728   }
 729 
 730   // Support for intptr_t get_previous_fp()
 731   //
 732   // This routine is used to find the previous frame pointer for the
 733   // caller (current_frame_guess). This is used as part of debugging
 734   // ps() is seemingly lost trying to find frames.
 735   // This code assumes that caller current_frame_guess) has a frame.
 736   address generate_get_previous_fp() {
 737     StubCodeMark mark(this, "StubRoutines", "get_previous_fp");
 738     const Address old_fp(rbp, 0);
 739     const Address older_fp(rax, 0);
 740     address start = __ pc();
 741 
 742     __ enter();
 743     __ movptr(rax, old_fp); // callers fp
 744     __ movptr(rax, older_fp); // the frame for ps()
 745     __ pop(rbp);
 746     __ ret(0);
 747 
 748     return start;
 749   }
 750 
 751   // Support for intptr_t get_previous_sp()
 752   //
 753   // This routine is used to find the previous stack pointer for the
 754   // caller.
 755   address generate_get_previous_sp() {
 756     StubCodeMark mark(this, "StubRoutines", "get_previous_sp");
 757     address start = __ pc();
 758 
 759     __ movptr(rax, rsp);
 760     __ addptr(rax, 8); // return address is at the top of the stack.
 761     __ ret(0);
 762 
 763     return start;
 764   }
 765 
 766   //----------------------------------------------------------------------------------------------------
 767   // Support for void verify_mxcsr()
 768   //
 769   // This routine is used with -Xcheck:jni to verify that native
 770   // JNI code does not return to Java code without restoring the
 771   // MXCSR register to our expected state.
 772 
 773   address generate_verify_mxcsr() {
 774     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 775     address start = __ pc();
 776 
 777     const Address mxcsr_save(rsp, 0);
 778 
 779     if (CheckJNICalls) {
 780       Label ok_ret;
 781       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 782       __ push(rax);
 783       __ subptr(rsp, wordSize);      // allocate a temp location
 784       __ stmxcsr(mxcsr_save);
 785       __ movl(rax, mxcsr_save);
 786       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 787       __ cmp32(rax, mxcsr_std);
 788       __ jcc(Assembler::equal, ok_ret);
 789 
 790       __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
 791 
 792       __ ldmxcsr(mxcsr_std);
 793 
 794       __ bind(ok_ret);
 795       __ addptr(rsp, wordSize);
 796       __ pop(rax);
 797     }
 798 
 799     __ ret(0);
 800 
 801     return start;
 802   }
 803 
 804   address generate_f2i_fixup() {
 805     StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
 806     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 807 
 808     address start = __ pc();
 809 
 810     Label L;
 811 
 812     __ push(rax);
 813     __ push(c_rarg3);
 814     __ push(c_rarg2);
 815     __ push(c_rarg1);
 816 
 817     __ movl(rax, 0x7f800000);
 818     __ xorl(c_rarg3, c_rarg3);
 819     __ movl(c_rarg2, inout);
 820     __ movl(c_rarg1, c_rarg2);
 821     __ andl(c_rarg1, 0x7fffffff);
 822     __ cmpl(rax, c_rarg1); // NaN? -> 0
 823     __ jcc(Assembler::negative, L);
 824     __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint
 825     __ movl(c_rarg3, 0x80000000);
 826     __ movl(rax, 0x7fffffff);
 827     __ cmovl(Assembler::positive, c_rarg3, rax);
 828 
 829     __ bind(L);
 830     __ movptr(inout, c_rarg3);
 831 
 832     __ pop(c_rarg1);
 833     __ pop(c_rarg2);
 834     __ pop(c_rarg3);
 835     __ pop(rax);
 836 
 837     __ ret(0);
 838 
 839     return start;
 840   }
 841 
 842   address generate_f2l_fixup() {
 843     StubCodeMark mark(this, "StubRoutines", "f2l_fixup");
 844     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 845     address start = __ pc();
 846 
 847     Label L;
 848 
 849     __ push(rax);
 850     __ push(c_rarg3);
 851     __ push(c_rarg2);
 852     __ push(c_rarg1);
 853 
 854     __ movl(rax, 0x7f800000);
 855     __ xorl(c_rarg3, c_rarg3);
 856     __ movl(c_rarg2, inout);
 857     __ movl(c_rarg1, c_rarg2);
 858     __ andl(c_rarg1, 0x7fffffff);
 859     __ cmpl(rax, c_rarg1); // NaN? -> 0
 860     __ jcc(Assembler::negative, L);
 861     __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong
 862     __ mov64(c_rarg3, 0x8000000000000000);
 863     __ mov64(rax, 0x7fffffffffffffff);
 864     __ cmov(Assembler::positive, c_rarg3, rax);
 865 
 866     __ bind(L);
 867     __ movptr(inout, c_rarg3);
 868 
 869     __ pop(c_rarg1);
 870     __ pop(c_rarg2);
 871     __ pop(c_rarg3);
 872     __ pop(rax);
 873 
 874     __ ret(0);
 875 
 876     return start;
 877   }
 878 
 879   address generate_d2i_fixup() {
 880     StubCodeMark mark(this, "StubRoutines", "d2i_fixup");
 881     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 882 
 883     address start = __ pc();
 884 
 885     Label L;
 886 
 887     __ push(rax);
 888     __ push(c_rarg3);
 889     __ push(c_rarg2);
 890     __ push(c_rarg1);
 891     __ push(c_rarg0);
 892 
 893     __ movl(rax, 0x7ff00000);
 894     __ movq(c_rarg2, inout);
 895     __ movl(c_rarg3, c_rarg2);
 896     __ mov(c_rarg1, c_rarg2);
 897     __ mov(c_rarg0, c_rarg2);
 898     __ negl(c_rarg3);
 899     __ shrptr(c_rarg1, 0x20);
 900     __ orl(c_rarg3, c_rarg2);
 901     __ andl(c_rarg1, 0x7fffffff);
 902     __ xorl(c_rarg2, c_rarg2);
 903     __ shrl(c_rarg3, 0x1f);
 904     __ orl(c_rarg1, c_rarg3);
 905     __ cmpl(rax, c_rarg1);
 906     __ jcc(Assembler::negative, L); // NaN -> 0
 907     __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint
 908     __ movl(c_rarg2, 0x80000000);
 909     __ movl(rax, 0x7fffffff);
 910     __ cmov(Assembler::positive, c_rarg2, rax);
 911 
 912     __ bind(L);
 913     __ movptr(inout, c_rarg2);
 914 
 915     __ pop(c_rarg0);
 916     __ pop(c_rarg1);
 917     __ pop(c_rarg2);
 918     __ pop(c_rarg3);
 919     __ pop(rax);
 920 
 921     __ ret(0);
 922 
 923     return start;
 924   }
 925 
 926   address generate_d2l_fixup() {
 927     StubCodeMark mark(this, "StubRoutines", "d2l_fixup");
 928     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 929 
 930     address start = __ pc();
 931 
 932     Label L;
 933 
 934     __ push(rax);
 935     __ push(c_rarg3);
 936     __ push(c_rarg2);
 937     __ push(c_rarg1);
 938     __ push(c_rarg0);
 939 
 940     __ movl(rax, 0x7ff00000);
 941     __ movq(c_rarg2, inout);
 942     __ movl(c_rarg3, c_rarg2);
 943     __ mov(c_rarg1, c_rarg2);
 944     __ mov(c_rarg0, c_rarg2);
 945     __ negl(c_rarg3);
 946     __ shrptr(c_rarg1, 0x20);
 947     __ orl(c_rarg3, c_rarg2);
 948     __ andl(c_rarg1, 0x7fffffff);
 949     __ xorl(c_rarg2, c_rarg2);
 950     __ shrl(c_rarg3, 0x1f);
 951     __ orl(c_rarg1, c_rarg3);
 952     __ cmpl(rax, c_rarg1);
 953     __ jcc(Assembler::negative, L); // NaN -> 0
 954     __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
 955     __ mov64(c_rarg2, 0x8000000000000000);
 956     __ mov64(rax, 0x7fffffffffffffff);
 957     __ cmovq(Assembler::positive, c_rarg2, rax);
 958 
 959     __ bind(L);
 960     __ movq(inout, c_rarg2);
 961 
 962     __ pop(c_rarg0);
 963     __ pop(c_rarg1);
 964     __ pop(c_rarg2);
 965     __ pop(c_rarg3);
 966     __ pop(rax);
 967 
 968     __ ret(0);
 969 
 970     return start;
 971   }
 972 
 973   address generate_fp_mask(const char *stub_name, int64_t mask) {
 974     __ align(CodeEntryAlignment);
 975     StubCodeMark mark(this, "StubRoutines", stub_name);
 976     address start = __ pc();
 977 
 978     __ emit_data64( mask, relocInfo::none );
 979     __ emit_data64( mask, relocInfo::none );
 980 
 981     return start;
 982   }
 983 
 984   address generate_vector_mask(const char *stub_name, int64_t mask) {
 985     __ align(CodeEntryAlignment);
 986     StubCodeMark mark(this, "StubRoutines", stub_name);
 987     address start = __ pc();
 988 
 989     __ emit_data64(mask, relocInfo::none);
 990     __ emit_data64(mask, relocInfo::none);
 991     __ emit_data64(mask, relocInfo::none);
 992     __ emit_data64(mask, relocInfo::none);
 993     __ emit_data64(mask, relocInfo::none);
 994     __ emit_data64(mask, relocInfo::none);
 995     __ emit_data64(mask, relocInfo::none);
 996     __ emit_data64(mask, relocInfo::none);
 997 
 998     return start;
 999   }
1000 
1001   address generate_vector_byte_perm_mask(const char *stub_name) {
1002     __ align(CodeEntryAlignment);
1003     StubCodeMark mark(this, "StubRoutines", stub_name);
1004     address start = __ pc();
1005 
1006     __ emit_data64(0x0000000000000001, relocInfo::none);
1007     __ emit_data64(0x0000000000000003, relocInfo::none);
1008     __ emit_data64(0x0000000000000005, relocInfo::none);
1009     __ emit_data64(0x0000000000000007, relocInfo::none);
1010     __ emit_data64(0x0000000000000000, relocInfo::none);
1011     __ emit_data64(0x0000000000000002, relocInfo::none);
1012     __ emit_data64(0x0000000000000004, relocInfo::none);
1013     __ emit_data64(0x0000000000000006, relocInfo::none);
1014 
1015     return start;
1016   }
1017 
1018   // Non-destructive plausibility checks for oops
1019   //
1020   // Arguments:
1021   //    all args on stack!
1022   //
1023   // Stack after saving c_rarg3:
1024   //    [tos + 0]: saved c_rarg3
1025   //    [tos + 1]: saved c_rarg2
1026   //    [tos + 2]: saved r12 (several TemplateTable methods use it)
1027   //    [tos + 3]: saved flags
1028   //    [tos + 4]: return address
1029   //  * [tos + 5]: error message (char*)
1030   //  * [tos + 6]: object to verify (oop)
1031   //  * [tos + 7]: saved rax - saved by caller and bashed
1032   //  * [tos + 8]: saved r10 (rscratch1) - saved by caller
1033   //  * = popped on exit
1034   address generate_verify_oop() {
1035     StubCodeMark mark(this, "StubRoutines", "verify_oop");
1036     address start = __ pc();
1037 
1038     Label exit, error;
1039 
1040     __ pushf();
1041     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
1042 
1043     __ push(r12);
1044 
1045     // save c_rarg2 and c_rarg3
1046     __ push(c_rarg2);
1047     __ push(c_rarg3);
1048 
1049     enum {
1050            // After previous pushes.
1051            oop_to_verify = 6 * wordSize,
1052            saved_rax     = 7 * wordSize,
1053            saved_r10     = 8 * wordSize,
1054 
1055            // Before the call to MacroAssembler::debug(), see below.
1056            return_addr   = 16 * wordSize,
1057            error_msg     = 17 * wordSize
1058     };
1059 
1060     // get object
1061     __ movptr(rax, Address(rsp, oop_to_verify));
1062 
1063     // make sure object is 'reasonable'
1064     __ testptr(rax, rax);
1065     __ jcc(Assembler::zero, exit); // if obj is NULL it is OK
1066 
1067 #if INCLUDE_ZGC
1068     if (UseZGC) {
1069       // Check if metadata bits indicate a bad oop
1070       __ testptr(rax, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));
1071       __ jcc(Assembler::notZero, error);
1072     }
1073 #endif
1074 
1075     // Check if the oop is in the right area of memory
1076     __ movptr(c_rarg2, rax);
1077     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask());
1078     __ andptr(c_rarg2, c_rarg3);
1079     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits());
1080     __ cmpptr(c_rarg2, c_rarg3);
1081     __ jcc(Assembler::notZero, error);
1082 
1083     // set r12 to heapbase for load_klass()
1084     __ reinit_heapbase();
1085 
1086     // make sure klass is 'reasonable', which is not zero.
1087     __ load_klass(rax, rax);  // get klass
1088     __ testptr(rax, rax);
1089     __ jcc(Assembler::zero, error); // if klass is NULL it is broken
1090 
1091     // return if everything seems ok
1092     __ bind(exit);
1093     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
1094     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1095     __ pop(c_rarg3);                             // restore c_rarg3
1096     __ pop(c_rarg2);                             // restore c_rarg2
1097     __ pop(r12);                                 // restore r12
1098     __ popf();                                   // restore flags
1099     __ ret(4 * wordSize);                        // pop caller saved stuff
1100 
1101     // handle errors
1102     __ bind(error);
1103     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
1104     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1105     __ pop(c_rarg3);                             // get saved c_rarg3 back
1106     __ pop(c_rarg2);                             // get saved c_rarg2 back
1107     __ pop(r12);                                 // get saved r12 back
1108     __ popf();                                   // get saved flags off stack --
1109                                                  // will be ignored
1110 
1111     __ pusha();                                  // push registers
1112                                                  // (rip is already
1113                                                  // already pushed)
1114     // debug(char* msg, int64_t pc, int64_t regs[])
1115     // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
1116     // pushed all the registers, so now the stack looks like:
1117     //     [tos +  0] 16 saved registers
1118     //     [tos + 16] return address
1119     //   * [tos + 17] error message (char*)
1120     //   * [tos + 18] object to verify (oop)
1121     //   * [tos + 19] saved rax - saved by caller and bashed
1122     //   * [tos + 20] saved r10 (rscratch1) - saved by caller
1123     //   * = popped on exit
1124 
1125     __ movptr(c_rarg0, Address(rsp, error_msg));    // pass address of error message
1126     __ movptr(c_rarg1, Address(rsp, return_addr));  // pass return address
1127     __ movq(c_rarg2, rsp);                          // pass address of regs on stack
1128     __ mov(r12, rsp);                               // remember rsp
1129     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1130     __ andptr(rsp, -16);                            // align stack as required by ABI
1131     BLOCK_COMMENT("call MacroAssembler::debug");
1132     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
1133     __ mov(rsp, r12);                               // restore rsp
1134     __ popa();                                      // pop registers (includes r12)
1135     __ ret(4 * wordSize);                           // pop caller saved stuff
1136 
1137     return start;
1138   }
1139 
1140   //
1141   // Verify that a register contains clean 32-bits positive value
1142   // (high 32-bits are 0) so it could be used in 64-bits shifts.
1143   //
1144   //  Input:
1145   //    Rint  -  32-bits value
1146   //    Rtmp  -  scratch
1147   //
1148   void assert_clean_int(Register Rint, Register Rtmp) {
1149 #ifdef ASSERT
1150     Label L;
1151     assert_different_registers(Rtmp, Rint);
1152     __ movslq(Rtmp, Rint);
1153     __ cmpq(Rtmp, Rint);
1154     __ jcc(Assembler::equal, L);
1155     __ stop("high 32-bits of int value are not 0");
1156     __ bind(L);
1157 #endif
1158   }
1159 
1160   //  Generate overlap test for array copy stubs
1161   //
1162   //  Input:
1163   //     c_rarg0 - from
1164   //     c_rarg1 - to
1165   //     c_rarg2 - element count
1166   //
1167   //  Output:
1168   //     rax   - &from[element count - 1]
1169   //
1170   void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) {
1171     assert(no_overlap_target != NULL, "must be generated");
1172     array_overlap_test(no_overlap_target, NULL, sf);
1173   }
1174   void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) {
1175     array_overlap_test(NULL, &L_no_overlap, sf);
1176   }
1177   void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) {
1178     const Register from     = c_rarg0;
1179     const Register to       = c_rarg1;
1180     const Register count    = c_rarg2;
1181     const Register end_from = rax;
1182 
1183     __ cmpptr(to, from);
1184     __ lea(end_from, Address(from, count, sf, 0));
1185     if (NOLp == NULL) {
1186       ExternalAddress no_overlap(no_overlap_target);
1187       __ jump_cc(Assembler::belowEqual, no_overlap);
1188       __ cmpptr(to, end_from);
1189       __ jump_cc(Assembler::aboveEqual, no_overlap);
1190     } else {
1191       __ jcc(Assembler::belowEqual, (*NOLp));
1192       __ cmpptr(to, end_from);
1193       __ jcc(Assembler::aboveEqual, (*NOLp));
1194     }
1195   }
1196 
1197   // Shuffle first three arg regs on Windows into Linux/Solaris locations.
1198   //
1199   // Outputs:
1200   //    rdi - rcx
1201   //    rsi - rdx
1202   //    rdx - r8
1203   //    rcx - r9
1204   //
1205   // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter
1206   // are non-volatile.  r9 and r10 should not be used by the caller.
1207   //
1208   DEBUG_ONLY(bool regs_in_thread;)
1209 
1210   void setup_arg_regs(int nargs = 3) {
1211     const Register saved_rdi = r9;
1212     const Register saved_rsi = r10;
1213     assert(nargs == 3 || nargs == 4, "else fix");
1214 #ifdef _WIN64
1215     assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,
1216            "unexpected argument registers");
1217     if (nargs >= 4)
1218       __ mov(rax, r9);  // r9 is also saved_rdi
1219     __ movptr(saved_rdi, rdi);
1220     __ movptr(saved_rsi, rsi);
1221     __ mov(rdi, rcx); // c_rarg0
1222     __ mov(rsi, rdx); // c_rarg1
1223     __ mov(rdx, r8);  // c_rarg2
1224     if (nargs >= 4)
1225       __ mov(rcx, rax); // c_rarg3 (via rax)
1226 #else
1227     assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,
1228            "unexpected argument registers");
1229 #endif
1230     DEBUG_ONLY(regs_in_thread = false;)
1231   }
1232 
1233   void restore_arg_regs() {
1234     assert(!regs_in_thread, "wrong call to restore_arg_regs");
1235     const Register saved_rdi = r9;
1236     const Register saved_rsi = r10;
1237 #ifdef _WIN64
1238     __ movptr(rdi, saved_rdi);
1239     __ movptr(rsi, saved_rsi);
1240 #endif
1241   }
1242 
1243   // This is used in places where r10 is a scratch register, and can
1244   // be adapted if r9 is needed also.
1245   void setup_arg_regs_using_thread() {
1246     const Register saved_r15 = r9;
1247 #ifdef _WIN64
1248     __ mov(saved_r15, r15);  // r15 is callee saved and needs to be restored
1249     __ get_thread(r15_thread);
1250     assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,
1251            "unexpected argument registers");
1252     __ movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset())), rdi);
1253     __ movptr(Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset())), rsi);
1254 
1255     __ mov(rdi, rcx); // c_rarg0
1256     __ mov(rsi, rdx); // c_rarg1
1257     __ mov(rdx, r8);  // c_rarg2
1258 #else
1259     assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,
1260            "unexpected argument registers");
1261 #endif
1262     DEBUG_ONLY(regs_in_thread = true;)
1263   }
1264 
1265   void restore_arg_regs_using_thread() {
1266     assert(regs_in_thread, "wrong call to restore_arg_regs");
1267     const Register saved_r15 = r9;
1268 #ifdef _WIN64
1269     __ get_thread(r15_thread);
1270     __ movptr(rsi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rsi_offset())));
1271     __ movptr(rdi, Address(r15_thread, in_bytes(JavaThread::windows_saved_rdi_offset())));
1272     __ mov(r15, saved_r15);  // r15 is callee saved and needs to be restored
1273 #endif
1274   }
1275 
1276   // Copy big chunks forward
1277   //
1278   // Inputs:
1279   //   end_from     - source arrays end address
1280   //   end_to       - destination array end address
1281   //   qword_count  - 64-bits element count, negative
1282   //   to           - scratch
1283   //   L_copy_bytes - entry label
1284   //   L_copy_8_bytes  - exit  label
1285   //
1286   void copy_bytes_forward(Register end_from, Register end_to,
1287                              Register qword_count, Register to,
1288                              Label& L_copy_bytes, Label& L_copy_8_bytes) {
1289     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1290     Label L_loop;
1291     __ align(OptoLoopAlignment);
1292     if (UseUnalignedLoadStores) {
1293       Label L_end;
1294       // Copy 64-bytes per iteration
1295       __ BIND(L_loop);
1296       if (UseAVX > 2) {
1297         __ evmovdqul(xmm0, Address(end_from, qword_count, Address::times_8, -56), Assembler::AVX_512bit);
1298         __ evmovdqul(Address(end_to, qword_count, Address::times_8, -56), xmm0, Assembler::AVX_512bit);
1299       } else if (UseAVX == 2) {
1300         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1301         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1302         __ vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24));
1303         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1);
1304       } else {
1305         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1306         __ movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1307         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40));
1308         __ movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1);
1309         __ movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24));
1310         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2);
1311         __ movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8));
1312         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3);
1313       }
1314       __ BIND(L_copy_bytes);
1315       __ addptr(qword_count, 8);
1316       __ jcc(Assembler::lessEqual, L_loop);
1317       __ subptr(qword_count, 4);  // sub(8) and add(4)
1318       __ jccb(Assembler::greater, L_end);
1319       // Copy trailing 32 bytes
1320       if (UseAVX >= 2) {
1321         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1322         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1323       } else {
1324         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1325         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1326         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, - 8));
1327         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm1);
1328       }
1329       __ addptr(qword_count, 4);
1330       __ BIND(L_end);
1331       if (UseAVX >= 2) {
1332         // clean upper bits of YMM registers
1333         __ vpxor(xmm0, xmm0);
1334         __ vpxor(xmm1, xmm1);
1335       }
1336     } else {
1337       // Copy 32-bytes per iteration
1338       __ BIND(L_loop);
1339       __ movq(to, Address(end_from, qword_count, Address::times_8, -24));
1340       __ movq(Address(end_to, qword_count, Address::times_8, -24), to);
1341       __ movq(to, Address(end_from, qword_count, Address::times_8, -16));
1342       __ movq(Address(end_to, qword_count, Address::times_8, -16), to);
1343       __ movq(to, Address(end_from, qword_count, Address::times_8, - 8));
1344       __ movq(Address(end_to, qword_count, Address::times_8, - 8), to);
1345       __ movq(to, Address(end_from, qword_count, Address::times_8, - 0));
1346       __ movq(Address(end_to, qword_count, Address::times_8, - 0), to);
1347 
1348       __ BIND(L_copy_bytes);
1349       __ addptr(qword_count, 4);
1350       __ jcc(Assembler::lessEqual, L_loop);
1351     }
1352     __ subptr(qword_count, 4);
1353     __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
1354   }
1355 
1356   // Copy big chunks backward
1357   //
1358   // Inputs:
1359   //   from         - source arrays address
1360   //   dest         - destination array address
1361   //   qword_count  - 64-bits element count
1362   //   to           - scratch
1363   //   L_copy_bytes - entry label
1364   //   L_copy_8_bytes  - exit  label
1365   //
1366   void copy_bytes_backward(Register from, Register dest,
1367                               Register qword_count, Register to,
1368                               Label& L_copy_bytes, Label& L_copy_8_bytes) {
1369     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1370     Label L_loop;
1371     __ align(OptoLoopAlignment);
1372     if (UseUnalignedLoadStores) {
1373       Label L_end;
1374       // Copy 64-bytes per iteration
1375       __ BIND(L_loop);
1376       if (UseAVX > 2) {
1377         __ evmovdqul(xmm0, Address(from, qword_count, Address::times_8, 0), Assembler::AVX_512bit);
1378         __ evmovdqul(Address(dest, qword_count, Address::times_8, 0), xmm0, Assembler::AVX_512bit);
1379       } else if (UseAVX == 2) {
1380         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32));
1381         __ vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0);
1382         __ vmovdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1383         __ vmovdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1384       } else {
1385         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 48));
1386         __ movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0);
1387         __ movdqu(xmm1, Address(from, qword_count, Address::times_8, 32));
1388         __ movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1);
1389         __ movdqu(xmm2, Address(from, qword_count, Address::times_8, 16));
1390         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2);
1391         __ movdqu(xmm3, Address(from, qword_count, Address::times_8,  0));
1392         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm3);
1393       }
1394       __ BIND(L_copy_bytes);
1395       __ subptr(qword_count, 8);
1396       __ jcc(Assembler::greaterEqual, L_loop);
1397 
1398       __ addptr(qword_count, 4);  // add(8) and sub(4)
1399       __ jccb(Assembler::less, L_end);
1400       // Copy trailing 32 bytes
1401       if (UseAVX >= 2) {
1402         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 0));
1403         __ vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm0);
1404       } else {
1405         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 16));
1406         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm0);
1407         __ movdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1408         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1409       }
1410       __ subptr(qword_count, 4);
1411       __ BIND(L_end);
1412       if (UseAVX >= 2) {
1413         // clean upper bits of YMM registers
1414         __ vpxor(xmm0, xmm0);
1415         __ vpxor(xmm1, xmm1);
1416       }
1417     } else {
1418       // Copy 32-bytes per iteration
1419       __ BIND(L_loop);
1420       __ movq(to, Address(from, qword_count, Address::times_8, 24));
1421       __ movq(Address(dest, qword_count, Address::times_8, 24), to);
1422       __ movq(to, Address(from, qword_count, Address::times_8, 16));
1423       __ movq(Address(dest, qword_count, Address::times_8, 16), to);
1424       __ movq(to, Address(from, qword_count, Address::times_8,  8));
1425       __ movq(Address(dest, qword_count, Address::times_8,  8), to);
1426       __ movq(to, Address(from, qword_count, Address::times_8,  0));
1427       __ movq(Address(dest, qword_count, Address::times_8,  0), to);
1428 
1429       __ BIND(L_copy_bytes);
1430       __ subptr(qword_count, 4);
1431       __ jcc(Assembler::greaterEqual, L_loop);
1432     }
1433     __ addptr(qword_count, 4);
1434     __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
1435   }
1436 
1437   // Arguments:
1438   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1439   //             ignored
1440   //   name    - stub name string
1441   //
1442   // Inputs:
1443   //   c_rarg0   - source array address
1444   //   c_rarg1   - destination array address
1445   //   c_rarg2   - element count, treated as ssize_t, can be zero
1446   //
1447   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1448   // we let the hardware handle it.  The one to eight bytes within words,
1449   // dwords or qwords that span cache line boundaries will still be loaded
1450   // and stored atomically.
1451   //
1452   // Side Effects:
1453   //   disjoint_byte_copy_entry is set to the no-overlap entry point
1454   //   used by generate_conjoint_byte_copy().
1455   //
1456   address generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) {
1457     __ align(CodeEntryAlignment);
1458     StubCodeMark mark(this, "StubRoutines", name);
1459     address start = __ pc();
1460 
1461     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1462     Label L_copy_byte, L_exit;
1463     const Register from        = rdi;  // source array address
1464     const Register to          = rsi;  // destination array address
1465     const Register count       = rdx;  // elements count
1466     const Register byte_count  = rcx;
1467     const Register qword_count = count;
1468     const Register end_from    = from; // source array end address
1469     const Register end_to      = to;   // destination array end address
1470     // End pointers are inclusive, and if count is not zero they point
1471     // to the last unit copied:  end_to[0] := end_from[0]
1472 
1473     __ enter(); // required for proper stackwalking of RuntimeStub frame
1474     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1475 
1476     if (entry != NULL) {
1477       *entry = __ pc();
1478        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1479       BLOCK_COMMENT("Entry:");
1480     }
1481 
1482     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1483                       // r9 and r10 may be used to save non-volatile registers
1484 
1485     {
1486       // UnsafeCopyMemory page error: continue after ucm
1487       UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1488       // 'from', 'to' and 'count' are now valid
1489       __ movptr(byte_count, count);
1490       __ shrptr(count, 3); // count => qword_count
1491 
1492       // Copy from low to high addresses.  Use 'to' as scratch.
1493       __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1494       __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1495       __ negptr(qword_count); // make the count negative
1496       __ jmp(L_copy_bytes);
1497 
1498       // Copy trailing qwords
1499     __ BIND(L_copy_8_bytes);
1500       __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1501       __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1502       __ increment(qword_count);
1503       __ jcc(Assembler::notZero, L_copy_8_bytes);
1504 
1505       // Check for and copy trailing dword
1506     __ BIND(L_copy_4_bytes);
1507       __ testl(byte_count, 4);
1508       __ jccb(Assembler::zero, L_copy_2_bytes);
1509       __ movl(rax, Address(end_from, 8));
1510       __ movl(Address(end_to, 8), rax);
1511 
1512       __ addptr(end_from, 4);
1513       __ addptr(end_to, 4);
1514 
1515       // Check for and copy trailing word
1516     __ BIND(L_copy_2_bytes);
1517       __ testl(byte_count, 2);
1518       __ jccb(Assembler::zero, L_copy_byte);
1519       __ movw(rax, Address(end_from, 8));
1520       __ movw(Address(end_to, 8), rax);
1521 
1522       __ addptr(end_from, 2);
1523       __ addptr(end_to, 2);
1524 
1525       // Check for and copy trailing byte
1526     __ BIND(L_copy_byte);
1527       __ testl(byte_count, 1);
1528       __ jccb(Assembler::zero, L_exit);
1529       __ movb(rax, Address(end_from, 8));
1530       __ movb(Address(end_to, 8), rax);
1531     }
1532   __ BIND(L_exit);
1533     address ucme_exit_pc = __ pc();
1534     restore_arg_regs();
1535     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1536     __ xorptr(rax, rax); // return 0
1537     __ vzeroupper();
1538     __ leave(); // required for proper stackwalking of RuntimeStub frame
1539     __ ret(0);
1540 
1541     {
1542       UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc);
1543       // Copy in multi-bytes chunks
1544       copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1545       __ jmp(L_copy_4_bytes);
1546     }
1547     return start;
1548   }
1549 
1550   // Arguments:
1551   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1552   //             ignored
1553   //   name    - stub name string
1554   //
1555   // Inputs:
1556   //   c_rarg0   - source array address
1557   //   c_rarg1   - destination array address
1558   //   c_rarg2   - element count, treated as ssize_t, can be zero
1559   //
1560   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1561   // we let the hardware handle it.  The one to eight bytes within words,
1562   // dwords or qwords that span cache line boundaries will still be loaded
1563   // and stored atomically.
1564   //
1565   address generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
1566                                       address* entry, const char *name) {
1567     __ align(CodeEntryAlignment);
1568     StubCodeMark mark(this, "StubRoutines", name);
1569     address start = __ pc();
1570 
1571     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1572     const Register from        = rdi;  // source array address
1573     const Register to          = rsi;  // destination array address
1574     const Register count       = rdx;  // elements count
1575     const Register byte_count  = rcx;
1576     const Register qword_count = count;
1577 
1578     __ enter(); // required for proper stackwalking of RuntimeStub frame
1579     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1580 
1581     if (entry != NULL) {
1582       *entry = __ pc();
1583       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1584       BLOCK_COMMENT("Entry:");
1585     }
1586 
1587     array_overlap_test(nooverlap_target, Address::times_1);
1588     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1589                       // r9 and r10 may be used to save non-volatile registers
1590 
1591     {
1592       // UnsafeCopyMemory page error: continue after ucm
1593       UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1594       // 'from', 'to' and 'count' are now valid
1595       __ movptr(byte_count, count);
1596       __ shrptr(count, 3);   // count => qword_count
1597 
1598       // Copy from high to low addresses.
1599 
1600       // Check for and copy trailing byte
1601       __ testl(byte_count, 1);
1602       __ jcc(Assembler::zero, L_copy_2_bytes);
1603       __ movb(rax, Address(from, byte_count, Address::times_1, -1));
1604       __ movb(Address(to, byte_count, Address::times_1, -1), rax);
1605       __ decrement(byte_count); // Adjust for possible trailing word
1606 
1607       // Check for and copy trailing word
1608     __ BIND(L_copy_2_bytes);
1609       __ testl(byte_count, 2);
1610       __ jcc(Assembler::zero, L_copy_4_bytes);
1611       __ movw(rax, Address(from, byte_count, Address::times_1, -2));
1612       __ movw(Address(to, byte_count, Address::times_1, -2), rax);
1613 
1614       // Check for and copy trailing dword
1615     __ BIND(L_copy_4_bytes);
1616       __ testl(byte_count, 4);
1617       __ jcc(Assembler::zero, L_copy_bytes);
1618       __ movl(rax, Address(from, qword_count, Address::times_8));
1619       __ movl(Address(to, qword_count, Address::times_8), rax);
1620       __ jmp(L_copy_bytes);
1621 
1622       // Copy trailing qwords
1623     __ BIND(L_copy_8_bytes);
1624       __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1625       __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1626       __ decrement(qword_count);
1627       __ jcc(Assembler::notZero, L_copy_8_bytes);
1628     }
1629     restore_arg_regs();
1630     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1631     __ xorptr(rax, rax); // return 0
1632     __ vzeroupper();
1633     __ leave(); // required for proper stackwalking of RuntimeStub frame
1634     __ ret(0);
1635 
1636     {
1637       // UnsafeCopyMemory page error: continue after ucm
1638       UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1639       // Copy in multi-bytes chunks
1640       copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1641     }
1642     restore_arg_regs();
1643     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1644     __ xorptr(rax, rax); // return 0
1645     __ vzeroupper();
1646     __ leave(); // required for proper stackwalking of RuntimeStub frame
1647     __ ret(0);
1648 
1649     return start;
1650   }
1651 
1652   // Arguments:
1653   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1654   //             ignored
1655   //   name    - stub name string
1656   //
1657   // Inputs:
1658   //   c_rarg0   - source array address
1659   //   c_rarg1   - destination array address
1660   //   c_rarg2   - element count, treated as ssize_t, can be zero
1661   //
1662   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1663   // let the hardware handle it.  The two or four words within dwords
1664   // or qwords that span cache line boundaries will still be loaded
1665   // and stored atomically.
1666   //
1667   // Side Effects:
1668   //   disjoint_short_copy_entry is set to the no-overlap entry point
1669   //   used by generate_conjoint_short_copy().
1670   //
1671   address generate_disjoint_short_copy(bool aligned, address *entry, const char *name) {
1672     __ align(CodeEntryAlignment);
1673     StubCodeMark mark(this, "StubRoutines", name);
1674     address start = __ pc();
1675 
1676     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
1677     const Register from        = rdi;  // source array address
1678     const Register to          = rsi;  // destination array address
1679     const Register count       = rdx;  // elements count
1680     const Register word_count  = rcx;
1681     const Register qword_count = count;
1682     const Register end_from    = from; // source array end address
1683     const Register end_to      = to;   // destination array end address
1684     // End pointers are inclusive, and if count is not zero they point
1685     // to the last unit copied:  end_to[0] := end_from[0]
1686 
1687     __ enter(); // required for proper stackwalking of RuntimeStub frame
1688     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1689 
1690     if (entry != NULL) {
1691       *entry = __ pc();
1692       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1693       BLOCK_COMMENT("Entry:");
1694     }
1695 
1696     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1697                       // r9 and r10 may be used to save non-volatile registers
1698 
1699     {
1700       // UnsafeCopyMemory page error: continue after ucm
1701       UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1702       // 'from', 'to' and 'count' are now valid
1703       __ movptr(word_count, count);
1704       __ shrptr(count, 2); // count => qword_count
1705 
1706       // Copy from low to high addresses.  Use 'to' as scratch.
1707       __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1708       __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1709       __ negptr(qword_count);
1710       __ jmp(L_copy_bytes);
1711 
1712       // Copy trailing qwords
1713     __ BIND(L_copy_8_bytes);
1714       __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1715       __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1716       __ increment(qword_count);
1717       __ jcc(Assembler::notZero, L_copy_8_bytes);
1718 
1719       // Original 'dest' is trashed, so we can't use it as a
1720       // base register for a possible trailing word copy
1721 
1722       // Check for and copy trailing dword
1723     __ BIND(L_copy_4_bytes);
1724       __ testl(word_count, 2);
1725       __ jccb(Assembler::zero, L_copy_2_bytes);
1726       __ movl(rax, Address(end_from, 8));
1727       __ movl(Address(end_to, 8), rax);
1728 
1729       __ addptr(end_from, 4);
1730       __ addptr(end_to, 4);
1731 
1732       // Check for and copy trailing word
1733     __ BIND(L_copy_2_bytes);
1734       __ testl(word_count, 1);
1735       __ jccb(Assembler::zero, L_exit);
1736       __ movw(rax, Address(end_from, 8));
1737       __ movw(Address(end_to, 8), rax);
1738     }
1739   __ BIND(L_exit);
1740     address ucme_exit_pc = __ pc();
1741     restore_arg_regs();
1742     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1743     __ xorptr(rax, rax); // return 0
1744     __ vzeroupper();
1745     __ leave(); // required for proper stackwalking of RuntimeStub frame
1746     __ ret(0);
1747 
1748     {
1749       UnsafeCopyMemoryMark ucmm(this, !aligned, false, ucme_exit_pc);
1750       // Copy in multi-bytes chunks
1751       copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1752       __ jmp(L_copy_4_bytes);
1753     }
1754 
1755     return start;
1756   }
1757 
1758   address generate_fill(BasicType t, bool aligned, const char *name) {
1759     __ align(CodeEntryAlignment);
1760     StubCodeMark mark(this, "StubRoutines", name);
1761     address start = __ pc();
1762 
1763     BLOCK_COMMENT("Entry:");
1764 
1765     const Register to       = c_rarg0;  // source array address
1766     const Register value    = c_rarg1;  // value
1767     const Register count    = c_rarg2;  // elements count
1768 
1769     __ enter(); // required for proper stackwalking of RuntimeStub frame
1770 
1771     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1772 
1773     __ vzeroupper();
1774     __ leave(); // required for proper stackwalking of RuntimeStub frame
1775     __ ret(0);
1776     return start;
1777   }
1778 
1779   // Arguments:
1780   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1781   //             ignored
1782   //   name    - stub name string
1783   //
1784   // Inputs:
1785   //   c_rarg0   - source array address
1786   //   c_rarg1   - destination array address
1787   //   c_rarg2   - element count, treated as ssize_t, can be zero
1788   //
1789   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1790   // let the hardware handle it.  The two or four words within dwords
1791   // or qwords that span cache line boundaries will still be loaded
1792   // and stored atomically.
1793   //
1794   address generate_conjoint_short_copy(bool aligned, address nooverlap_target,
1795                                        address *entry, const char *name) {
1796     __ align(CodeEntryAlignment);
1797     StubCodeMark mark(this, "StubRoutines", name);
1798     address start = __ pc();
1799 
1800     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes;
1801     const Register from        = rdi;  // source array address
1802     const Register to          = rsi;  // destination array address
1803     const Register count       = rdx;  // elements count
1804     const Register word_count  = rcx;
1805     const Register qword_count = count;
1806 
1807     __ enter(); // required for proper stackwalking of RuntimeStub frame
1808     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1809 
1810     if (entry != NULL) {
1811       *entry = __ pc();
1812       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1813       BLOCK_COMMENT("Entry:");
1814     }
1815 
1816     array_overlap_test(nooverlap_target, Address::times_2);
1817     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1818                       // r9 and r10 may be used to save non-volatile registers
1819 
1820     {
1821       // UnsafeCopyMemory page error: continue after ucm
1822       UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1823       // 'from', 'to' and 'count' are now valid
1824       __ movptr(word_count, count);
1825       __ shrptr(count, 2); // count => qword_count
1826 
1827       // Copy from high to low addresses.  Use 'to' as scratch.
1828 
1829       // Check for and copy trailing word
1830       __ testl(word_count, 1);
1831       __ jccb(Assembler::zero, L_copy_4_bytes);
1832       __ movw(rax, Address(from, word_count, Address::times_2, -2));
1833       __ movw(Address(to, word_count, Address::times_2, -2), rax);
1834 
1835      // Check for and copy trailing dword
1836     __ BIND(L_copy_4_bytes);
1837       __ testl(word_count, 2);
1838       __ jcc(Assembler::zero, L_copy_bytes);
1839       __ movl(rax, Address(from, qword_count, Address::times_8));
1840       __ movl(Address(to, qword_count, Address::times_8), rax);
1841       __ jmp(L_copy_bytes);
1842 
1843       // Copy trailing qwords
1844     __ BIND(L_copy_8_bytes);
1845       __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1846       __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1847       __ decrement(qword_count);
1848       __ jcc(Assembler::notZero, L_copy_8_bytes);
1849     }
1850     restore_arg_regs();
1851     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1852     __ xorptr(rax, rax); // return 0
1853     __ vzeroupper();
1854     __ leave(); // required for proper stackwalking of RuntimeStub frame
1855     __ ret(0);
1856 
1857     {
1858       // UnsafeCopyMemory page error: continue after ucm
1859       UnsafeCopyMemoryMark ucmm(this, !aligned, true);
1860       // Copy in multi-bytes chunks
1861       copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1862     }
1863     restore_arg_regs();
1864     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1865     __ xorptr(rax, rax); // return 0
1866     __ vzeroupper();
1867     __ leave(); // required for proper stackwalking of RuntimeStub frame
1868     __ ret(0);
1869 
1870     return start;
1871   }
1872 
1873   // Arguments:
1874   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1875   //             ignored
1876   //   is_oop  - true => oop array, so generate store check code
1877   //   name    - stub name string
1878   //
1879   // Inputs:
1880   //   c_rarg0   - source array address
1881   //   c_rarg1   - destination array address
1882   //   c_rarg2   - element count, treated as ssize_t, can be zero
1883   //
1884   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1885   // the hardware handle it.  The two dwords within qwords that span
1886   // cache line boundaries will still be loaded and stored atomicly.
1887   //
1888   // Side Effects:
1889   //   disjoint_int_copy_entry is set to the no-overlap entry point
1890   //   used by generate_conjoint_int_oop_copy().
1891   //
1892   address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry,
1893                                          const char *name, bool dest_uninitialized = false) {
1894     __ align(CodeEntryAlignment);
1895     StubCodeMark mark(this, "StubRoutines", name);
1896     address start = __ pc();
1897 
1898     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
1899     const Register from        = rdi;  // source array address
1900     const Register to          = rsi;  // destination array address
1901     const Register count       = rdx;  // elements count
1902     const Register dword_count = rcx;
1903     const Register qword_count = count;
1904     const Register end_from    = from; // source array end address
1905     const Register end_to      = to;   // destination array end address
1906     // End pointers are inclusive, and if count is not zero they point
1907     // to the last unit copied:  end_to[0] := end_from[0]
1908 
1909     __ enter(); // required for proper stackwalking of RuntimeStub frame
1910     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1911 
1912     if (entry != NULL) {
1913       *entry = __ pc();
1914       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1915       BLOCK_COMMENT("Entry:");
1916     }
1917 
1918     setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
1919                                    // r9 is used to save r15_thread
1920 
1921     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
1922     if (dest_uninitialized) {
1923       decorators |= IS_DEST_UNINITIALIZED;
1924     }
1925     if (aligned) {
1926       decorators |= ARRAYCOPY_ALIGNED;
1927     }
1928 
1929     BasicType type = is_oop ? T_OBJECT : T_INT;
1930     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1931     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1932 
1933     {
1934       // UnsafeCopyMemory page error: continue after ucm
1935       UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
1936       // 'from', 'to' and 'count' are now valid
1937       __ movptr(dword_count, count);
1938       __ shrptr(count, 1); // count => qword_count
1939 
1940       // Copy from low to high addresses.  Use 'to' as scratch.
1941       __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1942       __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1943       __ negptr(qword_count);
1944       __ jmp(L_copy_bytes);
1945 
1946       // Copy trailing qwords
1947     __ BIND(L_copy_8_bytes);
1948       __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1949       __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1950       __ increment(qword_count);
1951       __ jcc(Assembler::notZero, L_copy_8_bytes);
1952 
1953       // Check for and copy trailing dword
1954     __ BIND(L_copy_4_bytes);
1955       __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
1956       __ jccb(Assembler::zero, L_exit);
1957       __ movl(rax, Address(end_from, 8));
1958       __ movl(Address(end_to, 8), rax);
1959     }
1960   __ BIND(L_exit);
1961     address ucme_exit_pc = __ pc();
1962     bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
1963     restore_arg_regs_using_thread();
1964     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
1965     __ vzeroupper();
1966     __ xorptr(rax, rax); // return 0
1967     __ leave(); // required for proper stackwalking of RuntimeStub frame
1968     __ ret(0);
1969 
1970     {
1971       UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, false, ucme_exit_pc);
1972       // Copy in multi-bytes chunks
1973       copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1974       __ jmp(L_copy_4_bytes);
1975     }
1976 
1977     return start;
1978   }
1979 
1980   // Arguments:
1981   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1982   //             ignored
1983   //   is_oop  - true => oop array, so generate store check code
1984   //   name    - stub name string
1985   //
1986   // Inputs:
1987   //   c_rarg0   - source array address
1988   //   c_rarg1   - destination array address
1989   //   c_rarg2   - element count, treated as ssize_t, can be zero
1990   //
1991   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1992   // the hardware handle it.  The two dwords within qwords that span
1993   // cache line boundaries will still be loaded and stored atomicly.
1994   //
1995   address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target,
1996                                          address *entry, const char *name,
1997                                          bool dest_uninitialized = false) {
1998     __ align(CodeEntryAlignment);
1999     StubCodeMark mark(this, "StubRoutines", name);
2000     address start = __ pc();
2001 
2002     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2003     const Register from        = rdi;  // source array address
2004     const Register to          = rsi;  // destination array address
2005     const Register count       = rdx;  // elements count
2006     const Register dword_count = rcx;
2007     const Register qword_count = count;
2008 
2009     __ enter(); // required for proper stackwalking of RuntimeStub frame
2010     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2011 
2012     if (entry != NULL) {
2013       *entry = __ pc();
2014        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2015       BLOCK_COMMENT("Entry:");
2016     }
2017 
2018     array_overlap_test(nooverlap_target, Address::times_4);
2019     setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
2020                                    // r9 is used to save r15_thread
2021 
2022     DecoratorSet decorators = IN_HEAP | IS_ARRAY;
2023     if (dest_uninitialized) {
2024       decorators |= IS_DEST_UNINITIALIZED;
2025     }
2026     if (aligned) {
2027       decorators |= ARRAYCOPY_ALIGNED;
2028     }
2029 
2030     BasicType type = is_oop ? T_OBJECT : T_INT;
2031     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2032     // no registers are destroyed by this call
2033     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
2034 
2035     assert_clean_int(count, rax); // Make sure 'count' is clean int.
2036     {
2037       // UnsafeCopyMemory page error: continue after ucm
2038       UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2039       // 'from', 'to' and 'count' are now valid
2040       __ movptr(dword_count, count);
2041       __ shrptr(count, 1); // count => qword_count
2042 
2043       // Copy from high to low addresses.  Use 'to' as scratch.
2044 
2045       // Check for and copy trailing dword
2046       __ testl(dword_count, 1);
2047       __ jcc(Assembler::zero, L_copy_bytes);
2048       __ movl(rax, Address(from, dword_count, Address::times_4, -4));
2049       __ movl(Address(to, dword_count, Address::times_4, -4), rax);
2050       __ jmp(L_copy_bytes);
2051 
2052       // Copy trailing qwords
2053     __ BIND(L_copy_8_bytes);
2054       __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2055       __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2056       __ decrement(qword_count);
2057       __ jcc(Assembler::notZero, L_copy_8_bytes);
2058     }
2059     if (is_oop) {
2060       __ jmp(L_exit);
2061     }
2062     restore_arg_regs_using_thread();
2063     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
2064     __ xorptr(rax, rax); // return 0
2065     __ vzeroupper();
2066     __ leave(); // required for proper stackwalking of RuntimeStub frame
2067     __ ret(0);
2068 
2069     {
2070       // UnsafeCopyMemory page error: continue after ucm
2071       UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2072       // Copy in multi-bytes chunks
2073       copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2074     }
2075 
2076   __ BIND(L_exit);
2077     bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
2078     restore_arg_regs_using_thread();
2079     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
2080     __ xorptr(rax, rax); // return 0
2081     __ vzeroupper();
2082     __ leave(); // required for proper stackwalking of RuntimeStub frame
2083     __ ret(0);
2084 
2085     return start;
2086   }
2087 
2088   // Arguments:
2089   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2090   //             ignored
2091   //   is_oop  - true => oop array, so generate store check code
2092   //   name    - stub name string
2093   //
2094   // Inputs:
2095   //   c_rarg0   - source array address
2096   //   c_rarg1   - destination array address
2097   //   c_rarg2   - element count, treated as ssize_t, can be zero
2098   //
2099  // Side Effects:
2100   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
2101   //   no-overlap entry point used by generate_conjoint_long_oop_copy().
2102   //
2103   address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry,
2104                                           const char *name, bool dest_uninitialized = false) {
2105     __ align(CodeEntryAlignment);
2106     StubCodeMark mark(this, "StubRoutines", name);
2107     address start = __ pc();
2108 
2109     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2110     const Register from        = rdi;  // source array address
2111     const Register to          = rsi;  // destination array address
2112     const Register qword_count = rdx;  // elements count
2113     const Register end_from    = from; // source array end address
2114     const Register end_to      = rcx;  // destination array end address
2115     const Register saved_count = r11;
2116     // End pointers are inclusive, and if count is not zero they point
2117     // to the last unit copied:  end_to[0] := end_from[0]
2118 
2119     __ enter(); // required for proper stackwalking of RuntimeStub frame
2120     // Save no-overlap entry point for generate_conjoint_long_oop_copy()
2121     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2122 
2123     if (entry != NULL) {
2124       *entry = __ pc();
2125       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2126       BLOCK_COMMENT("Entry:");
2127     }
2128 
2129     setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
2130                                      // r9 is used to save r15_thread
2131     // 'from', 'to' and 'qword_count' are now valid
2132 
2133     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
2134     if (dest_uninitialized) {
2135       decorators |= IS_DEST_UNINITIALIZED;
2136     }
2137     if (aligned) {
2138       decorators |= ARRAYCOPY_ALIGNED;
2139     }
2140 
2141     BasicType type = is_oop ? T_OBJECT : T_LONG;
2142     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2143     bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
2144     {
2145       // UnsafeCopyMemory page error: continue after ucm
2146       UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2147 
2148       // Copy from low to high addresses.  Use 'to' as scratch.
2149       __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
2150       __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
2151       __ negptr(qword_count);
2152       __ jmp(L_copy_bytes);
2153 
2154       // Copy trailing qwords
2155     __ BIND(L_copy_8_bytes);
2156       __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2157       __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2158       __ increment(qword_count);
2159       __ jcc(Assembler::notZero, L_copy_8_bytes);
2160     }
2161     if (is_oop) {
2162       __ jmp(L_exit);
2163     } else {
2164       restore_arg_regs_using_thread();
2165       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2166       __ xorptr(rax, rax); // return 0
2167       __ vzeroupper();
2168       __ leave(); // required for proper stackwalking of RuntimeStub frame
2169       __ ret(0);
2170     }
2171 
2172     {
2173       // UnsafeCopyMemory page error: continue after ucm
2174       UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2175       // Copy in multi-bytes chunks
2176       copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2177     }
2178 
2179     __ BIND(L_exit);
2180     bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
2181     restore_arg_regs_using_thread();
2182     if (is_oop) {
2183       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2184     } else {
2185       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2186     }
2187     __ vzeroupper();
2188     __ xorptr(rax, rax); // return 0
2189     __ leave(); // required for proper stackwalking of RuntimeStub frame
2190     __ ret(0);
2191 
2192     return start;
2193   }
2194 
2195   // Arguments:
2196   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2197   //             ignored
2198   //   is_oop  - true => oop array, so generate store check code
2199   //   name    - stub name string
2200   //
2201   // Inputs:
2202   //   c_rarg0   - source array address
2203   //   c_rarg1   - destination array address
2204   //   c_rarg2   - element count, treated as ssize_t, can be zero
2205   //
2206   address generate_conjoint_long_oop_copy(bool aligned, bool is_oop,
2207                                           address nooverlap_target, address *entry,
2208                                           const char *name, bool dest_uninitialized = false) {
2209     __ align(CodeEntryAlignment);
2210     StubCodeMark mark(this, "StubRoutines", name);
2211     address start = __ pc();
2212 
2213     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2214     const Register from        = rdi;  // source array address
2215     const Register to          = rsi;  // destination array address
2216     const Register qword_count = rdx;  // elements count
2217     const Register saved_count = rcx;
2218 
2219     __ enter(); // required for proper stackwalking of RuntimeStub frame
2220     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2221 
2222     if (entry != NULL) {
2223       *entry = __ pc();
2224       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2225       BLOCK_COMMENT("Entry:");
2226     }
2227 
2228     array_overlap_test(nooverlap_target, Address::times_8);
2229     setup_arg_regs_using_thread(); // from => rdi, to => rsi, count => rdx
2230                                    // r9 is used to save r15_thread
2231     // 'from', 'to' and 'qword_count' are now valid
2232 
2233     DecoratorSet decorators = IN_HEAP | IS_ARRAY;
2234     if (dest_uninitialized) {
2235       decorators |= IS_DEST_UNINITIALIZED;
2236     }
2237     if (aligned) {
2238       decorators |= ARRAYCOPY_ALIGNED;
2239     }
2240 
2241     BasicType type = is_oop ? T_OBJECT : T_LONG;
2242     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2243     bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
2244     {
2245       // UnsafeCopyMemory page error: continue after ucm
2246       UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2247 
2248       __ jmp(L_copy_bytes);
2249 
2250       // Copy trailing qwords
2251     __ BIND(L_copy_8_bytes);
2252       __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2253       __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2254       __ decrement(qword_count);
2255       __ jcc(Assembler::notZero, L_copy_8_bytes);
2256     }
2257     if (is_oop) {
2258       __ jmp(L_exit);
2259     } else {
2260       restore_arg_regs_using_thread();
2261       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2262       __ xorptr(rax, rax); // return 0
2263       __ vzeroupper();
2264       __ leave(); // required for proper stackwalking of RuntimeStub frame
2265       __ ret(0);
2266     }
2267     {
2268       // UnsafeCopyMemory page error: continue after ucm
2269       UnsafeCopyMemoryMark ucmm(this, !is_oop && !aligned, true);
2270 
2271       // Copy in multi-bytes chunks
2272       copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2273     }
2274     __ BIND(L_exit);
2275     bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
2276     restore_arg_regs_using_thread();
2277     if (is_oop) {
2278       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2279     } else {
2280       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2281     }
2282     __ vzeroupper();
2283     __ xorptr(rax, rax); // return 0
2284     __ leave(); // required for proper stackwalking of RuntimeStub frame
2285     __ ret(0);
2286 
2287     return start;
2288   }
2289 
2290 
2291   // Helper for generating a dynamic type check.
2292   // Smashes no registers.
2293   void generate_type_check(Register sub_klass,
2294                            Register super_check_offset,
2295                            Register super_klass,
2296                            Label& L_success) {
2297     assert_different_registers(sub_klass, super_check_offset, super_klass);
2298 
2299     BLOCK_COMMENT("type_check:");
2300 
2301     Label L_miss;
2302 
2303     __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg,        &L_success, &L_miss, NULL,
2304                                      super_check_offset);
2305     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL);
2306 
2307     // Fall through on failure!
2308     __ BIND(L_miss);
2309   }
2310 
2311   //
2312   //  Generate checkcasting array copy stub
2313   //
2314   //  Input:
2315   //    c_rarg0   - source array address
2316   //    c_rarg1   - destination array address
2317   //    c_rarg2   - element count, treated as ssize_t, can be zero
2318   //    c_rarg3   - size_t ckoff (super_check_offset)
2319   // not Win64
2320   //    c_rarg4   - oop ckval (super_klass)
2321   // Win64
2322   //    rsp+40    - oop ckval (super_klass)
2323   //
2324   //  Output:
2325   //    rax ==  0  -  success
2326   //    rax == -1^K - failure, where K is partial transfer count
2327   //
2328   address generate_checkcast_copy(const char *name, address *entry,
2329                                   bool dest_uninitialized = false) {
2330 
2331     Label L_load_element, L_store_element, L_do_card_marks, L_done;
2332 
2333     // Input registers (after setup_arg_regs)
2334     const Register from        = rdi;   // source array address
2335     const Register to          = rsi;   // destination array address
2336     const Register length      = rdx;   // elements count
2337     const Register ckoff       = rcx;   // super_check_offset
2338     const Register ckval       = r8;    // super_klass
2339 
2340     // Registers used as temps (r13, r14 are save-on-entry)
2341     const Register end_from    = from;  // source array end address
2342     const Register end_to      = r13;   // destination array end address
2343     const Register count       = rdx;   // -(count_remaining)
2344     const Register r14_length  = r14;   // saved copy of length
2345     // End pointers are inclusive, and if length is not zero they point
2346     // to the last unit copied:  end_to[0] := end_from[0]
2347 
2348     const Register rax_oop    = rax;    // actual oop copied
2349     const Register r11_klass  = r11;    // oop._klass
2350 
2351     //---------------------------------------------------------------
2352     // Assembler stub will be used for this call to arraycopy
2353     // if the two arrays are subtypes of Object[] but the
2354     // destination array type is not equal to or a supertype
2355     // of the source type.  Each element must be separately
2356     // checked.
2357 
2358     __ align(CodeEntryAlignment);
2359     StubCodeMark mark(this, "StubRoutines", name);
2360     address start = __ pc();
2361 
2362     __ enter(); // required for proper stackwalking of RuntimeStub frame
2363 
2364 #ifdef ASSERT
2365     // caller guarantees that the arrays really are different
2366     // otherwise, we would have to make conjoint checks
2367     { Label L;
2368       array_overlap_test(L, TIMES_OOP);
2369       __ stop("checkcast_copy within a single array");
2370       __ bind(L);
2371     }
2372 #endif //ASSERT
2373 
2374     setup_arg_regs(4); // from => rdi, to => rsi, length => rdx
2375                        // ckoff => rcx, ckval => r8
2376                        // r9 and r10 may be used to save non-volatile registers
2377 #ifdef _WIN64
2378     // last argument (#4) is on stack on Win64
2379     __ movptr(ckval, Address(rsp, 6 * wordSize));
2380 #endif
2381 
2382     // Caller of this entry point must set up the argument registers.
2383     if (entry != NULL) {
2384       *entry = __ pc();
2385       BLOCK_COMMENT("Entry:");
2386     }
2387 
2388     // allocate spill slots for r13, r14
2389     enum {
2390       saved_r13_offset,
2391       saved_r14_offset,
2392       saved_r10_offset,
2393       saved_rbp_offset
2394     };
2395     __ subptr(rsp, saved_rbp_offset * wordSize);
2396     __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
2397     __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
2398     __ movptr(Address(rsp, saved_r10_offset * wordSize), r10);
2399 
2400 #ifdef ASSERT
2401       Label L2;
2402       __ get_thread(r14);
2403       __ cmpptr(r15_thread, r14);
2404       __ jcc(Assembler::equal, L2);
2405       __ stop("StubRoutines::call_stub: r15_thread is modified by call");
2406       __ bind(L2);
2407 #endif // ASSERT
2408 
2409     // check that int operands are properly extended to size_t
2410     assert_clean_int(length, rax);
2411     assert_clean_int(ckoff, rax);
2412 
2413 #ifdef ASSERT
2414     BLOCK_COMMENT("assert consistent ckoff/ckval");
2415     // The ckoff and ckval must be mutually consistent,
2416     // even though caller generates both.
2417     { Label L;
2418       int sco_offset = in_bytes(Klass::super_check_offset_offset());
2419       __ cmpl(ckoff, Address(ckval, sco_offset));
2420       __ jcc(Assembler::equal, L);
2421       __ stop("super_check_offset inconsistent");
2422       __ bind(L);
2423     }
2424 #endif //ASSERT
2425 
2426     // Loop-invariant addresses.  They are exclusive end pointers.
2427     Address end_from_addr(from, length, TIMES_OOP, 0);
2428     Address   end_to_addr(to,   length, TIMES_OOP, 0);
2429     // Loop-variant addresses.  They assume post-incremented count < 0.
2430     Address from_element_addr(end_from, count, TIMES_OOP, 0);
2431     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
2432 
2433     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST | ARRAYCOPY_DISJOINT;
2434     if (dest_uninitialized) {
2435       decorators |= IS_DEST_UNINITIALIZED;
2436     }
2437 
2438     BasicType type = T_OBJECT;
2439     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2440     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
2441 
2442     // Copy from low to high addresses, indexed from the end of each array.
2443     __ lea(end_from, end_from_addr);
2444     __ lea(end_to,   end_to_addr);
2445     __ movptr(r14_length, length);        // save a copy of the length
2446     assert(length == count, "");          // else fix next line:
2447     __ negptr(count);                     // negate and test the length
2448     __ jcc(Assembler::notZero, L_load_element);
2449 
2450     // Empty array:  Nothing to do.
2451     __ xorptr(rax, rax);                  // return 0 on (trivial) success
2452     __ jmp(L_done);
2453 
2454     // ======== begin loop ========
2455     // (Loop is rotated; its entry is L_load_element.)
2456     // Loop control:
2457     //   for (count = -count; count != 0; count++)
2458     // Base pointers src, dst are biased by 8*(count-1),to last element.
2459     __ align(OptoLoopAlignment);
2460 
2461     __ BIND(L_store_element);
2462     __ store_heap_oop(to_element_addr, rax_oop, noreg, noreg, AS_RAW);  // store the oop
2463     __ increment(count);               // increment the count toward zero
2464     __ jcc(Assembler::zero, L_do_card_marks);
2465 
2466     // ======== loop entry is here ========
2467     __ BIND(L_load_element);
2468     __ load_heap_oop(rax_oop, from_element_addr, noreg, noreg, AS_RAW); // load the oop
2469     __ testptr(rax_oop, rax_oop);
2470     __ jcc(Assembler::zero, L_store_element);
2471 
2472     __ load_klass(r11_klass, rax_oop);// query the object klass
2473     generate_type_check(r11_klass, ckoff, ckval, L_store_element);
2474     // ======== end loop ========
2475 
2476     // It was a real error; we must depend on the caller to finish the job.
2477     // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
2478     // Emit GC store barriers for the oops we have copied (r14 + rdx),
2479     // and report their number to the caller.
2480     assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1);
2481     Label L_post_barrier;
2482     __ addptr(r14_length, count);     // K = (original - remaining) oops
2483     __ movptr(rax, r14_length);       // save the value
2484     __ notptr(rax);                   // report (-1^K) to caller (does not affect flags)
2485     __ jccb(Assembler::notZero, L_post_barrier);
2486     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
2487 
2488     // Come here on success only.
2489     __ BIND(L_do_card_marks);
2490     __ xorptr(rax, rax);              // return 0 on success
2491 
2492     __ BIND(L_post_barrier);
2493     bs->arraycopy_epilogue(_masm, decorators, type, from, to, r14_length);
2494 
2495     // Common exit point (success or failure).
2496     __ BIND(L_done);
2497     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
2498     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
2499     __ movptr(r10, Address(rsp, saved_r10_offset * wordSize));
2500     restore_arg_regs();
2501     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); // Update counter after rscratch1 is free
2502     __ leave(); // required for proper stackwalking of RuntimeStub frame
2503     __ ret(0);
2504 
2505     return start;
2506   }
2507 
2508   //
2509   //  Generate 'unsafe' array copy stub
2510   //  Though just as safe as the other stubs, it takes an unscaled
2511   //  size_t argument instead of an element count.
2512   //
2513   //  Input:
2514   //    c_rarg0   - source array address
2515   //    c_rarg1   - destination array address
2516   //    c_rarg2   - byte count, treated as ssize_t, can be zero
2517   //
2518   // Examines the alignment of the operands and dispatches
2519   // to a long, int, short, or byte copy loop.
2520   //
2521   address generate_unsafe_copy(const char *name,
2522                                address byte_copy_entry, address short_copy_entry,
2523                                address int_copy_entry, address long_copy_entry) {
2524 
2525     Label L_long_aligned, L_int_aligned, L_short_aligned;
2526 
2527     // Input registers (before setup_arg_regs)
2528     const Register from        = c_rarg0;  // source array address
2529     const Register to          = c_rarg1;  // destination array address
2530     const Register size        = c_rarg2;  // byte count (size_t)
2531 
2532     // Register used as a temp
2533     const Register bits        = rax;      // test copy of low bits
2534 
2535     __ align(CodeEntryAlignment);
2536     StubCodeMark mark(this, "StubRoutines", name);
2537     address start = __ pc();
2538 
2539     __ enter(); // required for proper stackwalking of RuntimeStub frame
2540 
2541     // bump this on entry, not on exit:
2542     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2543 
2544     __ mov(bits, from);
2545     __ orptr(bits, to);
2546     __ orptr(bits, size);
2547 
2548     __ testb(bits, BytesPerLong-1);
2549     __ jccb(Assembler::zero, L_long_aligned);
2550 
2551     __ testb(bits, BytesPerInt-1);
2552     __ jccb(Assembler::zero, L_int_aligned);
2553 
2554     __ testb(bits, BytesPerShort-1);
2555     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2556 
2557     __ BIND(L_short_aligned);
2558     __ shrptr(size, LogBytesPerShort); // size => short_count
2559     __ jump(RuntimeAddress(short_copy_entry));
2560 
2561     __ BIND(L_int_aligned);
2562     __ shrptr(size, LogBytesPerInt); // size => int_count
2563     __ jump(RuntimeAddress(int_copy_entry));
2564 
2565     __ BIND(L_long_aligned);
2566     __ shrptr(size, LogBytesPerLong); // size => qword_count
2567     __ jump(RuntimeAddress(long_copy_entry));
2568 
2569     return start;
2570   }
2571 
2572   // Perform range checks on the proposed arraycopy.
2573   // Kills temp, but nothing else.
2574   // Also, clean the sign bits of src_pos and dst_pos.
2575   void arraycopy_range_checks(Register src,     // source array oop (c_rarg0)
2576                               Register src_pos, // source position (c_rarg1)
2577                               Register dst,     // destination array oo (c_rarg2)
2578                               Register dst_pos, // destination position (c_rarg3)
2579                               Register length,
2580                               Register temp,
2581                               Label& L_failed) {
2582     BLOCK_COMMENT("arraycopy_range_checks:");
2583 
2584     //  if (src_pos + length > arrayOop(src)->length())  FAIL;
2585     __ movl(temp, length);
2586     __ addl(temp, src_pos);             // src_pos + length
2587     __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
2588     __ jcc(Assembler::above, L_failed);
2589 
2590     //  if (dst_pos + length > arrayOop(dst)->length())  FAIL;
2591     __ movl(temp, length);
2592     __ addl(temp, dst_pos);             // dst_pos + length
2593     __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2594     __ jcc(Assembler::above, L_failed);
2595 
2596     // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
2597     // Move with sign extension can be used since they are positive.
2598     __ movslq(src_pos, src_pos);
2599     __ movslq(dst_pos, dst_pos);
2600 
2601     BLOCK_COMMENT("arraycopy_range_checks done");
2602   }
2603 
2604   //
2605   //  Generate generic array copy stubs
2606   //
2607   //  Input:
2608   //    c_rarg0    -  src oop
2609   //    c_rarg1    -  src_pos (32-bits)
2610   //    c_rarg2    -  dst oop
2611   //    c_rarg3    -  dst_pos (32-bits)
2612   // not Win64
2613   //    c_rarg4    -  element count (32-bits)
2614   // Win64
2615   //    rsp+40     -  element count (32-bits)
2616   //
2617   //  Output:
2618   //    rax ==  0  -  success
2619   //    rax == -1^K - failure, where K is partial transfer count
2620   //
2621   address generate_generic_copy(const char *name,
2622                                 address byte_copy_entry, address short_copy_entry,
2623                                 address int_copy_entry, address oop_copy_entry,
2624                                 address long_copy_entry, address checkcast_copy_entry) {
2625 
2626     Label L_failed, L_failed_0, L_objArray;
2627     Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
2628 
2629     // Input registers
2630     const Register src        = c_rarg0;  // source array oop
2631     const Register src_pos    = c_rarg1;  // source position
2632     const Register dst        = c_rarg2;  // destination array oop
2633     const Register dst_pos    = c_rarg3;  // destination position
2634 #ifndef _WIN64
2635     const Register length     = c_rarg4;
2636 #else
2637     const Address  length(rsp, 6 * wordSize);  // elements count is on stack on Win64
2638 #endif
2639 
2640     { int modulus = CodeEntryAlignment;
2641       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
2642       int advance = target - (__ offset() % modulus);
2643       if (advance < 0)  advance += modulus;
2644       if (advance > 0)  __ nop(advance);
2645     }
2646     StubCodeMark mark(this, "StubRoutines", name);
2647 
2648     // Short-hop target to L_failed.  Makes for denser prologue code.
2649     __ BIND(L_failed_0);
2650     __ jmp(L_failed);
2651     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
2652 
2653     __ align(CodeEntryAlignment);
2654     address start = __ pc();
2655 
2656     __ enter(); // required for proper stackwalking of RuntimeStub frame
2657 
2658     // bump this on entry, not on exit:
2659     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
2660 
2661     //-----------------------------------------------------------------------
2662     // Assembler stub will be used for this call to arraycopy
2663     // if the following conditions are met:
2664     //
2665     // (1) src and dst must not be null.
2666     // (2) src_pos must not be negative.
2667     // (3) dst_pos must not be negative.
2668     // (4) length  must not be negative.
2669     // (5) src klass and dst klass should be the same and not NULL.
2670     // (6) src and dst should be arrays.
2671     // (7) src_pos + length must not exceed length of src.
2672     // (8) dst_pos + length must not exceed length of dst.
2673     //
2674 
2675     //  if (src == NULL) return -1;
2676     __ testptr(src, src);         // src oop
2677     size_t j1off = __ offset();
2678     __ jccb(Assembler::zero, L_failed_0);
2679 
2680     //  if (src_pos < 0) return -1;
2681     __ testl(src_pos, src_pos); // src_pos (32-bits)
2682     __ jccb(Assembler::negative, L_failed_0);
2683 
2684     //  if (dst == NULL) return -1;
2685     __ testptr(dst, dst);         // dst oop
2686     __ jccb(Assembler::zero, L_failed_0);
2687 
2688     //  if (dst_pos < 0) return -1;
2689     __ testl(dst_pos, dst_pos); // dst_pos (32-bits)
2690     size_t j4off = __ offset();
2691     __ jccb(Assembler::negative, L_failed_0);
2692 
2693     // The first four tests are very dense code,
2694     // but not quite dense enough to put four
2695     // jumps in a 16-byte instruction fetch buffer.
2696     // That's good, because some branch predicters
2697     // do not like jumps so close together.
2698     // Make sure of this.
2699     guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
2700 
2701     // registers used as temp
2702     const Register r11_length    = r11; // elements count to copy
2703     const Register r10_src_klass = r10; // array klass
2704 
2705     //  if (length < 0) return -1;
2706     __ movl(r11_length, length);        // length (elements count, 32-bits value)
2707     __ testl(r11_length, r11_length);
2708     __ jccb(Assembler::negative, L_failed_0);
2709 
2710     __ load_klass(r10_src_klass, src);
2711 #ifdef ASSERT
2712     //  assert(src->klass() != NULL);
2713     {
2714       BLOCK_COMMENT("assert klasses not null {");
2715       Label L1, L2;
2716       __ testptr(r10_src_klass, r10_src_klass);
2717       __ jcc(Assembler::notZero, L2);   // it is broken if klass is NULL
2718       __ bind(L1);
2719       __ stop("broken null klass");
2720       __ bind(L2);
2721       __ load_klass(rax, dst);
2722       __ cmpq(rax, 0);
2723       __ jcc(Assembler::equal, L1);     // this would be broken also
2724       BLOCK_COMMENT("} assert klasses not null done");
2725     }
2726 #endif
2727 
2728     // Load layout helper (32-bits)
2729     //
2730     //  |array_tag|     | header_size | element_type |     |log2_element_size|
2731     // 32        30    24            16              8     2                 0
2732     //
2733     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2734     //
2735 
2736     const int lh_offset = in_bytes(Klass::layout_helper_offset());
2737 
2738     // Handle objArrays completely differently...
2739     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2740     __ cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
2741     __ jcc(Assembler::equal, L_objArray);
2742 
2743     //  if (src->klass() != dst->klass()) return -1;
2744     __ load_klass(rax, dst);
2745     __ cmpq(r10_src_klass, rax);
2746     __ jcc(Assembler::notEqual, L_failed);
2747 
2748     const Register rax_lh = rax;  // layout helper
2749     __ movl(rax_lh, Address(r10_src_klass, lh_offset));
2750 
2751     //  if (!src->is_Array()) return -1;
2752     __ cmpl(rax_lh, Klass::_lh_neutral_value);
2753     __ jcc(Assembler::greaterEqual, L_failed);
2754 
2755     // At this point, it is known to be a typeArray (array_tag 0x3).
2756 #ifdef ASSERT
2757     {
2758       BLOCK_COMMENT("assert primitive array {");
2759       Label L;
2760       __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
2761       __ jcc(Assembler::greaterEqual, L);
2762       __ stop("must be a primitive array");
2763       __ bind(L);
2764       BLOCK_COMMENT("} assert primitive array done");
2765     }
2766 #endif
2767 
2768     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2769                            r10, L_failed);
2770 
2771     // TypeArrayKlass
2772     //
2773     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2774     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2775     //
2776 
2777     const Register r10_offset = r10;    // array offset
2778     const Register rax_elsize = rax_lh; // element size
2779 
2780     __ movl(r10_offset, rax_lh);
2781     __ shrl(r10_offset, Klass::_lh_header_size_shift);
2782     __ andptr(r10_offset, Klass::_lh_header_size_mask);   // array_offset
2783     __ addptr(src, r10_offset);           // src array offset
2784     __ addptr(dst, r10_offset);           // dst array offset
2785     BLOCK_COMMENT("choose copy loop based on element size");
2786     __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
2787 
2788     // next registers should be set before the jump to corresponding stub
2789     const Register from     = c_rarg0;  // source array address
2790     const Register to       = c_rarg1;  // destination array address
2791     const Register count    = c_rarg2;  // elements count
2792 
2793     // 'from', 'to', 'count' registers should be set in such order
2794     // since they are the same as 'src', 'src_pos', 'dst'.
2795 
2796   __ BIND(L_copy_bytes);
2797     __ cmpl(rax_elsize, 0);
2798     __ jccb(Assembler::notEqual, L_copy_shorts);
2799     __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
2800     __ lea(to,   Address(dst, dst_pos, Address::times_1, 0));// dst_addr
2801     __ movl2ptr(count, r11_length); // length
2802     __ jump(RuntimeAddress(byte_copy_entry));
2803 
2804   __ BIND(L_copy_shorts);
2805     __ cmpl(rax_elsize, LogBytesPerShort);
2806     __ jccb(Assembler::notEqual, L_copy_ints);
2807     __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
2808     __ lea(to,   Address(dst, dst_pos, Address::times_2, 0));// dst_addr
2809     __ movl2ptr(count, r11_length); // length
2810     __ jump(RuntimeAddress(short_copy_entry));
2811 
2812   __ BIND(L_copy_ints);
2813     __ cmpl(rax_elsize, LogBytesPerInt);
2814     __ jccb(Assembler::notEqual, L_copy_longs);
2815     __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
2816     __ lea(to,   Address(dst, dst_pos, Address::times_4, 0));// dst_addr
2817     __ movl2ptr(count, r11_length); // length
2818     __ jump(RuntimeAddress(int_copy_entry));
2819 
2820   __ BIND(L_copy_longs);
2821 #ifdef ASSERT
2822     {
2823       BLOCK_COMMENT("assert long copy {");
2824       Label L;
2825       __ cmpl(rax_elsize, LogBytesPerLong);
2826       __ jcc(Assembler::equal, L);
2827       __ stop("must be long copy, but elsize is wrong");
2828       __ bind(L);
2829       BLOCK_COMMENT("} assert long copy done");
2830     }
2831 #endif
2832     __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
2833     __ lea(to,   Address(dst, dst_pos, Address::times_8, 0));// dst_addr
2834     __ movl2ptr(count, r11_length); // length
2835     __ jump(RuntimeAddress(long_copy_entry));
2836 
2837     // ObjArrayKlass
2838   __ BIND(L_objArray);
2839     // live at this point:  r10_src_klass, r11_length, src[_pos], dst[_pos]
2840 
2841     Label L_plain_copy, L_checkcast_copy;
2842     //  test array classes for subtyping
2843     __ load_klass(rax, dst);
2844     __ cmpq(r10_src_klass, rax); // usual case is exact equality
2845     __ jcc(Assembler::notEqual, L_checkcast_copy);
2846 
2847     // Identically typed arrays can be copied without element-wise checks.
2848     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2849                            r10, L_failed);
2850 
2851     __ lea(from, Address(src, src_pos, TIMES_OOP,
2852                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
2853     __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2854                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
2855     __ movl2ptr(count, r11_length); // length
2856   __ BIND(L_plain_copy);
2857     __ jump(RuntimeAddress(oop_copy_entry));
2858 
2859   __ BIND(L_checkcast_copy);
2860     // live at this point:  r10_src_klass, r11_length, rax (dst_klass)
2861     {
2862       // Before looking at dst.length, make sure dst is also an objArray.
2863       __ cmpl(Address(rax, lh_offset), objArray_lh);
2864       __ jcc(Assembler::notEqual, L_failed);
2865 
2866       // It is safe to examine both src.length and dst.length.
2867       arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2868                              rax, L_failed);
2869 
2870       const Register r11_dst_klass = r11;
2871       __ load_klass(r11_dst_klass, dst); // reload
2872 
2873       // Marshal the base address arguments now, freeing registers.
2874       __ lea(from, Address(src, src_pos, TIMES_OOP,
2875                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2876       __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2877                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2878       __ movl(count, length);           // length (reloaded)
2879       Register sco_temp = c_rarg3;      // this register is free now
2880       assert_different_registers(from, to, count, sco_temp,
2881                                  r11_dst_klass, r10_src_klass);
2882       assert_clean_int(count, sco_temp);
2883 
2884       // Generate the type check.
2885       const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2886       __ movl(sco_temp, Address(r11_dst_klass, sco_offset));
2887       assert_clean_int(sco_temp, rax);
2888       generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
2889 
2890       // Fetch destination element klass from the ObjArrayKlass header.
2891       int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2892       __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
2893       __ movl(  sco_temp,      Address(r11_dst_klass, sco_offset));
2894       assert_clean_int(sco_temp, rax);
2895 
2896       // the checkcast_copy loop needs two extra arguments:
2897       assert(c_rarg3 == sco_temp, "#3 already in place");
2898       // Set up arguments for checkcast_copy_entry.
2899       setup_arg_regs(4);
2900       __ movptr(r8, r11_dst_klass);  // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
2901       __ jump(RuntimeAddress(checkcast_copy_entry));
2902     }
2903 
2904   __ BIND(L_failed);
2905     __ xorptr(rax, rax);
2906     __ notptr(rax); // return -1
2907     __ leave();   // required for proper stackwalking of RuntimeStub frame
2908     __ ret(0);
2909 
2910     return start;
2911   }
2912 
2913   void generate_arraycopy_stubs() {
2914     address entry;
2915     address entry_jbyte_arraycopy;
2916     address entry_jshort_arraycopy;
2917     address entry_jint_arraycopy;
2918     address entry_oop_arraycopy;
2919     address entry_jlong_arraycopy;
2920     address entry_checkcast_arraycopy;
2921 
2922     StubRoutines::_jbyte_disjoint_arraycopy  = generate_disjoint_byte_copy(false, &entry,
2923                                                                            "jbyte_disjoint_arraycopy");
2924     StubRoutines::_jbyte_arraycopy           = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy,
2925                                                                            "jbyte_arraycopy");
2926 
2927     StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
2928                                                                             "jshort_disjoint_arraycopy");
2929     StubRoutines::_jshort_arraycopy          = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy,
2930                                                                             "jshort_arraycopy");
2931 
2932     StubRoutines::_jint_disjoint_arraycopy   = generate_disjoint_int_oop_copy(false, false, &entry,
2933                                                                               "jint_disjoint_arraycopy");
2934     StubRoutines::_jint_arraycopy            = generate_conjoint_int_oop_copy(false, false, entry,
2935                                                                               &entry_jint_arraycopy, "jint_arraycopy");
2936 
2937     StubRoutines::_jlong_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, false, &entry,
2938                                                                                "jlong_disjoint_arraycopy");
2939     StubRoutines::_jlong_arraycopy           = generate_conjoint_long_oop_copy(false, false, entry,
2940                                                                                &entry_jlong_arraycopy, "jlong_arraycopy");
2941 
2942 
2943     if (UseCompressedOops) {
2944       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_int_oop_copy(false, true, &entry,
2945                                                                               "oop_disjoint_arraycopy");
2946       StubRoutines::_oop_arraycopy           = generate_conjoint_int_oop_copy(false, true, entry,
2947                                                                               &entry_oop_arraycopy, "oop_arraycopy");
2948       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_int_oop_copy(false, true, &entry,
2949                                                                                      "oop_disjoint_arraycopy_uninit",
2950                                                                                      /*dest_uninitialized*/true);
2951       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_int_oop_copy(false, true, entry,
2952                                                                                      NULL, "oop_arraycopy_uninit",
2953                                                                                      /*dest_uninitialized*/true);
2954     } else {
2955       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, true, &entry,
2956                                                                                "oop_disjoint_arraycopy");
2957       StubRoutines::_oop_arraycopy           = generate_conjoint_long_oop_copy(false, true, entry,
2958                                                                                &entry_oop_arraycopy, "oop_arraycopy");
2959       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_long_oop_copy(false, true, &entry,
2960                                                                                       "oop_disjoint_arraycopy_uninit",
2961                                                                                       /*dest_uninitialized*/true);
2962       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_long_oop_copy(false, true, entry,
2963                                                                                       NULL, "oop_arraycopy_uninit",
2964                                                                                       /*dest_uninitialized*/true);
2965     }
2966 
2967     StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2968     StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL,
2969                                                                         /*dest_uninitialized*/true);
2970 
2971     StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy("unsafe_arraycopy",
2972                                                               entry_jbyte_arraycopy,
2973                                                               entry_jshort_arraycopy,
2974                                                               entry_jint_arraycopy,
2975                                                               entry_jlong_arraycopy);
2976     StubRoutines::_generic_arraycopy   = generate_generic_copy("generic_arraycopy",
2977                                                                entry_jbyte_arraycopy,
2978                                                                entry_jshort_arraycopy,
2979                                                                entry_jint_arraycopy,
2980                                                                entry_oop_arraycopy,
2981                                                                entry_jlong_arraycopy,
2982                                                                entry_checkcast_arraycopy);
2983 
2984     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2985     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2986     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2987     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2988     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2989     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2990 
2991     // We don't generate specialized code for HeapWord-aligned source
2992     // arrays, so just use the code we've already generated
2993     StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = StubRoutines::_jbyte_disjoint_arraycopy;
2994     StubRoutines::_arrayof_jbyte_arraycopy           = StubRoutines::_jbyte_arraycopy;
2995 
2996     StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
2997     StubRoutines::_arrayof_jshort_arraycopy          = StubRoutines::_jshort_arraycopy;
2998 
2999     StubRoutines::_arrayof_jint_disjoint_arraycopy   = StubRoutines::_jint_disjoint_arraycopy;
3000     StubRoutines::_arrayof_jint_arraycopy            = StubRoutines::_jint_arraycopy;
3001 
3002     StubRoutines::_arrayof_jlong_disjoint_arraycopy  = StubRoutines::_jlong_disjoint_arraycopy;
3003     StubRoutines::_arrayof_jlong_arraycopy           = StubRoutines::_jlong_arraycopy;
3004 
3005     StubRoutines::_arrayof_oop_disjoint_arraycopy    = StubRoutines::_oop_disjoint_arraycopy;
3006     StubRoutines::_arrayof_oop_arraycopy             = StubRoutines::_oop_arraycopy;
3007 
3008     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit    = StubRoutines::_oop_disjoint_arraycopy_uninit;
3009     StubRoutines::_arrayof_oop_arraycopy_uninit             = StubRoutines::_oop_arraycopy_uninit;
3010   }
3011 
3012   // AES intrinsic stubs
3013   enum {AESBlockSize = 16};
3014 
3015   address generate_key_shuffle_mask() {
3016     __ align(16);
3017     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
3018     address start = __ pc();
3019     __ emit_data64( 0x0405060700010203, relocInfo::none );
3020     __ emit_data64( 0x0c0d0e0f08090a0b, relocInfo::none );
3021     return start;
3022   }
3023 
3024   address generate_counter_shuffle_mask() {
3025     __ align(16);
3026     StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask");
3027     address start = __ pc();
3028     __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
3029     __ emit_data64(0x0001020304050607, relocInfo::none);
3030     return start;
3031   }
3032 
3033   // Utility routine for loading a 128-bit key word in little endian format
3034   // can optionally specify that the shuffle mask is already in an xmmregister
3035   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
3036     __ movdqu(xmmdst, Address(key, offset));
3037     if (xmm_shuf_mask != NULL) {
3038       __ pshufb(xmmdst, xmm_shuf_mask);
3039     } else {
3040       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3041     }
3042   }
3043 
3044   // Utility routine for increase 128bit counter (iv in CTR mode)
3045   void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) {
3046     __ pextrq(reg, xmmdst, 0x0);
3047     __ addq(reg, inc_delta);
3048     __ pinsrq(xmmdst, reg, 0x0);
3049     __ jcc(Assembler::carryClear, next_block); // jump if no carry
3050     __ pextrq(reg, xmmdst, 0x01); // Carry
3051     __ addq(reg, 0x01);
3052     __ pinsrq(xmmdst, reg, 0x01); //Carry end
3053     __ BIND(next_block);          // next instruction
3054   }
3055 
3056   // Arguments:
3057   //
3058   // Inputs:
3059   //   c_rarg0   - source byte array address
3060   //   c_rarg1   - destination byte array address
3061   //   c_rarg2   - K (key) in little endian int array
3062   //
3063   address generate_aescrypt_encryptBlock() {
3064     assert(UseAES, "need AES instructions and misaligned SSE support");
3065     __ align(CodeEntryAlignment);
3066     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
3067     Label L_doLast;
3068     address start = __ pc();
3069 
3070     const Register from        = c_rarg0;  // source array address
3071     const Register to          = c_rarg1;  // destination array address
3072     const Register key         = c_rarg2;  // key array address
3073     const Register keylen      = rax;
3074 
3075     const XMMRegister xmm_result = xmm0;
3076     const XMMRegister xmm_key_shuf_mask = xmm1;
3077     // On win64 xmm6-xmm15 must be preserved so don't use them.
3078     const XMMRegister xmm_temp1  = xmm2;
3079     const XMMRegister xmm_temp2  = xmm3;
3080     const XMMRegister xmm_temp3  = xmm4;
3081     const XMMRegister xmm_temp4  = xmm5;
3082 
3083     __ enter(); // required for proper stackwalking of RuntimeStub frame
3084 
3085     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3086     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3087 
3088     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3089     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
3090 
3091     // For encryption, the java expanded key ordering is just what we need
3092     // we don't know if the key is aligned, hence not using load-execute form
3093 
3094     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
3095     __ pxor(xmm_result, xmm_temp1);
3096 
3097     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3098     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3099     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3100     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3101 
3102     __ aesenc(xmm_result, xmm_temp1);
3103     __ aesenc(xmm_result, xmm_temp2);
3104     __ aesenc(xmm_result, xmm_temp3);
3105     __ aesenc(xmm_result, xmm_temp4);
3106 
3107     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3108     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3109     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3110     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3111 
3112     __ aesenc(xmm_result, xmm_temp1);
3113     __ aesenc(xmm_result, xmm_temp2);
3114     __ aesenc(xmm_result, xmm_temp3);
3115     __ aesenc(xmm_result, xmm_temp4);
3116 
3117     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3118     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3119 
3120     __ cmpl(keylen, 44);
3121     __ jccb(Assembler::equal, L_doLast);
3122 
3123     __ aesenc(xmm_result, xmm_temp1);
3124     __ aesenc(xmm_result, xmm_temp2);
3125 
3126     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3127     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3128 
3129     __ cmpl(keylen, 52);
3130     __ jccb(Assembler::equal, L_doLast);
3131 
3132     __ aesenc(xmm_result, xmm_temp1);
3133     __ aesenc(xmm_result, xmm_temp2);
3134 
3135     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3136     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3137 
3138     __ BIND(L_doLast);
3139     __ aesenc(xmm_result, xmm_temp1);
3140     __ aesenclast(xmm_result, xmm_temp2);
3141     __ movdqu(Address(to, 0), xmm_result);        // store the result
3142     __ xorptr(rax, rax); // return 0
3143     __ leave(); // required for proper stackwalking of RuntimeStub frame
3144     __ ret(0);
3145 
3146     return start;
3147   }
3148 
3149 
3150   // Arguments:
3151   //
3152   // Inputs:
3153   //   c_rarg0   - source byte array address
3154   //   c_rarg1   - destination byte array address
3155   //   c_rarg2   - K (key) in little endian int array
3156   //
3157   address generate_aescrypt_decryptBlock() {
3158     assert(UseAES, "need AES instructions and misaligned SSE support");
3159     __ align(CodeEntryAlignment);
3160     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
3161     Label L_doLast;
3162     address start = __ pc();
3163 
3164     const Register from        = c_rarg0;  // source array address
3165     const Register to          = c_rarg1;  // destination array address
3166     const Register key         = c_rarg2;  // key array address
3167     const Register keylen      = rax;
3168 
3169     const XMMRegister xmm_result = xmm0;
3170     const XMMRegister xmm_key_shuf_mask = xmm1;
3171     // On win64 xmm6-xmm15 must be preserved so don't use them.
3172     const XMMRegister xmm_temp1  = xmm2;
3173     const XMMRegister xmm_temp2  = xmm3;
3174     const XMMRegister xmm_temp3  = xmm4;
3175     const XMMRegister xmm_temp4  = xmm5;
3176 
3177     __ enter(); // required for proper stackwalking of RuntimeStub frame
3178 
3179     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3180     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3181 
3182     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3183     __ movdqu(xmm_result, Address(from, 0));
3184 
3185     // for decryption java expanded key ordering is rotated one position from what we want
3186     // so we start from 0x10 here and hit 0x00 last
3187     // we don't know if the key is aligned, hence not using load-execute form
3188     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3189     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3190     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3191     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3192 
3193     __ pxor  (xmm_result, xmm_temp1);
3194     __ aesdec(xmm_result, xmm_temp2);
3195     __ aesdec(xmm_result, xmm_temp3);
3196     __ aesdec(xmm_result, xmm_temp4);
3197 
3198     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3199     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3200     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3201     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3202 
3203     __ aesdec(xmm_result, xmm_temp1);
3204     __ aesdec(xmm_result, xmm_temp2);
3205     __ aesdec(xmm_result, xmm_temp3);
3206     __ aesdec(xmm_result, xmm_temp4);
3207 
3208     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3209     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3210     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
3211 
3212     __ cmpl(keylen, 44);
3213     __ jccb(Assembler::equal, L_doLast);
3214 
3215     __ aesdec(xmm_result, xmm_temp1);
3216     __ aesdec(xmm_result, xmm_temp2);
3217 
3218     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3219     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3220 
3221     __ cmpl(keylen, 52);
3222     __ jccb(Assembler::equal, L_doLast);
3223 
3224     __ aesdec(xmm_result, xmm_temp1);
3225     __ aesdec(xmm_result, xmm_temp2);
3226 
3227     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3228     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3229 
3230     __ BIND(L_doLast);
3231     __ aesdec(xmm_result, xmm_temp1);
3232     __ aesdec(xmm_result, xmm_temp2);
3233 
3234     // for decryption the aesdeclast operation is always on key+0x00
3235     __ aesdeclast(xmm_result, xmm_temp3);
3236     __ movdqu(Address(to, 0), xmm_result);  // store the result
3237     __ xorptr(rax, rax); // return 0
3238     __ leave(); // required for proper stackwalking of RuntimeStub frame
3239     __ ret(0);
3240 
3241     return start;
3242   }
3243 
3244 
3245   // Arguments:
3246   //
3247   // Inputs:
3248   //   c_rarg0   - source byte array address
3249   //   c_rarg1   - destination byte array address
3250   //   c_rarg2   - K (key) in little endian int array
3251   //   c_rarg3   - r vector byte array address
3252   //   c_rarg4   - input length
3253   //
3254   // Output:
3255   //   rax       - input length
3256   //
3257   address generate_cipherBlockChaining_encryptAESCrypt() {
3258     assert(UseAES, "need AES instructions and misaligned SSE support");
3259     __ align(CodeEntryAlignment);
3260     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
3261     address start = __ pc();
3262 
3263     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
3264     const Register from        = c_rarg0;  // source array address
3265     const Register to          = c_rarg1;  // destination array address
3266     const Register key         = c_rarg2;  // key array address
3267     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
3268                                            // and left with the results of the last encryption block
3269 #ifndef _WIN64
3270     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
3271 #else
3272     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
3273     const Register len_reg     = r11;      // pick the volatile windows register
3274 #endif
3275     const Register pos         = rax;
3276 
3277     // xmm register assignments for the loops below
3278     const XMMRegister xmm_result = xmm0;
3279     const XMMRegister xmm_temp   = xmm1;
3280     // keys 0-10 preloaded into xmm2-xmm12
3281     const int XMM_REG_NUM_KEY_FIRST = 2;
3282     const int XMM_REG_NUM_KEY_LAST  = 15;
3283     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3284     const XMMRegister xmm_key10  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+10);
3285     const XMMRegister xmm_key11  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+11);
3286     const XMMRegister xmm_key12  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+12);
3287     const XMMRegister xmm_key13  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+13);
3288 
3289     __ enter(); // required for proper stackwalking of RuntimeStub frame
3290 
3291 #ifdef _WIN64
3292     // on win64, fill len_reg from stack position
3293     __ movl(len_reg, len_mem);
3294 #else
3295     __ push(len_reg); // Save
3296 #endif
3297 
3298     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
3299     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3300     // load up xmm regs xmm2 thru xmm12 with key 0x00 - 0xa0
3301     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_FIRST+10; rnum++) {
3302       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3303       offset += 0x10;
3304     }
3305     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
3306 
3307     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3308     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3309     __ cmpl(rax, 44);
3310     __ jcc(Assembler::notEqual, L_key_192_256);
3311 
3312     // 128 bit code follows here
3313     __ movptr(pos, 0);
3314     __ align(OptoLoopAlignment);
3315 
3316     __ BIND(L_loopTop_128);
3317     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3318     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3319     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3320     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 9; rnum++) {
3321       __ aesenc(xmm_result, as_XMMRegister(rnum));
3322     }
3323     __ aesenclast(xmm_result, xmm_key10);
3324     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3325     // no need to store r to memory until we exit
3326     __ addptr(pos, AESBlockSize);
3327     __ subptr(len_reg, AESBlockSize);
3328     __ jcc(Assembler::notEqual, L_loopTop_128);
3329 
3330     __ BIND(L_exit);
3331     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
3332 
3333 #ifdef _WIN64
3334     __ movl(rax, len_mem);
3335 #else
3336     __ pop(rax); // return length
3337 #endif
3338     __ leave(); // required for proper stackwalking of RuntimeStub frame
3339     __ ret(0);
3340 
3341     __ BIND(L_key_192_256);
3342     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
3343     load_key(xmm_key11, key, 0xb0, xmm_key_shuf_mask);
3344     load_key(xmm_key12, key, 0xc0, xmm_key_shuf_mask);
3345     __ cmpl(rax, 52);
3346     __ jcc(Assembler::notEqual, L_key_256);
3347 
3348     // 192-bit code follows here (could be changed to use more xmm registers)
3349     __ movptr(pos, 0);
3350     __ align(OptoLoopAlignment);
3351 
3352     __ BIND(L_loopTop_192);
3353     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3354     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3355     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3356     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 11; rnum++) {
3357       __ aesenc(xmm_result, as_XMMRegister(rnum));
3358     }
3359     __ aesenclast(xmm_result, xmm_key12);
3360     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3361     // no need to store r to memory until we exit
3362     __ addptr(pos, AESBlockSize);
3363     __ subptr(len_reg, AESBlockSize);
3364     __ jcc(Assembler::notEqual, L_loopTop_192);
3365     __ jmp(L_exit);
3366 
3367     __ BIND(L_key_256);
3368     // 256-bit code follows here (could be changed to use more xmm registers)
3369     load_key(xmm_key13, key, 0xd0, xmm_key_shuf_mask);
3370     __ movptr(pos, 0);
3371     __ align(OptoLoopAlignment);
3372 
3373     __ BIND(L_loopTop_256);
3374     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3375     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3376     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3377     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 13; rnum++) {
3378       __ aesenc(xmm_result, as_XMMRegister(rnum));
3379     }
3380     load_key(xmm_temp, key, 0xe0);
3381     __ aesenclast(xmm_result, xmm_temp);
3382     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3383     // no need to store r to memory until we exit
3384     __ addptr(pos, AESBlockSize);
3385     __ subptr(len_reg, AESBlockSize);
3386     __ jcc(Assembler::notEqual, L_loopTop_256);
3387     __ jmp(L_exit);
3388 
3389     return start;
3390   }
3391 
3392   // Safefetch stubs.
3393   void generate_safefetch(const char* name, int size, address* entry,
3394                           address* fault_pc, address* continuation_pc) {
3395     // safefetch signatures:
3396     //   int      SafeFetch32(int*      adr, int      errValue);
3397     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3398     //
3399     // arguments:
3400     //   c_rarg0 = adr
3401     //   c_rarg1 = errValue
3402     //
3403     // result:
3404     //   PPC_RET  = *adr or errValue
3405 
3406     StubCodeMark mark(this, "StubRoutines", name);
3407 
3408     // Entry point, pc or function descriptor.
3409     *entry = __ pc();
3410 
3411     // Load *adr into c_rarg1, may fault.
3412     *fault_pc = __ pc();
3413     switch (size) {
3414       case 4:
3415         // int32_t
3416         __ movl(c_rarg1, Address(c_rarg0, 0));
3417         break;
3418       case 8:
3419         // int64_t
3420         __ movq(c_rarg1, Address(c_rarg0, 0));
3421         break;
3422       default:
3423         ShouldNotReachHere();
3424     }
3425 
3426     // return errValue or *adr
3427     *continuation_pc = __ pc();
3428     __ movq(rax, c_rarg1);
3429     __ ret(0);
3430   }
3431 
3432   // This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time
3433   // to hide instruction latency
3434   //
3435   // Arguments:
3436   //
3437   // Inputs:
3438   //   c_rarg0   - source byte array address
3439   //   c_rarg1   - destination byte array address
3440   //   c_rarg2   - K (key) in little endian int array
3441   //   c_rarg3   - r vector byte array address
3442   //   c_rarg4   - input length
3443   //
3444   // Output:
3445   //   rax       - input length
3446   //
3447   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
3448     assert(UseAES, "need AES instructions and misaligned SSE support");
3449     __ align(CodeEntryAlignment);
3450     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
3451     address start = __ pc();
3452 
3453     const Register from        = c_rarg0;  // source array address
3454     const Register to          = c_rarg1;  // destination array address
3455     const Register key         = c_rarg2;  // key array address
3456     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
3457                                            // and left with the results of the last encryption block
3458 #ifndef _WIN64
3459     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
3460 #else
3461     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
3462     const Register len_reg     = r11;      // pick the volatile windows register
3463 #endif
3464     const Register pos         = rax;
3465 
3466     const int PARALLEL_FACTOR = 4;
3467     const int ROUNDS[3] = { 10, 12, 14 }; // aes rounds for key128, key192, key256
3468 
3469     Label L_exit;
3470     Label L_singleBlock_loopTopHead[3]; // 128, 192, 256
3471     Label L_singleBlock_loopTopHead2[3]; // 128, 192, 256
3472     Label L_singleBlock_loopTop[3]; // 128, 192, 256
3473     Label L_multiBlock_loopTopHead[3]; // 128, 192, 256
3474     Label L_multiBlock_loopTop[3]; // 128, 192, 256
3475 
3476     // keys 0-10 preloaded into xmm5-xmm15
3477     const int XMM_REG_NUM_KEY_FIRST = 5;
3478     const int XMM_REG_NUM_KEY_LAST  = 15;
3479     const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3480     const XMMRegister xmm_key_last  = as_XMMRegister(XMM_REG_NUM_KEY_LAST);
3481 
3482     __ enter(); // required for proper stackwalking of RuntimeStub frame
3483 
3484 #ifdef _WIN64
3485     // on win64, fill len_reg from stack position
3486     __ movl(len_reg, len_mem);
3487 #else
3488     __ push(len_reg); // Save
3489 #endif
3490     __ push(rbx);
3491     // the java expanded key ordering is rotated one position from what we want
3492     // so we start from 0x10 here and hit 0x00 last
3493     const XMMRegister xmm_key_shuf_mask = xmm1;  // used temporarily to swap key bytes up front
3494     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3495     // load up xmm regs 5 thru 15 with key 0x10 - 0xa0 - 0x00
3496     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x10; rnum < XMM_REG_NUM_KEY_LAST; rnum++) {
3497       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3498       offset += 0x10;
3499     }
3500     load_key(xmm_key_last, key, 0x00, xmm_key_shuf_mask);
3501 
3502     const XMMRegister xmm_prev_block_cipher = xmm1;  // holds cipher of previous block
3503 
3504     // registers holding the four results in the parallelized loop
3505     const XMMRegister xmm_result0 = xmm0;
3506     const XMMRegister xmm_result1 = xmm2;
3507     const XMMRegister xmm_result2 = xmm3;
3508     const XMMRegister xmm_result3 = xmm4;
3509 
3510     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));   // initialize with initial rvec
3511 
3512     __ xorptr(pos, pos);
3513 
3514     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3515     __ movl(rbx, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3516     __ cmpl(rbx, 52);
3517     __ jcc(Assembler::equal, L_multiBlock_loopTopHead[1]);
3518     __ cmpl(rbx, 60);
3519     __ jcc(Assembler::equal, L_multiBlock_loopTopHead[2]);
3520 
3521 #define DoFour(opc, src_reg)           \
3522   __ opc(xmm_result0, src_reg);         \
3523   __ opc(xmm_result1, src_reg);         \
3524   __ opc(xmm_result2, src_reg);         \
3525   __ opc(xmm_result3, src_reg);         \
3526 
3527     for (int k = 0; k < 3; ++k) {
3528       __ BIND(L_multiBlock_loopTopHead[k]);
3529       if (k != 0) {
3530         __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
3531         __ jcc(Assembler::less, L_singleBlock_loopTopHead2[k]);
3532       }
3533       if (k == 1) {
3534         __ subptr(rsp, 6 * wordSize);
3535         __ movdqu(Address(rsp, 0), xmm15); //save last_key from xmm15
3536         load_key(xmm15, key, 0xb0); // 0xb0; 192-bit key goes up to 0xc0
3537         __ movdqu(Address(rsp, 2 * wordSize), xmm15);
3538         load_key(xmm1, key, 0xc0);  // 0xc0;
3539         __ movdqu(Address(rsp, 4 * wordSize), xmm1);
3540       } else if (k == 2) {
3541         __ subptr(rsp, 10 * wordSize);
3542         __ movdqu(Address(rsp, 0), xmm15); //save last_key from xmm15
3543         load_key(xmm15, key, 0xd0); // 0xd0; 256-bit key goes upto 0xe0
3544         __ movdqu(Address(rsp, 6 * wordSize), xmm15);
3545         load_key(xmm1, key, 0xe0);  // 0xe0;
3546         __ movdqu(Address(rsp, 8 * wordSize), xmm1);
3547         load_key(xmm15, key, 0xb0); // 0xb0;
3548         __ movdqu(Address(rsp, 2 * wordSize), xmm15);
3549         load_key(xmm1, key, 0xc0);  // 0xc0;
3550         __ movdqu(Address(rsp, 4 * wordSize), xmm1);
3551       }
3552       __ align(OptoLoopAlignment);
3553       __ BIND(L_multiBlock_loopTop[k]);
3554       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
3555       __ jcc(Assembler::less, L_singleBlock_loopTopHead[k]);
3556 
3557       if  (k != 0) {
3558         __ movdqu(xmm15, Address(rsp, 2 * wordSize));
3559         __ movdqu(xmm1, Address(rsp, 4 * wordSize));
3560       }
3561 
3562       __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers
3563       __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
3564       __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
3565       __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
3566 
3567       DoFour(pxor, xmm_key_first);
3568       if (k == 0) {
3569         for (int rnum = 1; rnum < ROUNDS[k]; rnum++) {
3570           DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3571         }
3572         DoFour(aesdeclast, xmm_key_last);
3573       } else if (k == 1) {
3574         for (int rnum = 1; rnum <= ROUNDS[k]-2; rnum++) {
3575           DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3576         }
3577         __ movdqu(xmm_key_last, Address(rsp, 0)); // xmm15 needs to be loaded again.
3578         DoFour(aesdec, xmm1);  // key : 0xc0
3579         __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));  // xmm1 needs to be loaded again
3580         DoFour(aesdeclast, xmm_key_last);
3581       } else if (k == 2) {
3582         for (int rnum = 1; rnum <= ROUNDS[k] - 4; rnum++) {
3583           DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3584         }
3585         DoFour(aesdec, xmm1);  // key : 0xc0
3586         __ movdqu(xmm15, Address(rsp, 6 * wordSize));
3587         __ movdqu(xmm1, Address(rsp, 8 * wordSize));
3588         DoFour(aesdec, xmm15);  // key : 0xd0
3589         __ movdqu(xmm_key_last, Address(rsp, 0)); // xmm15 needs to be loaded again.
3590         DoFour(aesdec, xmm1);  // key : 0xe0
3591         __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));  // xmm1 needs to be loaded again
3592         DoFour(aesdeclast, xmm_key_last);
3593       }
3594 
3595       // for each result, xor with the r vector of previous cipher block
3596       __ pxor(xmm_result0, xmm_prev_block_cipher);
3597       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize));
3598       __ pxor(xmm_result1, xmm_prev_block_cipher);
3599       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize));
3600       __ pxor(xmm_result2, xmm_prev_block_cipher);
3601       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize));
3602       __ pxor(xmm_result3, xmm_prev_block_cipher);
3603       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize));   // this will carry over to next set of blocks
3604       if (k != 0) {
3605         __ movdqu(Address(rvec, 0x00), xmm_prev_block_cipher);
3606       }
3607 
3608       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);     // store 4 results into the next 64 bytes of output
3609       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
3610       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
3611       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
3612 
3613       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize);
3614       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize);
3615       __ jmp(L_multiBlock_loopTop[k]);
3616 
3617       // registers used in the non-parallelized loops
3618       // xmm register assignments for the loops below
3619       const XMMRegister xmm_result = xmm0;
3620       const XMMRegister xmm_prev_block_cipher_save = xmm2;
3621       const XMMRegister xmm_key11 = xmm3;
3622       const XMMRegister xmm_key12 = xmm4;
3623       const XMMRegister key_tmp = xmm4;
3624 
3625       __ BIND(L_singleBlock_loopTopHead[k]);
3626       if (k == 1) {
3627         __ addptr(rsp, 6 * wordSize);
3628       } else if (k == 2) {
3629         __ addptr(rsp, 10 * wordSize);
3630       }
3631       __ cmpptr(len_reg, 0); // any blocks left??
3632       __ jcc(Assembler::equal, L_exit);
3633       __ BIND(L_singleBlock_loopTopHead2[k]);
3634       if (k == 1) {
3635         load_key(xmm_key11, key, 0xb0); // 0xb0; 192-bit key goes upto 0xc0
3636         load_key(xmm_key12, key, 0xc0); // 0xc0; 192-bit key goes upto 0xc0
3637       }
3638       if (k == 2) {
3639         load_key(xmm_key11, key, 0xb0); // 0xb0; 256-bit key goes upto 0xe0
3640       }
3641       __ align(OptoLoopAlignment);
3642       __ BIND(L_singleBlock_loopTop[k]);
3643       __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
3644       __ movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector
3645       __ pxor(xmm_result, xmm_key_first); // do the aes dec rounds
3646       for (int rnum = 1; rnum <= 9 ; rnum++) {
3647           __ aesdec(xmm_result, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3648       }
3649       if (k == 1) {
3650         __ aesdec(xmm_result, xmm_key11);
3651         __ aesdec(xmm_result, xmm_key12);
3652       }
3653       if (k == 2) {
3654         __ aesdec(xmm_result, xmm_key11);
3655         load_key(key_tmp, key, 0xc0);
3656         __ aesdec(xmm_result, key_tmp);
3657         load_key(key_tmp, key, 0xd0);
3658         __ aesdec(xmm_result, key_tmp);
3659         load_key(key_tmp, key, 0xe0);
3660         __ aesdec(xmm_result, key_tmp);
3661       }
3662 
3663       __ aesdeclast(xmm_result, xmm_key_last); // xmm15 always came from key+0
3664       __ pxor(xmm_result, xmm_prev_block_cipher); // xor with the current r vector
3665       __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3666       // no need to store r to memory until we exit
3667       __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block
3668       __ addptr(pos, AESBlockSize);
3669       __ subptr(len_reg, AESBlockSize);
3670       __ jcc(Assembler::notEqual, L_singleBlock_loopTop[k]);
3671       if (k != 2) {
3672         __ jmp(L_exit);
3673       }
3674     } //for 128/192/256
3675 
3676     __ BIND(L_exit);
3677     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher);     // final value of r stored in rvec of CipherBlockChaining object
3678     __ pop(rbx);
3679 #ifdef _WIN64
3680     __ movl(rax, len_mem);
3681 #else
3682     __ pop(rax); // return length
3683 #endif
3684     __ leave(); // required for proper stackwalking of RuntimeStub frame
3685     __ ret(0);
3686     return start;
3687 }
3688 
3689   address generate_upper_word_mask() {
3690     __ align(64);
3691     StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
3692     address start = __ pc();
3693     __ emit_data64(0x0000000000000000, relocInfo::none);
3694     __ emit_data64(0xFFFFFFFF00000000, relocInfo::none);
3695     return start;
3696   }
3697 
3698   address generate_shuffle_byte_flip_mask() {
3699     __ align(64);
3700     StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask");
3701     address start = __ pc();
3702     __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
3703     __ emit_data64(0x0001020304050607, relocInfo::none);
3704     return start;
3705   }
3706 
3707   // ofs and limit are use for multi-block byte array.
3708   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3709   address generate_sha1_implCompress(bool multi_block, const char *name) {
3710     __ align(CodeEntryAlignment);
3711     StubCodeMark mark(this, "StubRoutines", name);
3712     address start = __ pc();
3713 
3714     Register buf = c_rarg0;
3715     Register state = c_rarg1;
3716     Register ofs = c_rarg2;
3717     Register limit = c_rarg3;
3718 
3719     const XMMRegister abcd = xmm0;
3720     const XMMRegister e0 = xmm1;
3721     const XMMRegister e1 = xmm2;
3722     const XMMRegister msg0 = xmm3;
3723 
3724     const XMMRegister msg1 = xmm4;
3725     const XMMRegister msg2 = xmm5;
3726     const XMMRegister msg3 = xmm6;
3727     const XMMRegister shuf_mask = xmm7;
3728 
3729     __ enter();
3730 
3731     __ subptr(rsp, 4 * wordSize);
3732 
3733     __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask,
3734       buf, state, ofs, limit, rsp, multi_block);
3735 
3736     __ addptr(rsp, 4 * wordSize);
3737 
3738     __ leave();
3739     __ ret(0);
3740     return start;
3741   }
3742 
3743   address generate_pshuffle_byte_flip_mask() {
3744     __ align(64);
3745     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask");
3746     address start = __ pc();
3747     __ emit_data64(0x0405060700010203, relocInfo::none);
3748     __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
3749 
3750     if (VM_Version::supports_avx2()) {
3751       __ emit_data64(0x0405060700010203, relocInfo::none); // second copy
3752       __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
3753       // _SHUF_00BA
3754       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3755       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3756       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3757       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3758       // _SHUF_DC00
3759       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3760       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3761       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3762       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3763     }
3764 
3765     return start;
3766   }
3767 
3768   //Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
3769   address generate_pshuffle_byte_flip_mask_sha512() {
3770     __ align(32);
3771     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask_sha512");
3772     address start = __ pc();
3773     if (VM_Version::supports_avx2()) {
3774       __ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK
3775       __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
3776       __ emit_data64(0x1011121314151617, relocInfo::none);
3777       __ emit_data64(0x18191a1b1c1d1e1f, relocInfo::none);
3778       __ emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO
3779       __ emit_data64(0x0000000000000000, relocInfo::none);
3780       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3781       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3782     }
3783 
3784     return start;
3785   }
3786 
3787 // ofs and limit are use for multi-block byte array.
3788 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3789   address generate_sha256_implCompress(bool multi_block, const char *name) {
3790     assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), "");
3791     __ align(CodeEntryAlignment);
3792     StubCodeMark mark(this, "StubRoutines", name);
3793     address start = __ pc();
3794 
3795     Register buf = c_rarg0;
3796     Register state = c_rarg1;
3797     Register ofs = c_rarg2;
3798     Register limit = c_rarg3;
3799 
3800     const XMMRegister msg = xmm0;
3801     const XMMRegister state0 = xmm1;
3802     const XMMRegister state1 = xmm2;
3803     const XMMRegister msgtmp0 = xmm3;
3804 
3805     const XMMRegister msgtmp1 = xmm4;
3806     const XMMRegister msgtmp2 = xmm5;
3807     const XMMRegister msgtmp3 = xmm6;
3808     const XMMRegister msgtmp4 = xmm7;
3809 
3810     const XMMRegister shuf_mask = xmm8;
3811 
3812     __ enter();
3813 
3814     __ subptr(rsp, 4 * wordSize);
3815 
3816     if (VM_Version::supports_sha()) {
3817       __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3818         buf, state, ofs, limit, rsp, multi_block, shuf_mask);
3819     } else if (VM_Version::supports_avx2()) {
3820       __ sha256_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3821         buf, state, ofs, limit, rsp, multi_block, shuf_mask);
3822     }
3823     __ addptr(rsp, 4 * wordSize);
3824     __ vzeroupper();
3825     __ leave();
3826     __ ret(0);
3827     return start;
3828   }
3829 
3830   address generate_sha512_implCompress(bool multi_block, const char *name) {
3831     assert(VM_Version::supports_avx2(), "");
3832     assert(VM_Version::supports_bmi2(), "");
3833     __ align(CodeEntryAlignment);
3834     StubCodeMark mark(this, "StubRoutines", name);
3835     address start = __ pc();
3836 
3837     Register buf = c_rarg0;
3838     Register state = c_rarg1;
3839     Register ofs = c_rarg2;
3840     Register limit = c_rarg3;
3841 
3842     const XMMRegister msg = xmm0;
3843     const XMMRegister state0 = xmm1;
3844     const XMMRegister state1 = xmm2;
3845     const XMMRegister msgtmp0 = xmm3;
3846     const XMMRegister msgtmp1 = xmm4;
3847     const XMMRegister msgtmp2 = xmm5;
3848     const XMMRegister msgtmp3 = xmm6;
3849     const XMMRegister msgtmp4 = xmm7;
3850 
3851     const XMMRegister shuf_mask = xmm8;
3852 
3853     __ enter();
3854 
3855     __ sha512_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3856     buf, state, ofs, limit, rsp, multi_block, shuf_mask);
3857 
3858     __ vzeroupper();
3859     __ leave();
3860     __ ret(0);
3861     return start;
3862   }
3863 
3864   // This is a version of CTR/AES crypt which does 6 blocks in a loop at a time
3865   // to hide instruction latency
3866   //
3867   // Arguments:
3868   //
3869   // Inputs:
3870   //   c_rarg0   - source byte array address
3871   //   c_rarg1   - destination byte array address
3872   //   c_rarg2   - K (key) in little endian int array
3873   //   c_rarg3   - counter vector byte array address
3874   //   Linux
3875   //     c_rarg4   -          input length
3876   //     c_rarg5   -          saved encryptedCounter start
3877   //     rbp + 6 * wordSize - saved used length
3878   //   Windows
3879   //     rbp + 6 * wordSize - input length
3880   //     rbp + 7 * wordSize - saved encryptedCounter start
3881   //     rbp + 8 * wordSize - saved used length
3882   //
3883   // Output:
3884   //   rax       - input length
3885   //
3886   address generate_counterMode_AESCrypt_Parallel() {
3887     assert(UseAES, "need AES instructions and misaligned SSE support");
3888     __ align(CodeEntryAlignment);
3889     StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
3890     address start = __ pc();
3891     const Register from = c_rarg0; // source array address
3892     const Register to = c_rarg1; // destination array address
3893     const Register key = c_rarg2; // key array address
3894     const Register counter = c_rarg3; // counter byte array initialized from counter array address
3895                                       // and updated with the incremented counter in the end
3896 #ifndef _WIN64
3897     const Register len_reg = c_rarg4;
3898     const Register saved_encCounter_start = c_rarg5;
3899     const Register used_addr = r10;
3900     const Address  used_mem(rbp, 2 * wordSize);
3901     const Register used = r11;
3902 #else
3903     const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64
3904     const Address saved_encCounter_mem(rbp, 7 * wordSize); // length is on stack on Win64
3905     const Address used_mem(rbp, 8 * wordSize); // length is on stack on Win64
3906     const Register len_reg = r10; // pick the first volatile windows register
3907     const Register saved_encCounter_start = r11;
3908     const Register used_addr = r13;
3909     const Register used = r14;
3910 #endif
3911     const Register pos = rax;
3912 
3913     const int PARALLEL_FACTOR = 6;
3914     const XMMRegister xmm_counter_shuf_mask = xmm0;
3915     const XMMRegister xmm_key_shuf_mask = xmm1; // used temporarily to swap key bytes up front
3916     const XMMRegister xmm_curr_counter = xmm2;
3917 
3918     const XMMRegister xmm_key_tmp0 = xmm3;
3919     const XMMRegister xmm_key_tmp1 = xmm4;
3920 
3921     // registers holding the four results in the parallelized loop
3922     const XMMRegister xmm_result0 = xmm5;
3923     const XMMRegister xmm_result1 = xmm6;
3924     const XMMRegister xmm_result2 = xmm7;
3925     const XMMRegister xmm_result3 = xmm8;
3926     const XMMRegister xmm_result4 = xmm9;
3927     const XMMRegister xmm_result5 = xmm10;
3928 
3929     const XMMRegister xmm_from0 = xmm11;
3930     const XMMRegister xmm_from1 = xmm12;
3931     const XMMRegister xmm_from2 = xmm13;
3932     const XMMRegister xmm_from3 = xmm14; //the last one is xmm14. we have to preserve it on WIN64.
3933     const XMMRegister xmm_from4 = xmm3; //reuse xmm3~4. Because xmm_key_tmp0~1 are useless when loading input text
3934     const XMMRegister xmm_from5 = xmm4;
3935 
3936     //for key_128, key_192, key_256
3937     const int rounds[3] = {10, 12, 14};
3938     Label L_exit_preLoop, L_preLoop_start;
3939     Label L_multiBlock_loopTop[3];
3940     Label L_singleBlockLoopTop[3];
3941     Label L__incCounter[3][6]; //for 6 blocks
3942     Label L__incCounter_single[3]; //for single block, key128, key192, key256
3943     Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3];
3944     Label L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3];
3945 
3946     Label L_exit;
3947 
3948     __ enter(); // required for proper stackwalking of RuntimeStub frame
3949 
3950 #ifdef _WIN64
3951     // allocate spill slots for r13, r14
3952     enum {
3953         saved_r13_offset,
3954         saved_r14_offset
3955     };
3956     __ subptr(rsp, 2 * wordSize);
3957     __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
3958     __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
3959 
3960     // on win64, fill len_reg from stack position
3961     __ movl(len_reg, len_mem);
3962     __ movptr(saved_encCounter_start, saved_encCounter_mem);
3963     __ movptr(used_addr, used_mem);
3964     __ movl(used, Address(used_addr, 0));
3965 #else
3966     __ push(len_reg); // Save
3967     __ movptr(used_addr, used_mem);
3968     __ movl(used, Address(used_addr, 0));
3969 #endif
3970 
3971     __ push(rbx); // Save RBX
3972     __ movdqu(xmm_curr_counter, Address(counter, 0x00)); // initialize counter with initial counter
3973     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()), pos); // pos as scratch
3974     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled
3975     __ movptr(pos, 0);
3976 
3977     // Use the partially used encrpyted counter from last invocation
3978     __ BIND(L_preLoop_start);
3979     __ cmpptr(used, 16);
3980     __ jcc(Assembler::aboveEqual, L_exit_preLoop);
3981       __ cmpptr(len_reg, 0);
3982       __ jcc(Assembler::lessEqual, L_exit_preLoop);
3983       __ movb(rbx, Address(saved_encCounter_start, used));
3984       __ xorb(rbx, Address(from, pos));
3985       __ movb(Address(to, pos), rbx);
3986       __ addptr(pos, 1);
3987       __ addptr(used, 1);
3988       __ subptr(len_reg, 1);
3989 
3990     __ jmp(L_preLoop_start);
3991 
3992     __ BIND(L_exit_preLoop);
3993     __ movl(Address(used_addr, 0), used);
3994 
3995     // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
3996     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()), rbx); // rbx as scratch
3997     __ movl(rbx, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3998     __ cmpl(rbx, 52);
3999     __ jcc(Assembler::equal, L_multiBlock_loopTop[1]);
4000     __ cmpl(rbx, 60);
4001     __ jcc(Assembler::equal, L_multiBlock_loopTop[2]);
4002 
4003 #define CTR_DoSix(opc, src_reg)                \
4004     __ opc(xmm_result0, src_reg);              \
4005     __ opc(xmm_result1, src_reg);              \
4006     __ opc(xmm_result2, src_reg);              \
4007     __ opc(xmm_result3, src_reg);              \
4008     __ opc(xmm_result4, src_reg);              \
4009     __ opc(xmm_result5, src_reg);
4010 
4011     // k == 0 :  generate code for key_128
4012     // k == 1 :  generate code for key_192
4013     // k == 2 :  generate code for key_256
4014     for (int k = 0; k < 3; ++k) {
4015       //multi blocks starts here
4016       __ align(OptoLoopAlignment);
4017       __ BIND(L_multiBlock_loopTop[k]);
4018       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left
4019       __ jcc(Assembler::less, L_singleBlockLoopTop[k]);
4020       load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
4021 
4022       //load, then increase counters
4023       CTR_DoSix(movdqa, xmm_curr_counter);
4024       inc_counter(rbx, xmm_result1, 0x01, L__incCounter[k][0]);
4025       inc_counter(rbx, xmm_result2, 0x02, L__incCounter[k][1]);
4026       inc_counter(rbx, xmm_result3, 0x03, L__incCounter[k][2]);
4027       inc_counter(rbx, xmm_result4, 0x04, L__incCounter[k][3]);
4028       inc_counter(rbx, xmm_result5,  0x05, L__incCounter[k][4]);
4029       inc_counter(rbx, xmm_curr_counter, 0x06, L__incCounter[k][5]);
4030       CTR_DoSix(pshufb, xmm_counter_shuf_mask); // after increased, shuffled counters back for PXOR
4031       CTR_DoSix(pxor, xmm_key_tmp0);   //PXOR with Round 0 key
4032 
4033       //load two ROUND_KEYs at a time
4034       for (int i = 1; i < rounds[k]; ) {
4035         load_key(xmm_key_tmp1, key, (0x10 * i), xmm_key_shuf_mask);
4036         load_key(xmm_key_tmp0, key, (0x10 * (i+1)), xmm_key_shuf_mask);
4037         CTR_DoSix(aesenc, xmm_key_tmp1);
4038         i++;
4039         if (i != rounds[k]) {
4040           CTR_DoSix(aesenc, xmm_key_tmp0);
4041         } else {
4042           CTR_DoSix(aesenclast, xmm_key_tmp0);
4043         }
4044         i++;
4045       }
4046 
4047       // get next PARALLEL_FACTOR blocks into xmm_result registers
4048       __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
4049       __ movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
4050       __ movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
4051       __ movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
4052       __ movdqu(xmm_from4, Address(from, pos, Address::times_1, 4 * AESBlockSize));
4053       __ movdqu(xmm_from5, Address(from, pos, Address::times_1, 5 * AESBlockSize));
4054 
4055       __ pxor(xmm_result0, xmm_from0);
4056       __ pxor(xmm_result1, xmm_from1);
4057       __ pxor(xmm_result2, xmm_from2);
4058       __ pxor(xmm_result3, xmm_from3);
4059       __ pxor(xmm_result4, xmm_from4);
4060       __ pxor(xmm_result5, xmm_from5);
4061 
4062       // store 6 results into the next 64 bytes of output
4063       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
4064       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
4065       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
4066       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
4067       __ movdqu(Address(to, pos, Address::times_1, 4 * AESBlockSize), xmm_result4);
4068       __ movdqu(Address(to, pos, Address::times_1, 5 * AESBlockSize), xmm_result5);
4069 
4070       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text
4071       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length
4072       __ jmp(L_multiBlock_loopTop[k]);
4073 
4074       // singleBlock starts here
4075       __ align(OptoLoopAlignment);
4076       __ BIND(L_singleBlockLoopTop[k]);
4077       __ cmpptr(len_reg, 0);
4078       __ jcc(Assembler::lessEqual, L_exit);
4079       load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
4080       __ movdqa(xmm_result0, xmm_curr_counter);
4081       inc_counter(rbx, xmm_curr_counter, 0x01, L__incCounter_single[k]);
4082       __ pshufb(xmm_result0, xmm_counter_shuf_mask);
4083       __ pxor(xmm_result0, xmm_key_tmp0);
4084       for (int i = 1; i < rounds[k]; i++) {
4085         load_key(xmm_key_tmp0, key, (0x10 * i), xmm_key_shuf_mask);
4086         __ aesenc(xmm_result0, xmm_key_tmp0);
4087       }
4088       load_key(xmm_key_tmp0, key, (rounds[k] * 0x10), xmm_key_shuf_mask);
4089       __ aesenclast(xmm_result0, xmm_key_tmp0);
4090       __ cmpptr(len_reg, AESBlockSize);
4091       __ jcc(Assembler::less, L_processTail_insr[k]);
4092         __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
4093         __ pxor(xmm_result0, xmm_from0);
4094         __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
4095         __ addptr(pos, AESBlockSize);
4096         __ subptr(len_reg, AESBlockSize);
4097         __ jmp(L_singleBlockLoopTop[k]);
4098       __ BIND(L_processTail_insr[k]);                               // Process the tail part of the input array
4099         __ addptr(pos, len_reg);                                    // 1. Insert bytes from src array into xmm_from0 register
4100         __ testptr(len_reg, 8);
4101         __ jcc(Assembler::zero, L_processTail_4_insr[k]);
4102           __ subptr(pos,8);
4103           __ pinsrq(xmm_from0, Address(from, pos), 0);
4104         __ BIND(L_processTail_4_insr[k]);
4105         __ testptr(len_reg, 4);
4106         __ jcc(Assembler::zero, L_processTail_2_insr[k]);
4107           __ subptr(pos,4);
4108           __ pslldq(xmm_from0, 4);
4109           __ pinsrd(xmm_from0, Address(from, pos), 0);
4110         __ BIND(L_processTail_2_insr[k]);
4111         __ testptr(len_reg, 2);
4112         __ jcc(Assembler::zero, L_processTail_1_insr[k]);
4113           __ subptr(pos, 2);
4114           __ pslldq(xmm_from0, 2);
4115           __ pinsrw(xmm_from0, Address(from, pos), 0);
4116         __ BIND(L_processTail_1_insr[k]);
4117         __ testptr(len_reg, 1);
4118         __ jcc(Assembler::zero, L_processTail_exit_insr[k]);
4119           __ subptr(pos, 1);
4120           __ pslldq(xmm_from0, 1);
4121           __ pinsrb(xmm_from0, Address(from, pos), 0);
4122         __ BIND(L_processTail_exit_insr[k]);
4123 
4124         __ movdqu(Address(saved_encCounter_start, 0), xmm_result0);  // 2. Perform pxor of the encrypted counter and plaintext Bytes.
4125         __ pxor(xmm_result0, xmm_from0);                             //    Also the encrypted counter is saved for next invocation.
4126 
4127         __ testptr(len_reg, 8);
4128         __ jcc(Assembler::zero, L_processTail_4_extr[k]);            // 3. Extract bytes from xmm_result0 into the dest. array
4129           __ pextrq(Address(to, pos), xmm_result0, 0);
4130           __ psrldq(xmm_result0, 8);
4131           __ addptr(pos, 8);
4132         __ BIND(L_processTail_4_extr[k]);
4133         __ testptr(len_reg, 4);
4134         __ jcc(Assembler::zero, L_processTail_2_extr[k]);
4135           __ pextrd(Address(to, pos), xmm_result0, 0);
4136           __ psrldq(xmm_result0, 4);
4137           __ addptr(pos, 4);
4138         __ BIND(L_processTail_2_extr[k]);
4139         __ testptr(len_reg, 2);
4140         __ jcc(Assembler::zero, L_processTail_1_extr[k]);
4141           __ pextrw(Address(to, pos), xmm_result0, 0);
4142           __ psrldq(xmm_result0, 2);
4143           __ addptr(pos, 2);
4144         __ BIND(L_processTail_1_extr[k]);
4145         __ testptr(len_reg, 1);
4146         __ jcc(Assembler::zero, L_processTail_exit_extr[k]);
4147           __ pextrb(Address(to, pos), xmm_result0, 0);
4148 
4149         __ BIND(L_processTail_exit_extr[k]);
4150         __ movl(Address(used_addr, 0), len_reg);
4151         __ jmp(L_exit);
4152 
4153     }
4154 
4155     __ BIND(L_exit);
4156     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back.
4157     __ movdqu(Address(counter, 0), xmm_curr_counter); //save counter back
4158     __ pop(rbx); // pop the saved RBX.
4159 #ifdef _WIN64
4160     __ movl(rax, len_mem);
4161     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
4162     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
4163     __ addptr(rsp, 2 * wordSize);
4164 #else
4165     __ pop(rax); // return 'len'
4166 #endif
4167     __ leave(); // required for proper stackwalking of RuntimeStub frame
4168     __ ret(0);
4169     return start;
4170   }
4171 
4172 void roundDec(XMMRegister xmm_reg) {
4173   __ vaesdec(xmm1, xmm1, xmm_reg, Assembler::AVX_512bit);
4174   __ vaesdec(xmm2, xmm2, xmm_reg, Assembler::AVX_512bit);
4175   __ vaesdec(xmm3, xmm3, xmm_reg, Assembler::AVX_512bit);
4176   __ vaesdec(xmm4, xmm4, xmm_reg, Assembler::AVX_512bit);
4177   __ vaesdec(xmm5, xmm5, xmm_reg, Assembler::AVX_512bit);
4178   __ vaesdec(xmm6, xmm6, xmm_reg, Assembler::AVX_512bit);
4179   __ vaesdec(xmm7, xmm7, xmm_reg, Assembler::AVX_512bit);
4180   __ vaesdec(xmm8, xmm8, xmm_reg, Assembler::AVX_512bit);
4181 }
4182 
4183 void roundDeclast(XMMRegister xmm_reg) {
4184   __ vaesdeclast(xmm1, xmm1, xmm_reg, Assembler::AVX_512bit);
4185   __ vaesdeclast(xmm2, xmm2, xmm_reg, Assembler::AVX_512bit);
4186   __ vaesdeclast(xmm3, xmm3, xmm_reg, Assembler::AVX_512bit);
4187   __ vaesdeclast(xmm4, xmm4, xmm_reg, Assembler::AVX_512bit);
4188   __ vaesdeclast(xmm5, xmm5, xmm_reg, Assembler::AVX_512bit);
4189   __ vaesdeclast(xmm6, xmm6, xmm_reg, Assembler::AVX_512bit);
4190   __ vaesdeclast(xmm7, xmm7, xmm_reg, Assembler::AVX_512bit);
4191   __ vaesdeclast(xmm8, xmm8, xmm_reg, Assembler::AVX_512bit);
4192 }
4193 
4194   void ev_load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask = NULL) {
4195     __ movdqu(xmmdst, Address(key, offset));
4196     if (xmm_shuf_mask != NULL) {
4197       __ pshufb(xmmdst, xmm_shuf_mask);
4198     } else {
4199       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
4200     }
4201     __ evshufi64x2(xmmdst, xmmdst, xmmdst, 0x0, Assembler::AVX_512bit);
4202 
4203   }
4204 
4205 address generate_cipherBlockChaining_decryptVectorAESCrypt() {
4206     assert(VM_Version::supports_vaes(), "need AES instructions and misaligned SSE support");
4207     __ align(CodeEntryAlignment);
4208     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
4209     address start = __ pc();
4210 
4211     const Register from = c_rarg0;  // source array address
4212     const Register to = c_rarg1;  // destination array address
4213     const Register key = c_rarg2;  // key array address
4214     const Register rvec = c_rarg3;  // r byte array initialized from initvector array address
4215     // and left with the results of the last encryption block
4216 #ifndef _WIN64
4217     const Register len_reg = c_rarg4;  // src len (must be multiple of blocksize 16)
4218 #else
4219     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
4220     const Register len_reg = r11;      // pick the volatile windows register
4221 #endif
4222 
4223     Label Loop, Loop1, L_128, L_256, L_192, KEY_192, KEY_256, Loop2, Lcbc_dec_rem_loop,
4224           Lcbc_dec_rem_last, Lcbc_dec_ret, Lcbc_dec_rem, Lcbc_exit;
4225 
4226     __ enter();
4227 
4228 #ifdef _WIN64
4229   // on win64, fill len_reg from stack position
4230     __ movl(len_reg, len_mem);
4231 #else
4232     __ push(len_reg); // Save
4233 #endif
4234     __ push(rbx);
4235     __ vzeroupper();
4236 
4237     // Temporary variable declaration for swapping key bytes
4238     const XMMRegister xmm_key_shuf_mask = xmm1;
4239     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
4240 
4241     // Calculate number of rounds from key size: 44 for 10-rounds, 52 for 12-rounds, 60 for 14-rounds
4242     const Register rounds = rbx;
4243     __ movl(rounds, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
4244 
4245     const XMMRegister IV = xmm0;
4246     // Load IV and broadcast value to 512-bits
4247     __ evbroadcasti64x2(IV, Address(rvec, 0), Assembler::AVX_512bit);
4248 
4249     // Temporary variables for storing round keys
4250     const XMMRegister RK0 = xmm30;
4251     const XMMRegister RK1 = xmm9;
4252     const XMMRegister RK2 = xmm18;
4253     const XMMRegister RK3 = xmm19;
4254     const XMMRegister RK4 = xmm20;
4255     const XMMRegister RK5 = xmm21;
4256     const XMMRegister RK6 = xmm22;
4257     const XMMRegister RK7 = xmm23;
4258     const XMMRegister RK8 = xmm24;
4259     const XMMRegister RK9 = xmm25;
4260     const XMMRegister RK10 = xmm26;
4261 
4262      // Load and shuffle key
4263     // the java expanded key ordering is rotated one position from what we want
4264     // so we start from 1*16 here and hit 0*16 last
4265     ev_load_key(RK1, key, 1 * 16, xmm_key_shuf_mask);
4266     ev_load_key(RK2, key, 2 * 16, xmm_key_shuf_mask);
4267     ev_load_key(RK3, key, 3 * 16, xmm_key_shuf_mask);
4268     ev_load_key(RK4, key, 4 * 16, xmm_key_shuf_mask);
4269     ev_load_key(RK5, key, 5 * 16, xmm_key_shuf_mask);
4270     ev_load_key(RK6, key, 6 * 16, xmm_key_shuf_mask);
4271     ev_load_key(RK7, key, 7 * 16, xmm_key_shuf_mask);
4272     ev_load_key(RK8, key, 8 * 16, xmm_key_shuf_mask);
4273     ev_load_key(RK9, key, 9 * 16, xmm_key_shuf_mask);
4274     ev_load_key(RK10, key, 10 * 16, xmm_key_shuf_mask);
4275     ev_load_key(RK0, key, 0*16, xmm_key_shuf_mask);
4276 
4277     // Variables for storing source cipher text
4278     const XMMRegister S0 = xmm10;
4279     const XMMRegister S1 = xmm11;
4280     const XMMRegister S2 = xmm12;
4281     const XMMRegister S3 = xmm13;
4282     const XMMRegister S4 = xmm14;
4283     const XMMRegister S5 = xmm15;
4284     const XMMRegister S6 = xmm16;
4285     const XMMRegister S7 = xmm17;
4286 
4287     // Variables for storing decrypted text
4288     const XMMRegister B0 = xmm1;
4289     const XMMRegister B1 = xmm2;
4290     const XMMRegister B2 = xmm3;
4291     const XMMRegister B3 = xmm4;
4292     const XMMRegister B4 = xmm5;
4293     const XMMRegister B5 = xmm6;
4294     const XMMRegister B6 = xmm7;
4295     const XMMRegister B7 = xmm8;
4296 
4297     __ cmpl(rounds, 44);
4298     __ jcc(Assembler::greater, KEY_192);
4299     __ jmp(Loop);
4300 
4301     __ BIND(KEY_192);
4302     const XMMRegister RK11 = xmm27;
4303     const XMMRegister RK12 = xmm28;
4304     ev_load_key(RK11, key, 11*16, xmm_key_shuf_mask);
4305     ev_load_key(RK12, key, 12*16, xmm_key_shuf_mask);
4306 
4307     __ cmpl(rounds, 52);
4308     __ jcc(Assembler::greater, KEY_256);
4309     __ jmp(Loop);
4310 
4311     __ BIND(KEY_256);
4312     const XMMRegister RK13 = xmm29;
4313     const XMMRegister RK14 = xmm31;
4314     ev_load_key(RK13, key, 13*16, xmm_key_shuf_mask);
4315     ev_load_key(RK14, key, 14*16, xmm_key_shuf_mask);
4316 
4317     __ BIND(Loop);
4318     __ cmpl(len_reg, 512);
4319     __ jcc(Assembler::below, Lcbc_dec_rem);
4320     __ BIND(Loop1);
4321     __ subl(len_reg, 512);
4322     __ evmovdquq(S0, Address(from, 0 * 64), Assembler::AVX_512bit);
4323     __ evmovdquq(S1, Address(from, 1 * 64), Assembler::AVX_512bit);
4324     __ evmovdquq(S2, Address(from, 2 * 64), Assembler::AVX_512bit);
4325     __ evmovdquq(S3, Address(from, 3 * 64), Assembler::AVX_512bit);
4326     __ evmovdquq(S4, Address(from, 4 * 64), Assembler::AVX_512bit);
4327     __ evmovdquq(S5, Address(from, 5 * 64), Assembler::AVX_512bit);
4328     __ evmovdquq(S6, Address(from, 6 * 64), Assembler::AVX_512bit);
4329     __ evmovdquq(S7, Address(from, 7 * 64), Assembler::AVX_512bit);
4330     __ leaq(from, Address(from, 8 * 64));
4331 
4332     __ evpxorq(B0, S0, RK1, Assembler::AVX_512bit);
4333     __ evpxorq(B1, S1, RK1, Assembler::AVX_512bit);
4334     __ evpxorq(B2, S2, RK1, Assembler::AVX_512bit);
4335     __ evpxorq(B3, S3, RK1, Assembler::AVX_512bit);
4336     __ evpxorq(B4, S4, RK1, Assembler::AVX_512bit);
4337     __ evpxorq(B5, S5, RK1, Assembler::AVX_512bit);
4338     __ evpxorq(B6, S6, RK1, Assembler::AVX_512bit);
4339     __ evpxorq(B7, S7, RK1, Assembler::AVX_512bit);
4340 
4341     __ evalignq(IV, S0, IV, 0x06);
4342     __ evalignq(S0, S1, S0, 0x06);
4343     __ evalignq(S1, S2, S1, 0x06);
4344     __ evalignq(S2, S3, S2, 0x06);
4345     __ evalignq(S3, S4, S3, 0x06);
4346     __ evalignq(S4, S5, S4, 0x06);
4347     __ evalignq(S5, S6, S5, 0x06);
4348     __ evalignq(S6, S7, S6, 0x06);
4349 
4350     roundDec(RK2);
4351     roundDec(RK3);
4352     roundDec(RK4);
4353     roundDec(RK5);
4354     roundDec(RK6);
4355     roundDec(RK7);
4356     roundDec(RK8);
4357     roundDec(RK9);
4358     roundDec(RK10);
4359 
4360     __ cmpl(rounds, 44);
4361     __ jcc(Assembler::belowEqual, L_128);
4362     roundDec(RK11);
4363     roundDec(RK12);
4364 
4365     __ cmpl(rounds, 52);
4366     __ jcc(Assembler::belowEqual, L_192);
4367     roundDec(RK13);
4368     roundDec(RK14);
4369 
4370     __ BIND(L_256);
4371     roundDeclast(RK0);
4372     __ jmp(Loop2);
4373 
4374     __ BIND(L_128);
4375     roundDeclast(RK0);
4376     __ jmp(Loop2);
4377 
4378     __ BIND(L_192);
4379     roundDeclast(RK0);
4380 
4381     __ BIND(Loop2);
4382     __ evpxorq(B0, B0, IV, Assembler::AVX_512bit);
4383     __ evpxorq(B1, B1, S0, Assembler::AVX_512bit);
4384     __ evpxorq(B2, B2, S1, Assembler::AVX_512bit);
4385     __ evpxorq(B3, B3, S2, Assembler::AVX_512bit);
4386     __ evpxorq(B4, B4, S3, Assembler::AVX_512bit);
4387     __ evpxorq(B5, B5, S4, Assembler::AVX_512bit);
4388     __ evpxorq(B6, B6, S5, Assembler::AVX_512bit);
4389     __ evpxorq(B7, B7, S6, Assembler::AVX_512bit);
4390     __ evmovdquq(IV, S7, Assembler::AVX_512bit);
4391 
4392     __ evmovdquq(Address(to, 0 * 64), B0, Assembler::AVX_512bit);
4393     __ evmovdquq(Address(to, 1 * 64), B1, Assembler::AVX_512bit);
4394     __ evmovdquq(Address(to, 2 * 64), B2, Assembler::AVX_512bit);
4395     __ evmovdquq(Address(to, 3 * 64), B3, Assembler::AVX_512bit);
4396     __ evmovdquq(Address(to, 4 * 64), B4, Assembler::AVX_512bit);
4397     __ evmovdquq(Address(to, 5 * 64), B5, Assembler::AVX_512bit);
4398     __ evmovdquq(Address(to, 6 * 64), B6, Assembler::AVX_512bit);
4399     __ evmovdquq(Address(to, 7 * 64), B7, Assembler::AVX_512bit);
4400     __ leaq(to, Address(to, 8 * 64));
4401     __ jmp(Loop);
4402 
4403     __ BIND(Lcbc_dec_rem);
4404     __ evshufi64x2(IV, IV, IV, 0x03, Assembler::AVX_512bit);
4405 
4406     __ BIND(Lcbc_dec_rem_loop);
4407     __ subl(len_reg, 16);
4408     __ jcc(Assembler::carrySet, Lcbc_dec_ret);
4409 
4410     __ movdqu(S0, Address(from, 0));
4411     __ evpxorq(B0, S0, RK1, Assembler::AVX_512bit);
4412     __ vaesdec(B0, B0, RK2, Assembler::AVX_512bit);
4413     __ vaesdec(B0, B0, RK3, Assembler::AVX_512bit);
4414     __ vaesdec(B0, B0, RK4, Assembler::AVX_512bit);
4415     __ vaesdec(B0, B0, RK5, Assembler::AVX_512bit);
4416     __ vaesdec(B0, B0, RK6, Assembler::AVX_512bit);
4417     __ vaesdec(B0, B0, RK7, Assembler::AVX_512bit);
4418     __ vaesdec(B0, B0, RK8, Assembler::AVX_512bit);
4419     __ vaesdec(B0, B0, RK9, Assembler::AVX_512bit);
4420     __ vaesdec(B0, B0, RK10, Assembler::AVX_512bit);
4421     __ cmpl(rounds, 44);
4422     __ jcc(Assembler::belowEqual, Lcbc_dec_rem_last);
4423 
4424     __ vaesdec(B0, B0, RK11, Assembler::AVX_512bit);
4425     __ vaesdec(B0, B0, RK12, Assembler::AVX_512bit);
4426     __ cmpl(rounds, 52);
4427     __ jcc(Assembler::belowEqual, Lcbc_dec_rem_last);
4428 
4429     __ vaesdec(B0, B0, RK13, Assembler::AVX_512bit);
4430     __ vaesdec(B0, B0, RK14, Assembler::AVX_512bit);
4431 
4432     __ BIND(Lcbc_dec_rem_last);
4433     __ vaesdeclast(B0, B0, RK0, Assembler::AVX_512bit);
4434 
4435     __ evpxorq(B0, B0, IV, Assembler::AVX_512bit);
4436     __ evmovdquq(IV, S0, Assembler::AVX_512bit);
4437     __ movdqu(Address(to, 0), B0);
4438     __ leaq(from, Address(from, 16));
4439     __ leaq(to, Address(to, 16));
4440     __ jmp(Lcbc_dec_rem_loop);
4441 
4442     __ BIND(Lcbc_dec_ret);
4443     __ movdqu(Address(rvec, 0), IV);
4444 
4445     // Zero out the round keys
4446     __ evpxorq(RK0, RK0, RK0, Assembler::AVX_512bit);
4447     __ evpxorq(RK1, RK1, RK1, Assembler::AVX_512bit);
4448     __ evpxorq(RK2, RK2, RK2, Assembler::AVX_512bit);
4449     __ evpxorq(RK3, RK3, RK3, Assembler::AVX_512bit);
4450     __ evpxorq(RK4, RK4, RK4, Assembler::AVX_512bit);
4451     __ evpxorq(RK5, RK5, RK5, Assembler::AVX_512bit);
4452     __ evpxorq(RK6, RK6, RK6, Assembler::AVX_512bit);
4453     __ evpxorq(RK7, RK7, RK7, Assembler::AVX_512bit);
4454     __ evpxorq(RK8, RK8, RK8, Assembler::AVX_512bit);
4455     __ evpxorq(RK9, RK9, RK9, Assembler::AVX_512bit);
4456     __ evpxorq(RK10, RK10, RK10, Assembler::AVX_512bit);
4457     __ cmpl(rounds, 44);
4458     __ jcc(Assembler::belowEqual, Lcbc_exit);
4459     __ evpxorq(RK11, RK11, RK11, Assembler::AVX_512bit);
4460     __ evpxorq(RK12, RK12, RK12, Assembler::AVX_512bit);
4461     __ cmpl(rounds, 52);
4462     __ jcc(Assembler::belowEqual, Lcbc_exit);
4463     __ evpxorq(RK13, RK13, RK13, Assembler::AVX_512bit);
4464     __ evpxorq(RK14, RK14, RK14, Assembler::AVX_512bit);
4465 
4466     __ BIND(Lcbc_exit);
4467     __ pop(rbx);
4468 #ifdef _WIN64
4469     __ movl(rax, len_mem);
4470 #else
4471     __ pop(rax); // return length
4472 #endif
4473     __ leave(); // required for proper stackwalking of RuntimeStub frame
4474     __ ret(0);
4475     return start;
4476 }
4477 
4478 // Polynomial x^128+x^127+x^126+x^121+1
4479 address ghash_polynomial_addr() {
4480     __ align(CodeEntryAlignment);
4481     StubCodeMark mark(this, "StubRoutines", "_ghash_poly_addr");
4482     address start = __ pc();
4483     __ emit_data64(0x0000000000000001, relocInfo::none);
4484     __ emit_data64(0xc200000000000000, relocInfo::none);
4485     return start;
4486 }
4487 
4488 address ghash_shufflemask_addr() {
4489     __ align(CodeEntryAlignment);
4490     StubCodeMark mark(this, "StubRoutines", "_ghash_shuffmask_addr");
4491     address start = __ pc();
4492     __ emit_data64(0x0f0f0f0f0f0f0f0f, relocInfo::none);
4493     __ emit_data64(0x0f0f0f0f0f0f0f0f, relocInfo::none);
4494     return start;
4495 }
4496 
4497 // Ghash single and multi block operations using AVX instructions
4498 address generate_avx_ghash_processBlocks() {
4499     __ align(CodeEntryAlignment);
4500 
4501     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
4502     address start = __ pc();
4503 
4504     // arguments
4505     const Register state = c_rarg0;
4506     const Register htbl = c_rarg1;
4507     const Register data = c_rarg2;
4508     const Register blocks = c_rarg3;
4509     __ enter();
4510    // Save state before entering routine
4511     __ avx_ghash(state, htbl, data, blocks);
4512     __ leave(); // required for proper stackwalking of RuntimeStub frame
4513     __ ret(0);
4514     return start;
4515 }
4516 
4517   // byte swap x86 long
4518   address generate_ghash_long_swap_mask() {
4519     __ align(CodeEntryAlignment);
4520     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
4521     address start = __ pc();
4522     __ emit_data64(0x0f0e0d0c0b0a0908, relocInfo::none );
4523     __ emit_data64(0x0706050403020100, relocInfo::none );
4524   return start;
4525   }
4526 
4527   // byte swap x86 byte array
4528   address generate_ghash_byte_swap_mask() {
4529     __ align(CodeEntryAlignment);
4530     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
4531     address start = __ pc();
4532     __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none );
4533     __ emit_data64(0x0001020304050607, relocInfo::none );
4534   return start;
4535   }
4536 
4537   /* Single and multi-block ghash operations */
4538   address generate_ghash_processBlocks() {
4539     __ align(CodeEntryAlignment);
4540     Label L_ghash_loop, L_exit;
4541     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
4542     address start = __ pc();
4543 
4544     const Register state        = c_rarg0;
4545     const Register subkeyH      = c_rarg1;
4546     const Register data         = c_rarg2;
4547     const Register blocks       = c_rarg3;
4548 
4549     const XMMRegister xmm_temp0 = xmm0;
4550     const XMMRegister xmm_temp1 = xmm1;
4551     const XMMRegister xmm_temp2 = xmm2;
4552     const XMMRegister xmm_temp3 = xmm3;
4553     const XMMRegister xmm_temp4 = xmm4;
4554     const XMMRegister xmm_temp5 = xmm5;
4555     const XMMRegister xmm_temp6 = xmm6;
4556     const XMMRegister xmm_temp7 = xmm7;
4557     const XMMRegister xmm_temp8 = xmm8;
4558     const XMMRegister xmm_temp9 = xmm9;
4559     const XMMRegister xmm_temp10 = xmm10;
4560 
4561     __ enter();
4562 
4563     __ movdqu(xmm_temp10, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
4564 
4565     __ movdqu(xmm_temp0, Address(state, 0));
4566     __ pshufb(xmm_temp0, xmm_temp10);
4567 
4568 
4569     __ BIND(L_ghash_loop);
4570     __ movdqu(xmm_temp2, Address(data, 0));
4571     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
4572 
4573     __ movdqu(xmm_temp1, Address(subkeyH, 0));
4574     __ pshufb(xmm_temp1, xmm_temp10);
4575 
4576     __ pxor(xmm_temp0, xmm_temp2);
4577 
4578     //
4579     // Multiply with the hash key
4580     //
4581     __ movdqu(xmm_temp3, xmm_temp0);
4582     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
4583     __ movdqu(xmm_temp4, xmm_temp0);
4584     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
4585 
4586     __ movdqu(xmm_temp5, xmm_temp0);
4587     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
4588     __ movdqu(xmm_temp6, xmm_temp0);
4589     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
4590 
4591     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
4592 
4593     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
4594     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
4595     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
4596     __ pxor(xmm_temp3, xmm_temp5);
4597     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
4598                                         // of the carry-less multiplication of
4599                                         // xmm0 by xmm1.
4600 
4601     // We shift the result of the multiplication by one bit position
4602     // to the left to cope for the fact that the bits are reversed.
4603     __ movdqu(xmm_temp7, xmm_temp3);
4604     __ movdqu(xmm_temp8, xmm_temp6);
4605     __ pslld(xmm_temp3, 1);
4606     __ pslld(xmm_temp6, 1);
4607     __ psrld(xmm_temp7, 31);
4608     __ psrld(xmm_temp8, 31);
4609     __ movdqu(xmm_temp9, xmm_temp7);
4610     __ pslldq(xmm_temp8, 4);
4611     __ pslldq(xmm_temp7, 4);
4612     __ psrldq(xmm_temp9, 12);
4613     __ por(xmm_temp3, xmm_temp7);
4614     __ por(xmm_temp6, xmm_temp8);
4615     __ por(xmm_temp6, xmm_temp9);
4616 
4617     //
4618     // First phase of the reduction
4619     //
4620     // Move xmm3 into xmm7, xmm8, xmm9 in order to perform the shifts
4621     // independently.
4622     __ movdqu(xmm_temp7, xmm_temp3);
4623     __ movdqu(xmm_temp8, xmm_temp3);
4624     __ movdqu(xmm_temp9, xmm_temp3);
4625     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
4626     __ pslld(xmm_temp8, 30);    // packed right shift shifting << 30
4627     __ pslld(xmm_temp9, 25);    // packed right shift shifting << 25
4628     __ pxor(xmm_temp7, xmm_temp8);      // xor the shifted versions
4629     __ pxor(xmm_temp7, xmm_temp9);
4630     __ movdqu(xmm_temp8, xmm_temp7);
4631     __ pslldq(xmm_temp7, 12);
4632     __ psrldq(xmm_temp8, 4);
4633     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
4634 
4635     //
4636     // Second phase of the reduction
4637     //
4638     // Make 3 copies of xmm3 in xmm2, xmm4, xmm5 for doing these
4639     // shift operations.
4640     __ movdqu(xmm_temp2, xmm_temp3);
4641     __ movdqu(xmm_temp4, xmm_temp3);
4642     __ movdqu(xmm_temp5, xmm_temp3);
4643     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
4644     __ psrld(xmm_temp4, 2);     // packed left shifting >> 2
4645     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
4646     __ pxor(xmm_temp2, xmm_temp4);      // xor the shifted versions
4647     __ pxor(xmm_temp2, xmm_temp5);
4648     __ pxor(xmm_temp2, xmm_temp8);
4649     __ pxor(xmm_temp3, xmm_temp2);
4650     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
4651 
4652     __ decrement(blocks);
4653     __ jcc(Assembler::zero, L_exit);
4654     __ movdqu(xmm_temp0, xmm_temp6);
4655     __ addptr(data, 16);
4656     __ jmp(L_ghash_loop);
4657 
4658     __ BIND(L_exit);
4659     __ pshufb(xmm_temp6, xmm_temp10);          // Byte swap 16-byte result
4660     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
4661     __ leave();
4662     __ ret(0);
4663     return start;
4664   }
4665 
4666   //base64 character set
4667   address base64_charset_addr() {
4668     __ align(CodeEntryAlignment);
4669     StubCodeMark mark(this, "StubRoutines", "base64_charset");
4670     address start = __ pc();
4671     __ emit_data64(0x0000004200000041, relocInfo::none);
4672     __ emit_data64(0x0000004400000043, relocInfo::none);
4673     __ emit_data64(0x0000004600000045, relocInfo::none);
4674     __ emit_data64(0x0000004800000047, relocInfo::none);
4675     __ emit_data64(0x0000004a00000049, relocInfo::none);
4676     __ emit_data64(0x0000004c0000004b, relocInfo::none);
4677     __ emit_data64(0x0000004e0000004d, relocInfo::none);
4678     __ emit_data64(0x000000500000004f, relocInfo::none);
4679     __ emit_data64(0x0000005200000051, relocInfo::none);
4680     __ emit_data64(0x0000005400000053, relocInfo::none);
4681     __ emit_data64(0x0000005600000055, relocInfo::none);
4682     __ emit_data64(0x0000005800000057, relocInfo::none);
4683     __ emit_data64(0x0000005a00000059, relocInfo::none);
4684     __ emit_data64(0x0000006200000061, relocInfo::none);
4685     __ emit_data64(0x0000006400000063, relocInfo::none);
4686     __ emit_data64(0x0000006600000065, relocInfo::none);
4687     __ emit_data64(0x0000006800000067, relocInfo::none);
4688     __ emit_data64(0x0000006a00000069, relocInfo::none);
4689     __ emit_data64(0x0000006c0000006b, relocInfo::none);
4690     __ emit_data64(0x0000006e0000006d, relocInfo::none);
4691     __ emit_data64(0x000000700000006f, relocInfo::none);
4692     __ emit_data64(0x0000007200000071, relocInfo::none);
4693     __ emit_data64(0x0000007400000073, relocInfo::none);
4694     __ emit_data64(0x0000007600000075, relocInfo::none);
4695     __ emit_data64(0x0000007800000077, relocInfo::none);
4696     __ emit_data64(0x0000007a00000079, relocInfo::none);
4697     __ emit_data64(0x0000003100000030, relocInfo::none);
4698     __ emit_data64(0x0000003300000032, relocInfo::none);
4699     __ emit_data64(0x0000003500000034, relocInfo::none);
4700     __ emit_data64(0x0000003700000036, relocInfo::none);
4701     __ emit_data64(0x0000003900000038, relocInfo::none);
4702     __ emit_data64(0x0000002f0000002b, relocInfo::none);
4703     return start;
4704   }
4705 
4706   //base64 url character set
4707   address base64url_charset_addr() {
4708     __ align(CodeEntryAlignment);
4709     StubCodeMark mark(this, "StubRoutines", "base64url_charset");
4710     address start = __ pc();
4711     __ emit_data64(0x0000004200000041, relocInfo::none);
4712     __ emit_data64(0x0000004400000043, relocInfo::none);
4713     __ emit_data64(0x0000004600000045, relocInfo::none);
4714     __ emit_data64(0x0000004800000047, relocInfo::none);
4715     __ emit_data64(0x0000004a00000049, relocInfo::none);
4716     __ emit_data64(0x0000004c0000004b, relocInfo::none);
4717     __ emit_data64(0x0000004e0000004d, relocInfo::none);
4718     __ emit_data64(0x000000500000004f, relocInfo::none);
4719     __ emit_data64(0x0000005200000051, relocInfo::none);
4720     __ emit_data64(0x0000005400000053, relocInfo::none);
4721     __ emit_data64(0x0000005600000055, relocInfo::none);
4722     __ emit_data64(0x0000005800000057, relocInfo::none);
4723     __ emit_data64(0x0000005a00000059, relocInfo::none);
4724     __ emit_data64(0x0000006200000061, relocInfo::none);
4725     __ emit_data64(0x0000006400000063, relocInfo::none);
4726     __ emit_data64(0x0000006600000065, relocInfo::none);
4727     __ emit_data64(0x0000006800000067, relocInfo::none);
4728     __ emit_data64(0x0000006a00000069, relocInfo::none);
4729     __ emit_data64(0x0000006c0000006b, relocInfo::none);
4730     __ emit_data64(0x0000006e0000006d, relocInfo::none);
4731     __ emit_data64(0x000000700000006f, relocInfo::none);
4732     __ emit_data64(0x0000007200000071, relocInfo::none);
4733     __ emit_data64(0x0000007400000073, relocInfo::none);
4734     __ emit_data64(0x0000007600000075, relocInfo::none);
4735     __ emit_data64(0x0000007800000077, relocInfo::none);
4736     __ emit_data64(0x0000007a00000079, relocInfo::none);
4737     __ emit_data64(0x0000003100000030, relocInfo::none);
4738     __ emit_data64(0x0000003300000032, relocInfo::none);
4739     __ emit_data64(0x0000003500000034, relocInfo::none);
4740     __ emit_data64(0x0000003700000036, relocInfo::none);
4741     __ emit_data64(0x0000003900000038, relocInfo::none);
4742     __ emit_data64(0x0000005f0000002d, relocInfo::none);
4743 
4744     return start;
4745   }
4746 
4747   address base64_bswap_mask_addr() {
4748     __ align(CodeEntryAlignment);
4749     StubCodeMark mark(this, "StubRoutines", "bswap_mask_base64");
4750     address start = __ pc();
4751     __ emit_data64(0x0504038002010080, relocInfo::none);
4752     __ emit_data64(0x0b0a098008070680, relocInfo::none);
4753     __ emit_data64(0x0908078006050480, relocInfo::none);
4754     __ emit_data64(0x0f0e0d800c0b0a80, relocInfo::none);
4755     __ emit_data64(0x0605048003020180, relocInfo::none);
4756     __ emit_data64(0x0c0b0a8009080780, relocInfo::none);
4757     __ emit_data64(0x0504038002010080, relocInfo::none);
4758     __ emit_data64(0x0b0a098008070680, relocInfo::none);
4759 
4760     return start;
4761   }
4762 
4763   address base64_right_shift_mask_addr() {
4764     __ align(CodeEntryAlignment);
4765     StubCodeMark mark(this, "StubRoutines", "right_shift_mask");
4766     address start = __ pc();
4767     __ emit_data64(0x0006000400020000, relocInfo::none);
4768     __ emit_data64(0x0006000400020000, relocInfo::none);
4769     __ emit_data64(0x0006000400020000, relocInfo::none);
4770     __ emit_data64(0x0006000400020000, relocInfo::none);
4771     __ emit_data64(0x0006000400020000, relocInfo::none);
4772     __ emit_data64(0x0006000400020000, relocInfo::none);
4773     __ emit_data64(0x0006000400020000, relocInfo::none);
4774     __ emit_data64(0x0006000400020000, relocInfo::none);
4775 
4776     return start;
4777   }
4778 
4779   address base64_left_shift_mask_addr() {
4780     __ align(CodeEntryAlignment);
4781     StubCodeMark mark(this, "StubRoutines", "left_shift_mask");
4782     address start = __ pc();
4783     __ emit_data64(0x0000000200040000, relocInfo::none);
4784     __ emit_data64(0x0000000200040000, relocInfo::none);
4785     __ emit_data64(0x0000000200040000, relocInfo::none);
4786     __ emit_data64(0x0000000200040000, relocInfo::none);
4787     __ emit_data64(0x0000000200040000, relocInfo::none);
4788     __ emit_data64(0x0000000200040000, relocInfo::none);
4789     __ emit_data64(0x0000000200040000, relocInfo::none);
4790     __ emit_data64(0x0000000200040000, relocInfo::none);
4791 
4792     return start;
4793   }
4794 
4795   address base64_and_mask_addr() {
4796     __ align(CodeEntryAlignment);
4797     StubCodeMark mark(this, "StubRoutines", "and_mask");
4798     address start = __ pc();
4799     __ emit_data64(0x3f003f003f000000, relocInfo::none);
4800     __ emit_data64(0x3f003f003f000000, relocInfo::none);
4801     __ emit_data64(0x3f003f003f000000, relocInfo::none);
4802     __ emit_data64(0x3f003f003f000000, relocInfo::none);
4803     __ emit_data64(0x3f003f003f000000, relocInfo::none);
4804     __ emit_data64(0x3f003f003f000000, relocInfo::none);
4805     __ emit_data64(0x3f003f003f000000, relocInfo::none);
4806     __ emit_data64(0x3f003f003f000000, relocInfo::none);
4807     return start;
4808   }
4809 
4810   address base64_gather_mask_addr() {
4811     __ align(CodeEntryAlignment);
4812     StubCodeMark mark(this, "StubRoutines", "gather_mask");
4813     address start = __ pc();
4814     __ emit_data64(0xffffffffffffffff, relocInfo::none);
4815     return start;
4816   }
4817 
4818 // Code for generating Base64 encoding.
4819 // Intrinsic function prototype in Base64.java:
4820 // private void encodeBlock(byte[] src, int sp, int sl, byte[] dst, int dp, boolean isURL) {
4821   address generate_base64_encodeBlock() {
4822     __ align(CodeEntryAlignment);
4823     StubCodeMark mark(this, "StubRoutines", "implEncode");
4824     address start = __ pc();
4825     __ enter();
4826 
4827     // Save callee-saved registers before using them
4828     __ push(r12);
4829     __ push(r13);
4830     __ push(r14);
4831     __ push(r15);
4832 
4833     // arguments
4834     const Register source = c_rarg0; // Source Array
4835     const Register start_offset = c_rarg1; // start offset
4836     const Register end_offset = c_rarg2; // end offset
4837     const Register dest = c_rarg3; // destination array
4838 
4839 #ifndef _WIN64
4840     const Register dp = c_rarg4;  // Position for writing to dest array
4841     const Register isURL = c_rarg5;// Base64 or URL character set
4842 #else
4843     const Address  dp_mem(rbp, 6 * wordSize);  // length is on stack on Win64
4844     const Address isURL_mem(rbp, 7 * wordSize);
4845     const Register isURL = r10;      // pick the volatile windows register
4846     const Register dp = r12;
4847     __ movl(dp, dp_mem);
4848     __ movl(isURL, isURL_mem);
4849 #endif
4850 
4851     const Register length = r14;
4852     Label L_process80, L_process32, L_process3, L_exit, L_processdata;
4853 
4854     // calculate length from offsets
4855     __ movl(length, end_offset);
4856     __ subl(length, start_offset);
4857     __ cmpl(length, 0);
4858     __ jcc(Assembler::lessEqual, L_exit);
4859 
4860     __ lea(r11, ExternalAddress(StubRoutines::x86::base64_charset_addr()));
4861     // check if base64 charset(isURL=0) or base64 url charset(isURL=1) needs to be loaded
4862     __ cmpl(isURL, 0);
4863     __ jcc(Assembler::equal, L_processdata);
4864     __ lea(r11, ExternalAddress(StubRoutines::x86::base64url_charset_addr()));
4865 
4866     // load masks required for encoding data
4867     __ BIND(L_processdata);
4868     __ movdqu(xmm16, ExternalAddress(StubRoutines::x86::base64_gather_mask_addr()));
4869     // Set 64 bits of K register.
4870     __ evpcmpeqb(k3, xmm16, xmm16, Assembler::AVX_512bit);
4871     __ evmovdquq(xmm12, ExternalAddress(StubRoutines::x86::base64_bswap_mask_addr()), Assembler::AVX_256bit, r13);
4872     __ evmovdquq(xmm13, ExternalAddress(StubRoutines::x86::base64_right_shift_mask_addr()), Assembler::AVX_512bit, r13);
4873     __ evmovdquq(xmm14, ExternalAddress(StubRoutines::x86::base64_left_shift_mask_addr()), Assembler::AVX_512bit, r13);
4874     __ evmovdquq(xmm15, ExternalAddress(StubRoutines::x86::base64_and_mask_addr()), Assembler::AVX_512bit, r13);
4875 
4876     // Vector Base64 implementation, producing 96 bytes of encoded data
4877     __ BIND(L_process80);
4878     __ cmpl(length, 80);
4879     __ jcc(Assembler::below, L_process32);
4880     __ evmovdquq(xmm0, Address(source, start_offset, Address::times_1, 0), Assembler::AVX_256bit);
4881     __ evmovdquq(xmm1, Address(source, start_offset, Address::times_1, 24), Assembler::AVX_256bit);
4882     __ evmovdquq(xmm2, Address(source, start_offset, Address::times_1, 48), Assembler::AVX_256bit);
4883 
4884     //permute the input data in such a manner that we have continuity of the source
4885     __ vpermq(xmm3, xmm0, 148, Assembler::AVX_256bit);
4886     __ vpermq(xmm4, xmm1, 148, Assembler::AVX_256bit);
4887     __ vpermq(xmm5, xmm2, 148, Assembler::AVX_256bit);
4888 
4889     //shuffle input and group 3 bytes of data and to it add 0 as the 4th byte.
4890     //we can deal with 12 bytes at a time in a 128 bit register
4891     __ vpshufb(xmm3, xmm3, xmm12, Assembler::AVX_256bit);
4892     __ vpshufb(xmm4, xmm4, xmm12, Assembler::AVX_256bit);
4893     __ vpshufb(xmm5, xmm5, xmm12, Assembler::AVX_256bit);
4894 
4895     //convert byte to word. Each 128 bit register will have 6 bytes for processing
4896     __ vpmovzxbw(xmm3, xmm3, Assembler::AVX_512bit);
4897     __ vpmovzxbw(xmm4, xmm4, Assembler::AVX_512bit);
4898     __ vpmovzxbw(xmm5, xmm5, Assembler::AVX_512bit);
4899 
4900     // Extract bits in the following pattern 6, 4+2, 2+4, 6 to convert 3, 8 bit numbers to 4, 6 bit numbers
4901     __ evpsrlvw(xmm0, xmm3, xmm13,  Assembler::AVX_512bit);
4902     __ evpsrlvw(xmm1, xmm4, xmm13, Assembler::AVX_512bit);
4903     __ evpsrlvw(xmm2, xmm5, xmm13, Assembler::AVX_512bit);
4904 
4905     __ evpsllvw(xmm3, xmm3, xmm14, Assembler::AVX_512bit);
4906     __ evpsllvw(xmm4, xmm4, xmm14, Assembler::AVX_512bit);
4907     __ evpsllvw(xmm5, xmm5, xmm14, Assembler::AVX_512bit);
4908 
4909     __ vpsrlq(xmm0, xmm0, 8, Assembler::AVX_512bit);
4910     __ vpsrlq(xmm1, xmm1, 8, Assembler::AVX_512bit);
4911     __ vpsrlq(xmm2, xmm2, 8, Assembler::AVX_512bit);
4912 
4913     __ vpsllq(xmm3, xmm3, 8, Assembler::AVX_512bit);
4914     __ vpsllq(xmm4, xmm4, 8, Assembler::AVX_512bit);
4915     __ vpsllq(xmm5, xmm5, 8, Assembler::AVX_512bit);
4916 
4917     __ vpandq(xmm3, xmm3, xmm15, Assembler::AVX_512bit);
4918     __ vpandq(xmm4, xmm4, xmm15, Assembler::AVX_512bit);
4919     __ vpandq(xmm5, xmm5, xmm15, Assembler::AVX_512bit);
4920 
4921     // Get the final 4*6 bits base64 encoding
4922     __ vporq(xmm3, xmm3, xmm0, Assembler::AVX_512bit);
4923     __ vporq(xmm4, xmm4, xmm1, Assembler::AVX_512bit);
4924     __ vporq(xmm5, xmm5, xmm2, Assembler::AVX_512bit);
4925 
4926     // Shift
4927     __ vpsrlq(xmm3, xmm3, 8, Assembler::AVX_512bit);
4928     __ vpsrlq(xmm4, xmm4, 8, Assembler::AVX_512bit);
4929     __ vpsrlq(xmm5, xmm5, 8, Assembler::AVX_512bit);
4930 
4931     // look up 6 bits in the base64 character set to fetch the encoding
4932     // we are converting word to dword as gather instructions need dword indices for looking up encoding
4933     __ vextracti64x4(xmm6, xmm3, 0);
4934     __ vpmovzxwd(xmm0, xmm6, Assembler::AVX_512bit);
4935     __ vextracti64x4(xmm6, xmm3, 1);
4936     __ vpmovzxwd(xmm1, xmm6, Assembler::AVX_512bit);
4937 
4938     __ vextracti64x4(xmm6, xmm4, 0);
4939     __ vpmovzxwd(xmm2, xmm6, Assembler::AVX_512bit);
4940     __ vextracti64x4(xmm6, xmm4, 1);
4941     __ vpmovzxwd(xmm3, xmm6, Assembler::AVX_512bit);
4942 
4943     __ vextracti64x4(xmm4, xmm5, 0);
4944     __ vpmovzxwd(xmm6, xmm4, Assembler::AVX_512bit);
4945 
4946     __ vextracti64x4(xmm4, xmm5, 1);
4947     __ vpmovzxwd(xmm7, xmm4, Assembler::AVX_512bit);
4948 
4949     __ kmovql(k2, k3);
4950     __ evpgatherdd(xmm4, k2, Address(r11, xmm0, Address::times_4, 0), Assembler::AVX_512bit);
4951     __ kmovql(k2, k3);
4952     __ evpgatherdd(xmm5, k2, Address(r11, xmm1, Address::times_4, 0), Assembler::AVX_512bit);
4953     __ kmovql(k2, k3);
4954     __ evpgatherdd(xmm8, k2, Address(r11, xmm2, Address::times_4, 0), Assembler::AVX_512bit);
4955     __ kmovql(k2, k3);
4956     __ evpgatherdd(xmm9, k2, Address(r11, xmm3, Address::times_4, 0), Assembler::AVX_512bit);
4957     __ kmovql(k2, k3);
4958     __ evpgatherdd(xmm10, k2, Address(r11, xmm6, Address::times_4, 0), Assembler::AVX_512bit);
4959     __ kmovql(k2, k3);
4960     __ evpgatherdd(xmm11, k2, Address(r11, xmm7, Address::times_4, 0), Assembler::AVX_512bit);
4961 
4962     //Down convert dword to byte. Final output is 16*6 = 96 bytes long
4963     __ evpmovdb(Address(dest, dp, Address::times_1, 0), xmm4, Assembler::AVX_512bit);
4964     __ evpmovdb(Address(dest, dp, Address::times_1, 16), xmm5, Assembler::AVX_512bit);
4965     __ evpmovdb(Address(dest, dp, Address::times_1, 32), xmm8, Assembler::AVX_512bit);
4966     __ evpmovdb(Address(dest, dp, Address::times_1, 48), xmm9, Assembler::AVX_512bit);
4967     __ evpmovdb(Address(dest, dp, Address::times_1, 64), xmm10, Assembler::AVX_512bit);
4968     __ evpmovdb(Address(dest, dp, Address::times_1, 80), xmm11, Assembler::AVX_512bit);
4969 
4970     __ addq(dest, 96);
4971     __ addq(source, 72);
4972     __ subq(length, 72);
4973     __ jmp(L_process80);
4974 
4975     // Vector Base64 implementation generating 32 bytes of encoded data
4976     __ BIND(L_process32);
4977     __ cmpl(length, 32);
4978     __ jcc(Assembler::below, L_process3);
4979     __ evmovdquq(xmm0, Address(source, start_offset), Assembler::AVX_256bit);
4980     __ vpermq(xmm0, xmm0, 148, Assembler::AVX_256bit);
4981     __ vpshufb(xmm6, xmm0, xmm12, Assembler::AVX_256bit);
4982     __ vpmovzxbw(xmm6, xmm6, Assembler::AVX_512bit);
4983     __ evpsrlvw(xmm2, xmm6, xmm13, Assembler::AVX_512bit);
4984     __ evpsllvw(xmm3, xmm6, xmm14, Assembler::AVX_512bit);
4985 
4986     __ vpsrlq(xmm2, xmm2, 8, Assembler::AVX_512bit);
4987     __ vpsllq(xmm3, xmm3, 8, Assembler::AVX_512bit);
4988     __ vpandq(xmm3, xmm3, xmm15, Assembler::AVX_512bit);
4989     __ vporq(xmm1, xmm2, xmm3, Assembler::AVX_512bit);
4990     __ vpsrlq(xmm1, xmm1, 8, Assembler::AVX_512bit);
4991     __ vextracti64x4(xmm9, xmm1, 0);
4992     __ vpmovzxwd(xmm6, xmm9, Assembler::AVX_512bit);
4993     __ vextracti64x4(xmm9, xmm1, 1);
4994     __ vpmovzxwd(xmm5, xmm9,  Assembler::AVX_512bit);
4995     __ kmovql(k2, k3);
4996     __ evpgatherdd(xmm8, k2, Address(r11, xmm6, Address::times_4, 0), Assembler::AVX_512bit);
4997     __ kmovql(k2, k3);
4998     __ evpgatherdd(xmm10, k2, Address(r11, xmm5, Address::times_4, 0), Assembler::AVX_512bit);
4999     __ evpmovdb(Address(dest, dp, Address::times_1, 0), xmm8, Assembler::AVX_512bit);
5000     __ evpmovdb(Address(dest, dp, Address::times_1, 16), xmm10, Assembler::AVX_512bit);
5001     __ subq(length, 24);
5002     __ addq(dest, 32);
5003     __ addq(source, 24);
5004     __ jmp(L_process32);
5005 
5006     // Scalar data processing takes 3 bytes at a time and produces 4 bytes of encoded data
5007     /* This code corresponds to the scalar version of the following snippet in Base64.java
5008     ** int bits = (src[sp0++] & 0xff) << 16 |(src[sp0++] & 0xff) << 8 |(src[sp0++] & 0xff);
5009     ** dst[dp0++] = (byte)base64[(bits >> > 18) & 0x3f];
5010     ** dst[dp0++] = (byte)base64[(bits >> > 12) & 0x3f];
5011     ** dst[dp0++] = (byte)base64[(bits >> > 6) & 0x3f];
5012     ** dst[dp0++] = (byte)base64[bits & 0x3f];*/
5013     __ BIND(L_process3);
5014     __ cmpl(length, 3);
5015     __ jcc(Assembler::below, L_exit);
5016     // Read 1 byte at a time
5017     __ movzbl(rax, Address(source, start_offset));
5018     __ shll(rax, 0x10);
5019     __ movl(r15, rax);
5020     __ movzbl(rax, Address(source, start_offset, Address::times_1, 1));
5021     __ shll(rax, 0x8);
5022     __ movzwl(rax, rax);
5023     __ orl(r15, rax);
5024     __ movzbl(rax, Address(source, start_offset, Address::times_1, 2));
5025     __ orl(rax, r15);
5026     // Save 3 bytes read in r15
5027     __ movl(r15, rax);
5028     __ shrl(rax, 0x12);
5029     __ andl(rax, 0x3f);
5030     // rax contains the index, r11 contains base64 lookup table
5031     __ movb(rax, Address(r11, rax, Address::times_4));
5032     // Write the encoded byte to destination
5033     __ movb(Address(dest, dp, Address::times_1, 0), rax);
5034     __ movl(rax, r15);
5035     __ shrl(rax, 0xc);
5036     __ andl(rax, 0x3f);
5037     __ movb(rax, Address(r11, rax, Address::times_4));
5038     __ movb(Address(dest, dp, Address::times_1, 1), rax);
5039     __ movl(rax, r15);
5040     __ shrl(rax, 0x6);
5041     __ andl(rax, 0x3f);
5042     __ movb(rax, Address(r11, rax, Address::times_4));
5043     __ movb(Address(dest, dp, Address::times_1, 2), rax);
5044     __ movl(rax, r15);
5045     __ andl(rax, 0x3f);
5046     __ movb(rax, Address(r11, rax, Address::times_4));
5047     __ movb(Address(dest, dp, Address::times_1, 3), rax);
5048     __ subl(length, 3);
5049     __ addq(dest, 4);
5050     __ addq(source, 3);
5051     __ jmp(L_process3);
5052     __ BIND(L_exit);
5053     __ pop(r15);
5054     __ pop(r14);
5055     __ pop(r13);
5056     __ pop(r12);
5057     __ leave();
5058     __ ret(0);
5059     return start;
5060   }
5061 
5062   /**
5063    *  Arguments:
5064    *
5065    * Inputs:
5066    *   c_rarg0   - int crc
5067    *   c_rarg1   - byte* buf
5068    *   c_rarg2   - int length
5069    *
5070    * Ouput:
5071    *       rax   - int crc result
5072    */
5073   address generate_updateBytesCRC32() {
5074     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
5075 
5076     __ align(CodeEntryAlignment);
5077     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
5078 
5079     address start = __ pc();
5080     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
5081     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
5082     // rscratch1: r10
5083     const Register crc   = c_rarg0;  // crc
5084     const Register buf   = c_rarg1;  // source java byte array address
5085     const Register len   = c_rarg2;  // length
5086     const Register table = c_rarg3;  // crc_table address (reuse register)
5087     const Register tmp   = r11;
5088     assert_different_registers(crc, buf, len, table, tmp, rax);
5089 
5090     BLOCK_COMMENT("Entry:");
5091     __ enter(); // required for proper stackwalking of RuntimeStub frame
5092 
5093     __ kernel_crc32(crc, buf, len, table, tmp);
5094 
5095     __ movl(rax, crc);
5096     __ vzeroupper();
5097     __ leave(); // required for proper stackwalking of RuntimeStub frame
5098     __ ret(0);
5099 
5100     return start;
5101   }
5102 
5103   /**
5104   *  Arguments:
5105   *
5106   * Inputs:
5107   *   c_rarg0   - int crc
5108   *   c_rarg1   - byte* buf
5109   *   c_rarg2   - long length
5110   *   c_rarg3   - table_start - optional (present only when doing a library_call,
5111   *              not used by x86 algorithm)
5112   *
5113   * Ouput:
5114   *       rax   - int crc result
5115   */
5116   address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
5117       assert(UseCRC32CIntrinsics, "need SSE4_2");
5118       __ align(CodeEntryAlignment);
5119       StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
5120       address start = __ pc();
5121       //reg.arg        int#0        int#1        int#2        int#3        int#4        int#5        float regs
5122       //Windows        RCX          RDX          R8           R9           none         none         XMM0..XMM3
5123       //Lin / Sol      RDI          RSI          RDX          RCX          R8           R9           XMM0..XMM7
5124       const Register crc = c_rarg0;  // crc
5125       const Register buf = c_rarg1;  // source java byte array address
5126       const Register len = c_rarg2;  // length
5127       const Register a = rax;
5128       const Register j = r9;
5129       const Register k = r10;
5130       const Register l = r11;
5131 #ifdef _WIN64
5132       const Register y = rdi;
5133       const Register z = rsi;
5134 #else
5135       const Register y = rcx;
5136       const Register z = r8;
5137 #endif
5138       assert_different_registers(crc, buf, len, a, j, k, l, y, z);
5139 
5140       BLOCK_COMMENT("Entry:");
5141       __ enter(); // required for proper stackwalking of RuntimeStub frame
5142 #ifdef _WIN64
5143       __ push(y);
5144       __ push(z);
5145 #endif
5146       __ crc32c_ipl_alg2_alt2(crc, buf, len,
5147                               a, j, k,
5148                               l, y, z,
5149                               c_farg0, c_farg1, c_farg2,
5150                               is_pclmulqdq_supported);
5151       __ movl(rax, crc);
5152 #ifdef _WIN64
5153       __ pop(z);
5154       __ pop(y);
5155 #endif
5156       __ vzeroupper();
5157       __ leave(); // required for proper stackwalking of RuntimeStub frame
5158       __ ret(0);
5159 
5160       return start;
5161   }
5162 
5163   /**
5164    *  Arguments:
5165    *
5166    *  Input:
5167    *    c_rarg0   - x address
5168    *    c_rarg1   - x length
5169    *    c_rarg2   - y address
5170    *    c_rarg3   - y length
5171    * not Win64
5172    *    c_rarg4   - z address
5173    *    c_rarg5   - z length
5174    * Win64
5175    *    rsp+40    - z address
5176    *    rsp+48    - z length
5177    */
5178   address generate_multiplyToLen() {
5179     __ align(CodeEntryAlignment);
5180     StubCodeMark mark(this, "StubRoutines", "multiplyToLen");
5181 
5182     address start = __ pc();
5183     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
5184     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
5185     const Register x     = rdi;
5186     const Register xlen  = rax;
5187     const Register y     = rsi;
5188     const Register ylen  = rcx;
5189     const Register z     = r8;
5190     const Register zlen  = r11;
5191 
5192     // Next registers will be saved on stack in multiply_to_len().
5193     const Register tmp1  = r12;
5194     const Register tmp2  = r13;
5195     const Register tmp3  = r14;
5196     const Register tmp4  = r15;
5197     const Register tmp5  = rbx;
5198 
5199     BLOCK_COMMENT("Entry:");
5200     __ enter(); // required for proper stackwalking of RuntimeStub frame
5201 
5202 #ifndef _WIN64
5203     __ movptr(zlen, r9); // Save r9 in r11 - zlen
5204 #endif
5205     setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx
5206                        // ylen => rcx, z => r8, zlen => r11
5207                        // r9 and r10 may be used to save non-volatile registers
5208 #ifdef _WIN64
5209     // last 2 arguments (#4, #5) are on stack on Win64
5210     __ movptr(z, Address(rsp, 6 * wordSize));
5211     __ movptr(zlen, Address(rsp, 7 * wordSize));
5212 #endif
5213 
5214     __ movptr(xlen, rsi);
5215     __ movptr(y,    rdx);
5216     __ multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5);
5217 
5218     restore_arg_regs();
5219 
5220     __ leave(); // required for proper stackwalking of RuntimeStub frame
5221     __ ret(0);
5222 
5223     return start;
5224   }
5225 
5226   /**
5227   *  Arguments:
5228   *
5229   *  Input:
5230   *    c_rarg0   - obja     address
5231   *    c_rarg1   - objb     address
5232   *    c_rarg3   - length   length
5233   *    c_rarg4   - scale    log2_array_indxscale
5234   *
5235   *  Output:
5236   *        rax   - int >= mismatched index, < 0 bitwise complement of tail
5237   */
5238   address generate_vectorizedMismatch() {
5239     __ align(CodeEntryAlignment);
5240     StubCodeMark mark(this, "StubRoutines", "vectorizedMismatch");
5241     address start = __ pc();
5242 
5243     BLOCK_COMMENT("Entry:");
5244     __ enter();
5245 
5246 #ifdef _WIN64  // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
5247     const Register scale = c_rarg0;  //rcx, will exchange with r9
5248     const Register objb = c_rarg1;   //rdx
5249     const Register length = c_rarg2; //r8
5250     const Register obja = c_rarg3;   //r9
5251     __ xchgq(obja, scale);  //now obja and scale contains the correct contents
5252 
5253     const Register tmp1 = r10;
5254     const Register tmp2 = r11;
5255 #endif
5256 #ifndef _WIN64 // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
5257     const Register obja = c_rarg0;   //U:rdi
5258     const Register objb = c_rarg1;   //U:rsi
5259     const Register length = c_rarg2; //U:rdx
5260     const Register scale = c_rarg3;  //U:rcx
5261     const Register tmp1 = r8;
5262     const Register tmp2 = r9;
5263 #endif
5264     const Register result = rax; //return value
5265     const XMMRegister vec0 = xmm0;
5266     const XMMRegister vec1 = xmm1;
5267     const XMMRegister vec2 = xmm2;
5268 
5269     __ vectorized_mismatch(obja, objb, length, scale, result, tmp1, tmp2, vec0, vec1, vec2);
5270 
5271     __ vzeroupper();
5272     __ leave();
5273     __ ret(0);
5274 
5275     return start;
5276   }
5277 
5278 /**
5279    *  Arguments:
5280    *
5281   //  Input:
5282   //    c_rarg0   - x address
5283   //    c_rarg1   - x length
5284   //    c_rarg2   - z address
5285   //    c_rarg3   - z lenth
5286    *
5287    */
5288   address generate_squareToLen() {
5289 
5290     __ align(CodeEntryAlignment);
5291     StubCodeMark mark(this, "StubRoutines", "squareToLen");
5292 
5293     address start = __ pc();
5294     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
5295     // Unix:  rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...)
5296     const Register x      = rdi;
5297     const Register len    = rsi;
5298     const Register z      = r8;
5299     const Register zlen   = rcx;
5300 
5301    const Register tmp1      = r12;
5302    const Register tmp2      = r13;
5303    const Register tmp3      = r14;
5304    const Register tmp4      = r15;
5305    const Register tmp5      = rbx;
5306 
5307     BLOCK_COMMENT("Entry:");
5308     __ enter(); // required for proper stackwalking of RuntimeStub frame
5309 
5310     setup_arg_regs(4); // x => rdi, len => rsi, z => rdx
5311                        // zlen => rcx
5312                        // r9 and r10 may be used to save non-volatile registers
5313     __ movptr(r8, rdx);
5314     __ square_to_len(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
5315 
5316     restore_arg_regs();
5317 
5318     __ leave(); // required for proper stackwalking of RuntimeStub frame
5319     __ ret(0);
5320 
5321     return start;
5322   }
5323 
5324   address generate_method_entry_barrier() {
5325     __ align(CodeEntryAlignment);
5326     StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier");
5327 
5328     Label deoptimize_label;
5329 
5330     address start = __ pc();
5331 
5332     __ push(-1); // cookie, this is used for writing the new rsp when deoptimizing
5333 
5334     BLOCK_COMMENT("Entry:");
5335     __ enter(); // save rbp
5336 
5337     // save c_rarg0, because we want to use that value.
5338     // We could do without it but then we depend on the number of slots used by pusha
5339     __ push(c_rarg0);
5340 
5341     __ lea(c_rarg0, Address(rsp, wordSize * 3)); // 1 for cookie, 1 for rbp, 1 for c_rarg0 - this should be the return address
5342 
5343     __ pusha();
5344 
5345     // The method may have floats as arguments, and we must spill them before calling
5346     // the VM runtime.
5347     assert(Argument::n_float_register_parameters_j == 8, "Assumption");
5348     const int xmm_size = wordSize * 2;
5349     const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j;
5350     __ subptr(rsp, xmm_spill_size);
5351     __ movdqu(Address(rsp, xmm_size * 7), xmm7);
5352     __ movdqu(Address(rsp, xmm_size * 6), xmm6);
5353     __ movdqu(Address(rsp, xmm_size * 5), xmm5);
5354     __ movdqu(Address(rsp, xmm_size * 4), xmm4);
5355     __ movdqu(Address(rsp, xmm_size * 3), xmm3);
5356     __ movdqu(Address(rsp, xmm_size * 2), xmm2);
5357     __ movdqu(Address(rsp, xmm_size * 1), xmm1);
5358     __ movdqu(Address(rsp, xmm_size * 0), xmm0);
5359 
5360     __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(address*)>(BarrierSetNMethod::nmethod_stub_entry_barrier)), 1);
5361 
5362     __ movdqu(xmm0, Address(rsp, xmm_size * 0));
5363     __ movdqu(xmm1, Address(rsp, xmm_size * 1));
5364     __ movdqu(xmm2, Address(rsp, xmm_size * 2));
5365     __ movdqu(xmm3, Address(rsp, xmm_size * 3));
5366     __ movdqu(xmm4, Address(rsp, xmm_size * 4));
5367     __ movdqu(xmm5, Address(rsp, xmm_size * 5));
5368     __ movdqu(xmm6, Address(rsp, xmm_size * 6));
5369     __ movdqu(xmm7, Address(rsp, xmm_size * 7));
5370     __ addptr(rsp, xmm_spill_size);
5371 
5372     __ cmpl(rax, 1); // 1 means deoptimize
5373     __ jcc(Assembler::equal, deoptimize_label);
5374 
5375     __ popa();
5376     __ pop(c_rarg0);
5377 
5378     __ leave();
5379 
5380     __ addptr(rsp, 1 * wordSize); // cookie
5381     __ ret(0);
5382 
5383 
5384     __ BIND(deoptimize_label);
5385 
5386     __ popa();
5387     __ pop(c_rarg0);
5388 
5389     __ leave();
5390 
5391     // this can be taken out, but is good for verification purposes. getting a SIGSEGV
5392     // here while still having a correct stack is valuable
5393     __ testptr(rsp, Address(rsp, 0));
5394 
5395     __ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier
5396     __ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point
5397 
5398     return start;
5399   }
5400 
5401    /**
5402    *  Arguments:
5403    *
5404    *  Input:
5405    *    c_rarg0   - out address
5406    *    c_rarg1   - in address
5407    *    c_rarg2   - offset
5408    *    c_rarg3   - len
5409    * not Win64
5410    *    c_rarg4   - k
5411    * Win64
5412    *    rsp+40    - k
5413    */
5414   address generate_mulAdd() {
5415     __ align(CodeEntryAlignment);
5416     StubCodeMark mark(this, "StubRoutines", "mulAdd");
5417 
5418     address start = __ pc();
5419     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
5420     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
5421     const Register out     = rdi;
5422     const Register in      = rsi;
5423     const Register offset  = r11;
5424     const Register len     = rcx;
5425     const Register k       = r8;
5426 
5427     // Next registers will be saved on stack in mul_add().
5428     const Register tmp1  = r12;
5429     const Register tmp2  = r13;
5430     const Register tmp3  = r14;
5431     const Register tmp4  = r15;
5432     const Register tmp5  = rbx;
5433 
5434     BLOCK_COMMENT("Entry:");
5435     __ enter(); // required for proper stackwalking of RuntimeStub frame
5436 
5437     setup_arg_regs(4); // out => rdi, in => rsi, offset => rdx
5438                        // len => rcx, k => r8
5439                        // r9 and r10 may be used to save non-volatile registers
5440 #ifdef _WIN64
5441     // last argument is on stack on Win64
5442     __ movl(k, Address(rsp, 6 * wordSize));
5443 #endif
5444     __ movptr(r11, rdx);  // move offset in rdx to offset(r11)
5445     __ mul_add(out, in, offset, len, k, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
5446 
5447     restore_arg_regs();
5448 
5449     __ leave(); // required for proper stackwalking of RuntimeStub frame
5450     __ ret(0);
5451 
5452     return start;
5453   }
5454 
5455   address generate_libmExp() {
5456     StubCodeMark mark(this, "StubRoutines", "libmExp");
5457 
5458     address start = __ pc();
5459 
5460     const XMMRegister x0  = xmm0;
5461     const XMMRegister x1  = xmm1;
5462     const XMMRegister x2  = xmm2;
5463     const XMMRegister x3  = xmm3;
5464 
5465     const XMMRegister x4  = xmm4;
5466     const XMMRegister x5  = xmm5;
5467     const XMMRegister x6  = xmm6;
5468     const XMMRegister x7  = xmm7;
5469 
5470     const Register tmp   = r11;
5471 
5472     BLOCK_COMMENT("Entry:");
5473     __ enter(); // required for proper stackwalking of RuntimeStub frame
5474 
5475     __ fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
5476 
5477     __ leave(); // required for proper stackwalking of RuntimeStub frame
5478     __ ret(0);
5479 
5480     return start;
5481 
5482   }
5483 
5484   address generate_libmLog() {
5485     StubCodeMark mark(this, "StubRoutines", "libmLog");
5486 
5487     address start = __ pc();
5488 
5489     const XMMRegister x0 = xmm0;
5490     const XMMRegister x1 = xmm1;
5491     const XMMRegister x2 = xmm2;
5492     const XMMRegister x3 = xmm3;
5493 
5494     const XMMRegister x4 = xmm4;
5495     const XMMRegister x5 = xmm5;
5496     const XMMRegister x6 = xmm6;
5497     const XMMRegister x7 = xmm7;
5498 
5499     const Register tmp1 = r11;
5500     const Register tmp2 = r8;
5501 
5502     BLOCK_COMMENT("Entry:");
5503     __ enter(); // required for proper stackwalking of RuntimeStub frame
5504 
5505     __ fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2);
5506 
5507     __ leave(); // required for proper stackwalking of RuntimeStub frame
5508     __ ret(0);
5509 
5510     return start;
5511 
5512   }
5513 
5514   address generate_libmLog10() {
5515     StubCodeMark mark(this, "StubRoutines", "libmLog10");
5516 
5517     address start = __ pc();
5518 
5519     const XMMRegister x0 = xmm0;
5520     const XMMRegister x1 = xmm1;
5521     const XMMRegister x2 = xmm2;
5522     const XMMRegister x3 = xmm3;
5523 
5524     const XMMRegister x4 = xmm4;
5525     const XMMRegister x5 = xmm5;
5526     const XMMRegister x6 = xmm6;
5527     const XMMRegister x7 = xmm7;
5528 
5529     const Register tmp = r11;
5530 
5531     BLOCK_COMMENT("Entry:");
5532     __ enter(); // required for proper stackwalking of RuntimeStub frame
5533 
5534     __ fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
5535 
5536     __ leave(); // required for proper stackwalking of RuntimeStub frame
5537     __ ret(0);
5538 
5539     return start;
5540 
5541   }
5542 
5543   address generate_libmPow() {
5544     StubCodeMark mark(this, "StubRoutines", "libmPow");
5545 
5546     address start = __ pc();
5547 
5548     const XMMRegister x0 = xmm0;
5549     const XMMRegister x1 = xmm1;
5550     const XMMRegister x2 = xmm2;
5551     const XMMRegister x3 = xmm3;
5552 
5553     const XMMRegister x4 = xmm4;
5554     const XMMRegister x5 = xmm5;
5555     const XMMRegister x6 = xmm6;
5556     const XMMRegister x7 = xmm7;
5557 
5558     const Register tmp1 = r8;
5559     const Register tmp2 = r9;
5560     const Register tmp3 = r10;
5561     const Register tmp4 = r11;
5562 
5563     BLOCK_COMMENT("Entry:");
5564     __ enter(); // required for proper stackwalking of RuntimeStub frame
5565 
5566     __ fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
5567 
5568     __ leave(); // required for proper stackwalking of RuntimeStub frame
5569     __ ret(0);
5570 
5571     return start;
5572 
5573   }
5574 
5575   address generate_libmSin() {
5576     StubCodeMark mark(this, "StubRoutines", "libmSin");
5577 
5578     address start = __ pc();
5579 
5580     const XMMRegister x0 = xmm0;
5581     const XMMRegister x1 = xmm1;
5582     const XMMRegister x2 = xmm2;
5583     const XMMRegister x3 = xmm3;
5584 
5585     const XMMRegister x4 = xmm4;
5586     const XMMRegister x5 = xmm5;
5587     const XMMRegister x6 = xmm6;
5588     const XMMRegister x7 = xmm7;
5589 
5590     const Register tmp1 = r8;
5591     const Register tmp2 = r9;
5592     const Register tmp3 = r10;
5593     const Register tmp4 = r11;
5594 
5595     BLOCK_COMMENT("Entry:");
5596     __ enter(); // required for proper stackwalking of RuntimeStub frame
5597 
5598 #ifdef _WIN64
5599     __ push(rsi);
5600     __ push(rdi);
5601 #endif
5602     __ fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
5603 
5604 #ifdef _WIN64
5605     __ pop(rdi);
5606     __ pop(rsi);
5607 #endif
5608 
5609     __ leave(); // required for proper stackwalking of RuntimeStub frame
5610     __ ret(0);
5611 
5612     return start;
5613 
5614   }
5615 
5616   address generate_libmCos() {
5617     StubCodeMark mark(this, "StubRoutines", "libmCos");
5618 
5619     address start = __ pc();
5620 
5621     const XMMRegister x0 = xmm0;
5622     const XMMRegister x1 = xmm1;
5623     const XMMRegister x2 = xmm2;
5624     const XMMRegister x3 = xmm3;
5625 
5626     const XMMRegister x4 = xmm4;
5627     const XMMRegister x5 = xmm5;
5628     const XMMRegister x6 = xmm6;
5629     const XMMRegister x7 = xmm7;
5630 
5631     const Register tmp1 = r8;
5632     const Register tmp2 = r9;
5633     const Register tmp3 = r10;
5634     const Register tmp4 = r11;
5635 
5636     BLOCK_COMMENT("Entry:");
5637     __ enter(); // required for proper stackwalking of RuntimeStub frame
5638 
5639 #ifdef _WIN64
5640     __ push(rsi);
5641     __ push(rdi);
5642 #endif
5643     __ fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
5644 
5645 #ifdef _WIN64
5646     __ pop(rdi);
5647     __ pop(rsi);
5648 #endif
5649 
5650     __ leave(); // required for proper stackwalking of RuntimeStub frame
5651     __ ret(0);
5652 
5653     return start;
5654 
5655   }
5656 
5657   address generate_libmTan() {
5658     StubCodeMark mark(this, "StubRoutines", "libmTan");
5659 
5660     address start = __ pc();
5661 
5662     const XMMRegister x0 = xmm0;
5663     const XMMRegister x1 = xmm1;
5664     const XMMRegister x2 = xmm2;
5665     const XMMRegister x3 = xmm3;
5666 
5667     const XMMRegister x4 = xmm4;
5668     const XMMRegister x5 = xmm5;
5669     const XMMRegister x6 = xmm6;
5670     const XMMRegister x7 = xmm7;
5671 
5672     const Register tmp1 = r8;
5673     const Register tmp2 = r9;
5674     const Register tmp3 = r10;
5675     const Register tmp4 = r11;
5676 
5677     BLOCK_COMMENT("Entry:");
5678     __ enter(); // required for proper stackwalking of RuntimeStub frame
5679 
5680 #ifdef _WIN64
5681     __ push(rsi);
5682     __ push(rdi);
5683 #endif
5684     __ fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
5685 
5686 #ifdef _WIN64
5687     __ pop(rdi);
5688     __ pop(rsi);
5689 #endif
5690 
5691     __ leave(); // required for proper stackwalking of RuntimeStub frame
5692     __ ret(0);
5693 
5694     return start;
5695 
5696   }
5697 
5698 void push_FrameInfo(MacroAssembler* _masm, Register fi, Register sp, Register fp, address pc) {
5699   if (!sp->is_valid()) { __ push(0); } else {
5700     if (sp == rsp) {
5701       __ movptr(fi, rsp);
5702       __ push(fi);
5703     } else {
5704       __ push(sp);
5705     }
5706   }
5707 
5708   if (!fp->is_valid()) __ push(0); else __ push(fp);
5709 
5710   __ lea(fi, ExternalAddress(pc));
5711   __ push(fi);
5712 
5713   __ movptr(fi, rsp); // make fi point to the beginning of FramInfo
5714 }
5715 
5716 void push_FrameInfo(MacroAssembler* _masm, Register fi, Register sp, Register fp, Register pc) {
5717   if (!sp->is_valid()) { __ push(0); } else {
5718     if (sp == rsp) {
5719       __ movptr(fi, rsp);
5720       __ push(fi);
5721     } else {
5722       __ push(sp);
5723     }
5724   }
5725 
5726   if (!fp->is_valid()) __ push(0); else __ push(fp);
5727 
5728   if (!pc->is_valid()) __ push(0); else __ push(pc);
5729 
5730   __ movptr(fi, rsp); // make fi point to the beginning of FramInfo
5731 }
5732 
5733 void pop_FrameInfo(MacroAssembler* _masm, Register sp, Register fp, Register pc) {
5734   if (!pc->is_valid()) __ lea(rsp, Address(rsp, wordSize)); else __ pop(pc);
5735   if (!fp->is_valid()) __ lea(rsp, Address(rsp, wordSize)); else __ pop(fp);
5736   if (!sp->is_valid()) __ lea(rsp, Address(rsp, wordSize)); else __ pop(sp);
5737 }
5738 
5739 static Register get_thread() {
5740 #ifdef _LP64
5741   return r15_thread;
5742 #else
5743   get_thread(rdi);
5744   return rdi;
5745 #endif // LP64
5746 }
5747 
5748 static void setup_freeze_invocation(MacroAssembler* _masm, address pc) {
5749   Register thread = get_thread();
5750   NOT_LP64(__ push(thread));
5751   LP64_ONLY(__ movptr(c_rarg0, thread));
5752   __ set_last_Java_frame(rsp, rbp, pc);
5753 }
5754 
5755 static void teardown_freeze_invocation(MacroAssembler* _masm) {
5756   __ reset_last_Java_frame(true);
5757   NOT_LP64(__ pop(rdi));
5758 }
5759 
5760 // c_rarg1 is from interpreter
5761 RuntimeStub* generate_cont_doYield() {
5762     const char *name = "cont_doYield";
5763 
5764     enum layout {
5765       frameinfo_11,
5766       frameinfo_12,
5767       frameinfo_21,
5768       frameinfo_22,
5769       frameinfo_31,
5770       frameinfo_32,
5771       rbp_off,
5772       rbpH_off,
5773       return_off,
5774       return_off2,
5775       framesize // inclusive of return address
5776     };
5777     // assert(is_even(framesize/2), "sp not 16-byte aligned");
5778     int insts_size = 512;
5779     int locs_size  = 64;
5780     CodeBuffer code(name, insts_size, locs_size);
5781     OopMapSet* oop_maps  = new OopMapSet();
5782     MacroAssembler* masm = new MacroAssembler(&code);
5783     MacroAssembler* _masm = masm;
5784 
5785     // MacroAssembler* masm = _masm;
5786     // StubCodeMark mark(this, "StubRoutines", name);
5787 
5788     // second argument is the FrameInfo
5789     Register fi = c_rarg1;
5790 
5791     address start = __ pc();
5792 
5793     __ movl(c_rarg2, c_rarg1);          // save from interpreter
5794     __ movptr(rax, Address(rsp, 0));    // use return address as the frame pc // __ lea(rax, InternalAddress(pcxxxx));
5795     __ lea(fi, Address(rsp, wordSize)); // skip return address
5796     __ movptr(c_rarg3, rbp);
5797 
5798     // __ stop("FFFFF");
5799     __ enter();
5800 
5801     // // return address and rbp are already in place
5802     // __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
5803 
5804     push_FrameInfo(masm, fi, fi, c_rarg3, rax);
5805 
5806     int frame_complete = __ pc() - start;
5807     address the_pc = __ pc();
5808 
5809     __ post_call_nop(); // this must be exactly after the pc value that is pushed into the frame info, we use this nop for fast CodeBlob lookup
5810 
5811     if (ContPerfTest > 5) {
5812       setup_freeze_invocation(_masm, the_pc);
5813       __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::freeze), 3);
5814       teardown_freeze_invocation(_masm);
5815 
5816       // if (from_java) {
5817       //__ set_last_Java_frame(rsp, rbp, the_pc); // may be unnecessary. also, consider MacroAssembler::call_VM_leaf_base
5818       //__ call_VM(noreg, CAST_FROM_FN_PTR(address, Continuation::freeze), fi, false); // do NOT check exceptions; they'll get forwarded to the caller
5819       // } else {
5820       //   __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::freeze_C), fi);
5821       // }
5822     }
5823 
5824     Label pinned;
5825     __ pop(c_rarg2); // read the pc from the FrameInfo
5826     if (ContPerfTest <= 5) { __ xorq(c_rarg2, c_rarg2); __ xorq(rax, rax); }
5827     __ testq(c_rarg2, c_rarg2);
5828     __ jcc(Assembler::zero, pinned);
5829 
5830     __ pop(rbp); // not pinned -- jump to Continuation.run (the entry frame)
5831     __ movptr(rbp, Address(rbp, 0)); // frame_info->fp has an indirection here. See Continuation::freeze for an explanation.
5832     __ pop(fi);
5833     __ movptr(rsp, fi);
5834     __ jmp(c_rarg2);
5835 
5836     __ bind(pinned); // pinned -- return to caller
5837     __ lea(rsp, Address(rsp, wordSize*2)); // "pop" the rest of the FrameInfo struct
5838 
5839     __ leave();
5840     __ ret(0);
5841 
5842     // return start;
5843 
5844     OopMap* map = new OopMap(framesize, 1);
5845     // map->set_callee_saved(VMRegImpl::stack2reg(rbp_off), rbp->as_VMReg());
5846     oop_maps->add_gc_map(the_pc - start, map);
5847 
5848     RuntimeStub* stub = // codeBlob framesize is in words (not VMRegImpl::slot_size)
5849     RuntimeStub::new_runtime_stub(name,
5850                                   &code,
5851                                   frame_complete,
5852                                   (framesize >> (LogBytesPerWord - LogBytesPerInt)),
5853                                   oop_maps, false);
5854     return stub;
5855   }
5856 
5857   address generate_cont_jump_from_safepoint() {
5858     StubCodeMark mark(this, "StubRoutines","Continuation jump from safepoint");
5859 
5860     Register fi = rbx;
5861 
5862     address start = __ pc();
5863 
5864     __ get_thread(r15_thread);
5865     __ reset_last_Java_frame(true); // false would be fine, too, I guess
5866 
5867     __ lea(fi, Address(r15_thread, JavaThread::cont_frame_offset()));
5868     __ movptr(rdx, Address(fi, wordSize*0)); // pc
5869     __ movptr(rbp, Address(fi, wordSize*1)); // fp
5870     __ movptr(rbp, Address(rbp, 0)); // fp is indirect. See Continuation::freeze for an explanation.
5871     __ movptr(rsp, Address(fi, wordSize*2)); // sp
5872 
5873     __ xorq(rax, rax);
5874     __ movptr(Address(fi, wordSize*0), rax); // pc
5875     __ movptr(Address(fi, wordSize*1), rax); // fp
5876     __ movptr(Address(fi, wordSize*2), rax); // sp
5877     __ movb(Address(r15_thread, JavaThread::cont_preempt_offset()), 0);
5878 
5879     __ jmp(rdx);
5880 
5881     return start;
5882   }
5883 
5884   // c_rarg1 - sp
5885   // c_rarg2 - fp
5886   // c_rarg3 - pc
5887   address generate_cont_jump() {
5888     StubCodeMark mark(this, "StubRoutines","Continuation Jump");
5889     address start = __ pc();
5890 
5891     __ movptr(rbp, c_rarg2);
5892     __ movptr(rbp, Address(rbp, 0)); // rbp is indirect. See Continuation::freeze for an explanation.
5893     __ movptr(rsp, c_rarg1);
5894     __ jmp(c_rarg3);
5895 
5896     return start;
5897   }
5898 
5899   address generate_cont_thaw(bool return_barrier, bool exception) {
5900     assert (return_barrier || !exception, "must be");
5901 
5902     address start = __ pc();
5903 
5904     // TODO: Handle Valhalla return types. May require generating different return barriers.
5905 
5906     Register fi = r11;
5907 
5908     if (!return_barrier) {
5909       __ pop(c_rarg3); // pop return address. if we don't do this, we get a drift, where the bottom-most frozen frame continuously grows
5910       // __ lea(rsp, Address(rsp, wordSize)); // pop return address. if we don't do this, we get a drift, where the bottom-most frozen frame continuously grows
5911       // write sp to thread->_cont_frame.sp
5912       __ lea(fi, Address(r15_thread, JavaThread::cont_frame_offset()));
5913       __ movptr(Address(fi, wordSize*2), rsp); // sp
5914     } else {
5915       Label no_saved_sp;
5916       __ lea(fi, Address(r15_thread, JavaThread::cont_frame_offset()));
5917       __ movptr(fi, Address(fi, wordSize*2)); // sp
5918       __ testq(fi, fi);
5919       __ jcc(Assembler::zero, no_saved_sp);
5920       __ movptr(rsp, fi);
5921       __ bind(no_saved_sp);
5922     }
5923 
5924     Label thaw_success;
5925     __ movptr(fi, rsp);
5926     if (return_barrier) {
5927       __ push(rax); __ push_d(xmm0); // preserve possible return value from a method returning to the return barrier
5928     }
5929     __ movl(c_rarg1, return_barrier);
5930     push_FrameInfo(_masm, fi, fi, rbp, c_rarg3);
5931     if (ContPerfTest > 105) {
5932       __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::prepare_thaw), fi, c_rarg1);
5933     } else {
5934       __ xorq(rax, rax);
5935     }
5936     __ testq(rax, rax);           // rax contains the size of the frames to thaw, 0 if overflow or no more frames
5937     __ jcc(Assembler::notZero, thaw_success);
5938 
5939     pop_FrameInfo(_masm, fi, rbp, rbx);
5940     if (return_barrier) {
5941       __ pop_d(xmm0); __ pop(rax); // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
5942     }
5943     __ movptr(rsp, fi); // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
5944     __ jmp(rbx); // a jump to StubRoutines::throw_StackOverflowError_entry
5945 
5946     __ bind(thaw_success);
5947 
5948     pop_FrameInfo(_masm, fi, rbp, c_rarg3); // c_rarg3 would still be our return address
5949     if (return_barrier) {
5950       __ pop_d(xmm0); __ pop(rdx);   // TEMPORARILY restore return value (we're going to push it again, but rsp is about to move)
5951     }
5952 
5953     __ subq(rsp, rax);             // make room for the thawed frames
5954     __ subptr(rsp, wordSize);      // make room for return address
5955     __ andptr(rsp, -16); // align
5956     if (return_barrier) {
5957       __ push(rdx); __ push_d(xmm0); // save original return value -- again
5958     }
5959     push_FrameInfo(_masm, fi, fi, rbp, c_rarg3);
5960     __ movl(c_rarg1, return_barrier);
5961     __ movl(c_rarg2, exception);
5962     if (ContPerfTest > 112) {
5963       if (!return_barrier && JvmtiExport::can_support_continuations()) {
5964         __ call_VM(noreg, CAST_FROM_FN_PTR(address, Continuation::thaw), fi, c_rarg1, c_rarg2);
5965       } else {
5966         __ call_VM_leaf(CAST_FROM_FN_PTR(address, Continuation::thaw_leaf), fi, c_rarg1, c_rarg2);
5967       }
5968     }
5969     if (exception) {
5970       __ movptr(rdx, rax); // rdx must contain the original pc in the case of exception
5971     }
5972     pop_FrameInfo(_masm, fi, rbp, rbx);
5973     if (return_barrier) {
5974       __ pop_d(xmm0); __ pop(rax); // restore return value (no safepoint in the call to thaw, so even an oop return value should be OK)
5975     }
5976 
5977     __ movptr(rsp, fi); // we're now on the yield frame (which is in an address above us b/c rsp has been pushed down)
5978 
5979     if (!return_barrier) {
5980       // This is necessary for forced yields, as the return addres (in rbx) is captured in a call_VM, and skips the restoration of rbcp and locals
5981       // ... but it does no harm even for ordinary yields
5982       // TODO: use InterpreterMacroAssembler
5983       static const Register _locals_register = LP64_ONLY(r14) NOT_LP64(rdi);
5984       static const Register _bcp_register    = LP64_ONLY(r13) NOT_LP64(rsi);
5985 
5986       Label not_interpreter;
5987       __ testq(rax, rax); // rax is true iff we're jumping into the interpreter
5988       __ jcc(Assembler::zero, not_interpreter);
5989 
5990       // see InterpreterMacroAssembler::restore_bcp/restore_locals
5991       __ movptr(_bcp_register,    Address(rbp, frame::interpreter_frame_bcp_offset    * wordSize));
5992       __ movptr(_locals_register, Address(rbp, frame::interpreter_frame_locals_offset * wordSize));
5993       // __ reinit_heapbase();
5994 
5995       __ bind(not_interpreter);
5996 
5997       __ movl(rax, 0); // return 0 (success) from doYield
5998     }
5999 
6000     __ jmp(rbx);
6001 
6002     return start;
6003   }
6004 
6005   address generate_cont_thaw() {
6006     StubCodeMark mark(this, "StubRoutines", "Cont thaw");
6007     address start = __ pc();
6008     generate_cont_thaw(false, false);
6009     return start;
6010   }
6011 
6012   address generate_cont_returnBarrier() {
6013     // TODO: will probably need multiple return barriers depending on return type
6014     StubCodeMark mark(this, "StubRoutines", "cont return barrier");
6015     address start = __ pc();
6016 
6017     if (CONT_FULL_STACK)
6018       __ stop("RETURN BARRIER -- UNREACHABLE 0");
6019 
6020     generate_cont_thaw(true, false);
6021 
6022     return start;
6023   }
6024 
6025   address generate_cont_returnBarrier_exception() {
6026     StubCodeMark mark(this, "StubRoutines", "cont return barrier exception handler");
6027     address start = __ pc();
6028 
6029     if (CONT_FULL_STACK)
6030       __ stop("RETURN BARRIER -- UNREACHABLE 0");
6031 
6032     generate_cont_thaw(true, true);
6033 
6034     return start;
6035   }
6036 
6037   address generate_cont_getPC() {
6038     StubCodeMark mark(this, "StubRoutines", "GetPC");
6039     address start = __ pc();
6040 
6041     __ movptr(rax, Address(rsp, 0));
6042     __ ret(0);
6043 
6044     return start;
6045   }
6046 
6047   address generate_cont_getSP() { // used by C2
6048     StubCodeMark mark(this, "StubRoutines", "getSP");
6049     address start = __ pc();
6050 
6051     __ set_cont_fastpath(get_thread(), 1);
6052     __ lea(rax, Address(rsp, wordSize));
6053     __ ret(0);
6054 
6055     return start;
6056   }
6057 
6058   address generate_cont_getFP() {
6059     StubCodeMark mark(this, "StubRoutines", "GetFP");
6060     address start = __ pc();
6061 
6062     __ stop("WHAT?");
6063     __ lea(rax, Address(rsp, wordSize));
6064     __ ret(0);
6065 
6066     return start;
6067   }
6068 
6069 #undef __
6070 #define __ masm->
6071 
6072   // Continuation point for throwing of implicit exceptions that are
6073   // not handled in the current activation. Fabricates an exception
6074   // oop and initiates normal exception dispatching in this
6075   // frame. Since we need to preserve callee-saved values (currently
6076   // only for C2, but done for C1 as well) we need a callee-saved oop
6077   // map and therefore have to make these stubs into RuntimeStubs
6078   // rather than BufferBlobs.  If the compiler needs all registers to
6079   // be preserved between the fault point and the exception handler
6080   // then it must assume responsibility for that in
6081   // AbstractCompiler::continuation_for_implicit_null_exception or
6082   // continuation_for_implicit_division_by_zero_exception. All other
6083   // implicit exceptions (e.g., NullPointerException or
6084   // AbstractMethodError on entry) are either at call sites or
6085   // otherwise assume that stack unwinding will be initiated, so
6086   // caller saved registers were assumed volatile in the compiler.
6087   address generate_throw_exception(const char* name,
6088                                    address runtime_entry,
6089                                    Register arg1 = noreg,
6090                                    Register arg2 = noreg) {
6091     // Information about frame layout at time of blocking runtime call.
6092     // Note that we only have to preserve callee-saved registers since
6093     // the compilers are responsible for supplying a continuation point
6094     // if they expect all registers to be preserved.
6095     enum layout {
6096       rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
6097       rbp_off2,
6098       return_off,
6099       return_off2,
6100       framesize // inclusive of return address
6101     };
6102 
6103     int insts_size = 512;
6104     int locs_size  = 64;
6105 
6106     CodeBuffer code(name, insts_size, locs_size);
6107     OopMapSet* oop_maps  = new OopMapSet();
6108     MacroAssembler* masm = new MacroAssembler(&code);
6109 
6110     address start = __ pc();
6111 
6112     // This is an inlined and slightly modified version of call_VM
6113     // which has the ability to fetch the return PC out of
6114     // thread-local storage and also sets up last_Java_sp slightly
6115     // differently than the real call_VM
6116 
6117     __ enter(); // required for proper stackwalking of RuntimeStub frame
6118 
6119     assert(is_even(framesize/2), "sp not 16-byte aligned");
6120 
6121     // return address and rbp are already in place
6122     __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
6123 
6124     int frame_complete = __ pc() - start;
6125 
6126     // Set up last_Java_sp and last_Java_fp
6127     address the_pc = __ pc();
6128     __ set_last_Java_frame(rsp, rbp, the_pc);
6129     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
6130 
6131     // Call runtime
6132     if (arg1 != noreg) {
6133       assert(arg2 != c_rarg1, "clobbered");
6134       __ movptr(c_rarg1, arg1);
6135     }
6136     if (arg2 != noreg) {
6137       __ movptr(c_rarg2, arg2);
6138     }
6139     __ movptr(c_rarg0, r15_thread);
6140     BLOCK_COMMENT("call runtime_entry");
6141     __ call(RuntimeAddress(runtime_entry));
6142 
6143     // Generate oop map
6144     OopMap* map = new OopMap(framesize, 0);
6145 
6146     oop_maps->add_gc_map(the_pc - start, map);
6147 
6148     __ reset_last_Java_frame(true);
6149 
6150     __ leave(); // required for proper stackwalking of RuntimeStub frame
6151 
6152     // check for pending exceptions
6153 #ifdef ASSERT
6154     Label L;
6155     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()),
6156             (int32_t) NULL_WORD);
6157     __ jcc(Assembler::notEqual, L);
6158     __ should_not_reach_here();
6159     __ bind(L);
6160 #endif // ASSERT
6161     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
6162 
6163 
6164     // codeBlob framesize is in words (not VMRegImpl::slot_size)
6165     RuntimeStub* stub =
6166       RuntimeStub::new_runtime_stub(name,
6167                                     &code,
6168                                     frame_complete,
6169                                     (framesize >> (LogBytesPerWord - LogBytesPerInt)),
6170                                     oop_maps, false);
6171     return stub->entry_point();
6172   }
6173 
6174   void create_control_words() {
6175     // Round to nearest, 53-bit mode, exceptions masked
6176     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
6177     // Round to zero, 53-bit mode, exception mased
6178     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
6179     // Round to nearest, 24-bit mode, exceptions masked
6180     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
6181     // Round to nearest, 64-bit mode, exceptions masked
6182     StubRoutines::_mxcsr_std           = 0x1F80;
6183     // Note: the following two constants are 80-bit values
6184     //       layout is critical for correct loading by FPU.
6185     // Bias for strict fp multiply/divide
6186     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
6187     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
6188     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
6189     // Un-Bias for strict fp multiply/divide
6190     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
6191     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
6192     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
6193   }
6194 
6195   // Initialization
6196   void generate_initial() {
6197     // Generates all stubs and initializes the entry points
6198 
6199     // This platform-specific settings are needed by generate_call_stub()
6200     create_control_words();
6201 
6202     // entry points that exist in all platforms Note: This is code
6203     // that could be shared among different platforms - however the
6204     // benefit seems to be smaller than the disadvantage of having a
6205     // much more complicated generator structure. See also comment in
6206     // stubRoutines.hpp.
6207 
6208     StubRoutines::_forward_exception_entry = generate_forward_exception();
6209 
6210     StubRoutines::_call_stub_entry =
6211       generate_call_stub(StubRoutines::_call_stub_return_address);
6212 
6213     // is referenced by megamorphic call
6214     StubRoutines::_catch_exception_entry = generate_catch_exception();
6215 
6216     // atomic calls
6217     StubRoutines::_atomic_xchg_entry          = generate_atomic_xchg();
6218     StubRoutines::_atomic_xchg_long_entry     = generate_atomic_xchg_long();
6219     StubRoutines::_atomic_cmpxchg_entry       = generate_atomic_cmpxchg();
6220     StubRoutines::_atomic_cmpxchg_byte_entry  = generate_atomic_cmpxchg_byte();
6221     StubRoutines::_atomic_cmpxchg_long_entry  = generate_atomic_cmpxchg_long();
6222     StubRoutines::_atomic_add_entry           = generate_atomic_add();
6223     StubRoutines::_atomic_add_long_entry      = generate_atomic_add_long();
6224     StubRoutines::_fence_entry                = generate_orderaccess_fence();
6225 
6226     // platform dependent
6227     StubRoutines::x86::_get_previous_fp_entry = generate_get_previous_fp();
6228     StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp();
6229 
6230     StubRoutines::x86::_verify_mxcsr_entry    = generate_verify_mxcsr();
6231 
6232     // Build this early so it's available for the interpreter.
6233     StubRoutines::_throw_StackOverflowError_entry =
6234       generate_throw_exception("StackOverflowError throw_exception",
6235                                CAST_FROM_FN_PTR(address,
6236                                                 SharedRuntime::
6237                                                 throw_StackOverflowError));
6238     StubRoutines::_throw_delayed_StackOverflowError_entry =
6239       generate_throw_exception("delayed StackOverflowError throw_exception",
6240                                CAST_FROM_FN_PTR(address,
6241                                                 SharedRuntime::
6242                                                 throw_delayed_StackOverflowError));
6243     if (UseCRC32Intrinsics) {
6244       // set table address before stub generation which use it
6245       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
6246       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
6247     }
6248 
6249     if (UseCRC32CIntrinsics) {
6250       bool supports_clmul = VM_Version::supports_clmul();
6251       StubRoutines::x86::generate_CRC32C_table(supports_clmul);
6252       StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table;
6253       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
6254     }
6255     if (VM_Version::supports_sse2() && UseLibmIntrinsic && InlineIntrinsics) {
6256       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
6257           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
6258           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
6259         StubRoutines::x86::_ONEHALF_adr = (address)StubRoutines::x86::_ONEHALF;
6260         StubRoutines::x86::_P_2_adr = (address)StubRoutines::x86::_P_2;
6261         StubRoutines::x86::_SC_4_adr = (address)StubRoutines::x86::_SC_4;
6262         StubRoutines::x86::_Ctable_adr = (address)StubRoutines::x86::_Ctable;
6263         StubRoutines::x86::_SC_2_adr = (address)StubRoutines::x86::_SC_2;
6264         StubRoutines::x86::_SC_3_adr = (address)StubRoutines::x86::_SC_3;
6265         StubRoutines::x86::_SC_1_adr = (address)StubRoutines::x86::_SC_1;
6266         StubRoutines::x86::_PI_INV_TABLE_adr = (address)StubRoutines::x86::_PI_INV_TABLE;
6267         StubRoutines::x86::_PI_4_adr = (address)StubRoutines::x86::_PI_4;
6268         StubRoutines::x86::_PI32INV_adr = (address)StubRoutines::x86::_PI32INV;
6269         StubRoutines::x86::_SIGN_MASK_adr = (address)StubRoutines::x86::_SIGN_MASK;
6270         StubRoutines::x86::_P_1_adr = (address)StubRoutines::x86::_P_1;
6271         StubRoutines::x86::_P_3_adr = (address)StubRoutines::x86::_P_3;
6272         StubRoutines::x86::_NEG_ZERO_adr = (address)StubRoutines::x86::_NEG_ZERO;
6273       }
6274       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) {
6275         StubRoutines::_dexp = generate_libmExp();
6276       }
6277       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
6278         StubRoutines::_dlog = generate_libmLog();
6279       }
6280       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) {
6281         StubRoutines::_dlog10 = generate_libmLog10();
6282       }
6283       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) {
6284         StubRoutines::_dpow = generate_libmPow();
6285       }
6286       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
6287         StubRoutines::_dsin = generate_libmSin();
6288       }
6289       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
6290         StubRoutines::_dcos = generate_libmCos();
6291       }
6292       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
6293         StubRoutines::_dtan = generate_libmTan();
6294       }
6295     }
6296   }
6297 
6298   void generate_phase1() {
6299     // Continuation stubs:
6300     StubRoutines::_cont_thaw          = generate_cont_thaw();
6301     StubRoutines::_cont_returnBarrier = generate_cont_returnBarrier();
6302     StubRoutines::_cont_returnBarrierExc = generate_cont_returnBarrier_exception();
6303     StubRoutines::_cont_doYield_stub = generate_cont_doYield();
6304     StubRoutines::_cont_doYield    = StubRoutines::_cont_doYield_stub->entry_point();
6305     StubRoutines::_cont_jump_from_sp = generate_cont_jump_from_safepoint();
6306     StubRoutines::_cont_jump       = generate_cont_jump();
6307     StubRoutines::_cont_getSP      = generate_cont_getSP();
6308     StubRoutines::_cont_getPC      = generate_cont_getPC();
6309   }
6310 
6311   void generate_all() {
6312     // Generates all stubs and initializes the entry points
6313 
6314     // These entry points require SharedInfo::stack0 to be set up in
6315     // non-core builds and need to be relocatable, so they each
6316     // fabricate a RuntimeStub internally.
6317     StubRoutines::_throw_AbstractMethodError_entry =
6318       generate_throw_exception("AbstractMethodError throw_exception",
6319                                CAST_FROM_FN_PTR(address,
6320                                                 SharedRuntime::
6321                                                 throw_AbstractMethodError));
6322 
6323     StubRoutines::_throw_IncompatibleClassChangeError_entry =
6324       generate_throw_exception("IncompatibleClassChangeError throw_exception",
6325                                CAST_FROM_FN_PTR(address,
6326                                                 SharedRuntime::
6327                                                 throw_IncompatibleClassChangeError));
6328 
6329     StubRoutines::_throw_NullPointerException_at_call_entry =
6330       generate_throw_exception("NullPointerException at call throw_exception",
6331                                CAST_FROM_FN_PTR(address,
6332                                                 SharedRuntime::
6333                                                 throw_NullPointerException_at_call));
6334 
6335     // entry points that are platform specific
6336     StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
6337     StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
6338     StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
6339     StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
6340 
6341     StubRoutines::x86::_float_sign_mask  = generate_fp_mask("float_sign_mask",  0x7FFFFFFF7FFFFFFF);
6342     StubRoutines::x86::_float_sign_flip  = generate_fp_mask("float_sign_flip",  0x8000000080000000);
6343     StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
6344     StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
6345     StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF7FFFFFFF);
6346     StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x8000000080000000);
6347     StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask("vector_double_sign_mask", 0x7FFFFFFFFFFFFFFF);
6348     StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask("vector_double_sign_flip", 0x8000000000000000);
6349     StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff00ff00ff);
6350     StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
6351     StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask("vector_long_sign_mask", 0x8000000000000000);
6352 
6353     // support for verify_oop (must happen after universe_init)
6354     StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
6355 
6356     // arraycopy stubs used by compilers
6357     generate_arraycopy_stubs();
6358 
6359     // don't bother generating these AES intrinsic stubs unless global flag is set
6360     if (UseAESIntrinsics) {
6361       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // needed by the others
6362       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
6363       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
6364       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
6365       if (VM_Version::supports_vaes() &&  VM_Version::supports_avx512vl() && VM_Version::supports_avx512dq() ) {
6366         StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptVectorAESCrypt();
6367       } else {
6368         StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
6369       }
6370     }
6371     if (UseAESCTRIntrinsics){
6372       StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
6373       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
6374     }
6375 
6376     if (UseSHA1Intrinsics) {
6377       StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask();
6378       StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask();
6379       StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
6380       StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
6381     }
6382     if (UseSHA256Intrinsics) {
6383       StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
6384       char* dst = (char*)StubRoutines::x86::_k256_W;
6385       char* src = (char*)StubRoutines::x86::_k256;
6386       for (int ii = 0; ii < 16; ++ii) {
6387         memcpy(dst + 32 * ii,      src + 16 * ii, 16);
6388         memcpy(dst + 32 * ii + 16, src + 16 * ii, 16);
6389       }
6390       StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W;
6391       StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
6392       StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
6393       StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
6394     }
6395     if (UseSHA512Intrinsics) {
6396       StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W;
6397       StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512();
6398       StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress");
6399       StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB");
6400     }
6401 
6402     // Generate GHASH intrinsics code
6403     if (UseGHASHIntrinsics) {
6404     StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
6405     StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
6406       if (VM_Version::supports_avx()) {
6407         StubRoutines::x86::_ghash_shuffmask_addr = ghash_shufflemask_addr();
6408         StubRoutines::x86::_ghash_poly_addr = ghash_polynomial_addr();
6409         StubRoutines::_ghash_processBlocks = generate_avx_ghash_processBlocks();
6410       } else {
6411         StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
6412       }
6413     }
6414 
6415     if (UseBASE64Intrinsics) {
6416       StubRoutines::x86::_and_mask = base64_and_mask_addr();
6417       StubRoutines::x86::_bswap_mask = base64_bswap_mask_addr();
6418       StubRoutines::x86::_base64_charset = base64_charset_addr();
6419       StubRoutines::x86::_url_charset = base64url_charset_addr();
6420       StubRoutines::x86::_gather_mask = base64_gather_mask_addr();
6421       StubRoutines::x86::_left_shift_mask = base64_left_shift_mask_addr();
6422       StubRoutines::x86::_right_shift_mask = base64_right_shift_mask_addr();
6423       StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock();
6424     }
6425 
6426     // Safefetch stubs.
6427     generate_safefetch("SafeFetch32", sizeof(int),     &StubRoutines::_safefetch32_entry,
6428                                                        &StubRoutines::_safefetch32_fault_pc,
6429                                                        &StubRoutines::_safefetch32_continuation_pc);
6430     generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
6431                                                        &StubRoutines::_safefetchN_fault_pc,
6432                                                        &StubRoutines::_safefetchN_continuation_pc);
6433 
6434     BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
6435     if (bs_nm != NULL) {
6436       StubRoutines::x86::_method_entry_barrier = generate_method_entry_barrier();
6437     }
6438 #ifdef COMPILER2
6439     if (UseMultiplyToLenIntrinsic) {
6440       StubRoutines::_multiplyToLen = generate_multiplyToLen();
6441     }
6442     if (UseSquareToLenIntrinsic) {
6443       StubRoutines::_squareToLen = generate_squareToLen();
6444     }
6445     if (UseMulAddIntrinsic) {
6446       StubRoutines::_mulAdd = generate_mulAdd();
6447     }
6448 #ifndef _WINDOWS
6449     if (UseMontgomeryMultiplyIntrinsic) {
6450       StubRoutines::_montgomeryMultiply
6451         = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_multiply);
6452     }
6453     if (UseMontgomerySquareIntrinsic) {
6454       StubRoutines::_montgomerySquare
6455         = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square);
6456     }
6457 #endif // WINDOWS
6458 #endif // COMPILER2
6459 
6460     if (UseVectorizedMismatchIntrinsic) {
6461       StubRoutines::_vectorizedMismatch = generate_vectorizedMismatch();
6462     }
6463   }
6464 
6465  public:
6466   StubGenerator(CodeBuffer* code, int phase) : StubCodeGenerator(code) {
6467     if (phase == 0) {
6468       generate_initial();
6469     } else if (phase == 1) {
6470       generate_phase1();
6471     } else {
6472       generate_all();
6473     }
6474   }
6475 }; // end class declaration
6476 
6477 #define UCM_TABLE_MAX_ENTRIES 16
6478 void StubGenerator_generate(CodeBuffer* code, int phase) {
6479   if (UnsafeCopyMemory::_table == NULL) {
6480     UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
6481   }
6482   StubGenerator g(code, phase);
6483 }