1 /*
   2  * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/barrierSetAssembler.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "nativeInst_x86.hpp"
  32 #include "oops/instanceOop.hpp"
  33 #include "oops/method.hpp"
  34 #include "oops/objArrayKlass.hpp"
  35 #include "oops/oop.inline.hpp"
  36 #include "prims/methodHandles.hpp"
  37 #include "runtime/frame.inline.hpp"
  38 #include "runtime/handles.inline.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 #include "runtime/stubCodeGenerator.hpp"
  41 #include "runtime/stubRoutines.hpp"
  42 #include "runtime/thread.inline.hpp"
  43 #ifdef COMPILER2
  44 #include "opto/runtime.hpp"
  45 #endif
  46 
  47 // Declaration and definition of StubGenerator (no .hpp file).
  48 // For a more detailed description of the stub routine structure
  49 // see the comment in stubRoutines.hpp
  50 
  51 #define __ _masm->
  52 #define a__ ((Assembler*)_masm)->
  53 
  54 #ifdef PRODUCT
  55 #define BLOCK_COMMENT(str) /* nothing */
  56 #else
  57 #define BLOCK_COMMENT(str) __ block_comment(str)
  58 #endif
  59 
  60 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  61 
  62 const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions
  63 const int FPU_CNTRL_WRD_MASK = 0xFFFF;
  64 
  65 // -------------------------------------------------------------------------------------------------------------------------
  66 // Stub Code definitions
  67 
  68 class StubGenerator: public StubCodeGenerator {
  69  private:
  70 
  71 #ifdef PRODUCT
  72 #define inc_counter_np(counter) ((void)0)
  73 #else
  74   void inc_counter_np_(int& counter) {
  75     __ incrementl(ExternalAddress((address)&counter));
  76   }
  77 #define inc_counter_np(counter) \
  78   BLOCK_COMMENT("inc_counter " #counter); \
  79   inc_counter_np_(counter);
  80 #endif //PRODUCT
  81 
  82   void inc_copy_counter_np(BasicType t) {
  83 #ifndef PRODUCT
  84     switch (t) {
  85     case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
  86     case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
  87     case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
  88     case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
  89     case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
  90     default:        ShouldNotReachHere();
  91     }
  92 #endif //PRODUCT
  93   }
  94 
  95   //------------------------------------------------------------------------------------------------------------------------
  96   // Call stubs are used to call Java from C
  97   //
  98   //    [ return_from_Java     ] <--- rsp
  99   //    [ argument word n      ]
 100   //      ...
 101   // -N [ argument word 1      ]
 102   // -7 [ Possible padding for stack alignment ]
 103   // -6 [ Possible padding for stack alignment ]
 104   // -5 [ Possible padding for stack alignment ]
 105   // -4 [ mxcsr save           ] <--- rsp_after_call
 106   // -3 [ saved rbx,            ]
 107   // -2 [ saved rsi            ]
 108   // -1 [ saved rdi            ]
 109   //  0 [ saved rbp,            ] <--- rbp,
 110   //  1 [ return address       ]
 111   //  2 [ ptr. to call wrapper ]
 112   //  3 [ result               ]
 113   //  4 [ result_type          ]
 114   //  5 [ method               ]
 115   //  6 [ entry_point          ]
 116   //  7 [ parameters           ]
 117   //  8 [ parameter_size       ]
 118   //  9 [ thread               ]
 119 
 120 
 121   address generate_call_stub(address& return_address) {
 122     StubCodeMark mark(this, "StubRoutines", "call_stub");
 123     address start = __ pc();
 124 
 125     // stub code parameters / addresses
 126     assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
 127     bool  sse_save = false;
 128     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
 129     const int     locals_count_in_bytes  (4*wordSize);
 130     const Address mxcsr_save    (rbp, -4 * wordSize);
 131     const Address saved_rbx     (rbp, -3 * wordSize);
 132     const Address saved_rsi     (rbp, -2 * wordSize);
 133     const Address saved_rdi     (rbp, -1 * wordSize);
 134     const Address result        (rbp,  3 * wordSize);
 135     const Address result_type   (rbp,  4 * wordSize);
 136     const Address method        (rbp,  5 * wordSize);
 137     const Address entry_point   (rbp,  6 * wordSize);
 138     const Address parameters    (rbp,  7 * wordSize);
 139     const Address parameter_size(rbp,  8 * wordSize);
 140     const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!
 141     sse_save =  UseSSE > 0;
 142 
 143     // stub code
 144     __ enter();
 145     __ movptr(rcx, parameter_size);              // parameter counter
 146     __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes
 147     __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves
 148     __ subptr(rsp, rcx);
 149     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
 150 
 151     // save rdi, rsi, & rbx, according to C calling conventions
 152     __ movptr(saved_rdi, rdi);
 153     __ movptr(saved_rsi, rsi);
 154     __ movptr(saved_rbx, rbx);
 155 
 156     // save and initialize %mxcsr
 157     if (sse_save) {
 158       Label skip_ldmx;
 159       __ stmxcsr(mxcsr_save);
 160       __ movl(rax, mxcsr_save);
 161       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 162       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 163       __ cmp32(rax, mxcsr_std);
 164       __ jcc(Assembler::equal, skip_ldmx);
 165       __ ldmxcsr(mxcsr_std);
 166       __ bind(skip_ldmx);
 167     }
 168 
 169     // make sure the control word is correct.
 170     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 171 
 172 #ifdef ASSERT
 173     // make sure we have no pending exceptions
 174     { Label L;
 175       __ movptr(rcx, thread);
 176       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 177       __ jcc(Assembler::equal, L);
 178       __ stop("StubRoutines::call_stub: entered with pending exception");
 179       __ bind(L);
 180     }
 181 #endif
 182 
 183     // pass parameters if any
 184     BLOCK_COMMENT("pass parameters if any");
 185     Label parameters_done;
 186     __ movl(rcx, parameter_size);  // parameter counter
 187     __ testl(rcx, rcx);
 188     __ jcc(Assembler::zero, parameters_done);
 189 
 190     // parameter passing loop
 191 
 192     Label loop;
 193     // Copy Java parameters in reverse order (receiver last)
 194     // Note that the argument order is inverted in the process
 195     // source is rdx[rcx: N-1..0]
 196     // dest   is rsp[rbx: 0..N-1]
 197 
 198     __ movptr(rdx, parameters);          // parameter pointer
 199     __ xorptr(rbx, rbx);
 200 
 201     __ BIND(loop);
 202 
 203     // get parameter
 204     __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
 205     __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
 206                     Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter
 207     __ increment(rbx);
 208     __ decrement(rcx);
 209     __ jcc(Assembler::notZero, loop);
 210 
 211     // call Java function
 212     __ BIND(parameters_done);
 213     __ movptr(rbx, method);           // get Method*
 214     __ movptr(rax, entry_point);      // get entry_point
 215     __ mov(rsi, rsp);                 // set sender sp
 216     BLOCK_COMMENT("call Java function");
 217     __ call(rax);
 218 
 219     BLOCK_COMMENT("call_stub_return_address:");
 220     return_address = __ pc();
 221 
 222 #ifdef COMPILER2
 223     {
 224       Label L_skip;
 225       if (UseSSE >= 2) {
 226         __ verify_FPU(0, "call_stub_return");
 227       } else {
 228         for (int i = 1; i < 8; i++) {
 229           __ ffree(i);
 230         }
 231 
 232         // UseSSE <= 1 so double result should be left on TOS
 233         __ movl(rsi, result_type);
 234         __ cmpl(rsi, T_DOUBLE);
 235         __ jcc(Assembler::equal, L_skip);
 236         if (UseSSE == 0) {
 237           // UseSSE == 0 so float result should be left on TOS
 238           __ cmpl(rsi, T_FLOAT);
 239           __ jcc(Assembler::equal, L_skip);
 240         }
 241         __ ffree(0);
 242       }
 243       __ BIND(L_skip);
 244     }
 245 #endif // COMPILER2
 246 
 247     // store result depending on type
 248     // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 249     __ movptr(rdi, result);
 250     Label is_long, is_float, is_double, exit;
 251     __ movl(rsi, result_type);
 252     __ cmpl(rsi, T_LONG);
 253     __ jcc(Assembler::equal, is_long);
 254     __ cmpl(rsi, T_FLOAT);
 255     __ jcc(Assembler::equal, is_float);
 256     __ cmpl(rsi, T_DOUBLE);
 257     __ jcc(Assembler::equal, is_double);
 258 
 259     // handle T_INT case
 260     __ movl(Address(rdi, 0), rax);
 261     __ BIND(exit);
 262 
 263     // check that FPU stack is empty
 264     __ verify_FPU(0, "generate_call_stub");
 265 
 266     // pop parameters
 267     __ lea(rsp, rsp_after_call);
 268 
 269     // restore %mxcsr
 270     if (sse_save) {
 271       __ ldmxcsr(mxcsr_save);
 272     }
 273 
 274     // restore rdi, rsi and rbx,
 275     __ movptr(rbx, saved_rbx);
 276     __ movptr(rsi, saved_rsi);
 277     __ movptr(rdi, saved_rdi);
 278     __ addptr(rsp, 4*wordSize);
 279 
 280     // return
 281     __ pop(rbp);
 282     __ ret(0);
 283 
 284     // handle return types different from T_INT
 285     __ BIND(is_long);
 286     __ movl(Address(rdi, 0 * wordSize), rax);
 287     __ movl(Address(rdi, 1 * wordSize), rdx);
 288     __ jmp(exit);
 289 
 290     __ BIND(is_float);
 291     // interpreter uses xmm0 for return values
 292     if (UseSSE >= 1) {
 293       __ movflt(Address(rdi, 0), xmm0);
 294     } else {
 295       __ fstp_s(Address(rdi, 0));
 296     }
 297     __ jmp(exit);
 298 
 299     __ BIND(is_double);
 300     // interpreter uses xmm0 for return values
 301     if (UseSSE >= 2) {
 302       __ movdbl(Address(rdi, 0), xmm0);
 303     } else {
 304       __ fstp_d(Address(rdi, 0));
 305     }
 306     __ jmp(exit);
 307 
 308     return start;
 309   }
 310 
 311 
 312   //------------------------------------------------------------------------------------------------------------------------
 313   // Return point for a Java call if there's an exception thrown in Java code.
 314   // The exception is caught and transformed into a pending exception stored in
 315   // JavaThread that can be tested from within the VM.
 316   //
 317   // Note: Usually the parameters are removed by the callee. In case of an exception
 318   //       crossing an activation frame boundary, that is not the case if the callee
 319   //       is compiled code => need to setup the rsp.
 320   //
 321   // rax,: exception oop
 322 
 323   address generate_catch_exception() {
 324     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 325     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
 326     const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!
 327     address start = __ pc();
 328 
 329     // get thread directly
 330     __ movptr(rcx, thread);
 331 #ifdef ASSERT
 332     // verify that threads correspond
 333     { Label L;
 334       __ get_thread(rbx);
 335       __ cmpptr(rbx, rcx);
 336       __ jcc(Assembler::equal, L);
 337       __ stop("StubRoutines::catch_exception: threads must correspond");
 338       __ bind(L);
 339     }
 340 #endif
 341     // set pending exception
 342     __ verify_oop(rax);
 343     __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );
 344     __ lea(Address(rcx, Thread::exception_file_offset   ()),
 345            ExternalAddress((address)__FILE__));
 346     __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );
 347     // complete return to VM
 348     assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
 349     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 350 
 351     return start;
 352   }
 353 
 354 
 355   //------------------------------------------------------------------------------------------------------------------------
 356   // Continuation point for runtime calls returning with a pending exception.
 357   // The pending exception check happened in the runtime or native call stub.
 358   // The pending exception in Thread is converted into a Java-level exception.
 359   //
 360   // Contract with Java-level exception handlers:
 361   // rax: exception
 362   // rdx: throwing pc
 363   //
 364   // NOTE: At entry of this stub, exception-pc must be on stack !!
 365 
 366   address generate_forward_exception() {
 367     StubCodeMark mark(this, "StubRoutines", "forward exception");
 368     address start = __ pc();
 369     const Register thread = rcx;
 370 
 371     // other registers used in this stub
 372     const Register exception_oop = rax;
 373     const Register handler_addr  = rbx;
 374     const Register exception_pc  = rdx;
 375 
 376     // Upon entry, the sp points to the return address returning into Java
 377     // (interpreted or compiled) code; i.e., the return address becomes the
 378     // throwing pc.
 379     //
 380     // Arguments pushed before the runtime call are still on the stack but
 381     // the exception handler will reset the stack pointer -> ignore them.
 382     // A potential result in registers can be ignored as well.
 383 
 384 #ifdef ASSERT
 385     // make sure this code is only executed if there is a pending exception
 386     { Label L;
 387       __ get_thread(thread);
 388       __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 389       __ jcc(Assembler::notEqual, L);
 390       __ stop("StubRoutines::forward exception: no pending exception (1)");
 391       __ bind(L);
 392     }
 393 #endif
 394 
 395     // compute exception handler into rbx,
 396     __ get_thread(thread);
 397     __ movptr(exception_pc, Address(rsp, 0));
 398     BLOCK_COMMENT("call exception_handler_for_return_address");
 399     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
 400     __ mov(handler_addr, rax);
 401 
 402     // setup rax & rdx, remove return address & clear pending exception
 403     __ get_thread(thread);
 404     __ pop(exception_pc);
 405     __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
 406     __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
 407 
 408 #ifdef ASSERT
 409     // make sure exception is set
 410     { Label L;
 411       __ testptr(exception_oop, exception_oop);
 412       __ jcc(Assembler::notEqual, L);
 413       __ stop("StubRoutines::forward exception: no pending exception (2)");
 414       __ bind(L);
 415     }
 416 #endif
 417 
 418     // Verify that there is really a valid exception in RAX.
 419     __ verify_oop(exception_oop);
 420 
 421     // continue at exception handler (return address removed)
 422     // rax: exception
 423     // rbx: exception handler
 424     // rdx: throwing pc
 425     __ jmp(handler_addr);
 426 
 427     return start;
 428   }
 429 
 430 
 431   //----------------------------------------------------------------------------------------------------
 432   // Support for int32_t Atomic::xchg(int32_t exchange_value, volatile int32_t* dest)
 433   //
 434   // xchg exists as far back as 8086, lock needed for MP only
 435   // Stack layout immediately after call:
 436   //
 437   // 0 [ret addr ] <--- rsp
 438   // 1 [  ex     ]
 439   // 2 [  dest   ]
 440   //
 441   // Result:   *dest <- ex, return (old *dest)
 442   //
 443   // Note: win32 does not currently use this code
 444 
 445   address generate_atomic_xchg() {
 446     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 447     address start = __ pc();
 448 
 449     __ push(rdx);
 450     Address exchange(rsp, 2 * wordSize);
 451     Address dest_addr(rsp, 3 * wordSize);
 452     __ movl(rax, exchange);
 453     __ movptr(rdx, dest_addr);
 454     __ xchgl(rax, Address(rdx, 0));
 455     __ pop(rdx);
 456     __ ret(0);
 457 
 458     return start;
 459   }
 460 
 461   //----------------------------------------------------------------------------------------------------
 462   // Support for void verify_mxcsr()
 463   //
 464   // This routine is used with -Xcheck:jni to verify that native
 465   // JNI code does not return to Java code without restoring the
 466   // MXCSR register to our expected state.
 467 
 468 
 469   address generate_verify_mxcsr() {
 470     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 471     address start = __ pc();
 472 
 473     const Address mxcsr_save(rsp, 0);
 474 
 475     if (CheckJNICalls && UseSSE > 0 ) {
 476       Label ok_ret;
 477       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 478       __ push(rax);
 479       __ subptr(rsp, wordSize);      // allocate a temp location
 480       __ stmxcsr(mxcsr_save);
 481       __ movl(rax, mxcsr_save);
 482       __ andl(rax, MXCSR_MASK);
 483       __ cmp32(rax, mxcsr_std);
 484       __ jcc(Assembler::equal, ok_ret);
 485 
 486       __ warn("MXCSR changed by native JNI code.");
 487 
 488       __ ldmxcsr(mxcsr_std);
 489 
 490       __ bind(ok_ret);
 491       __ addptr(rsp, wordSize);
 492       __ pop(rax);
 493     }
 494 
 495     __ ret(0);
 496 
 497     return start;
 498   }
 499 
 500 
 501   //---------------------------------------------------------------------------
 502   // Support for void verify_fpu_cntrl_wrd()
 503   //
 504   // This routine is used with -Xcheck:jni to verify that native
 505   // JNI code does not return to Java code without restoring the
 506   // FP control word to our expected state.
 507 
 508   address generate_verify_fpu_cntrl_wrd() {
 509     StubCodeMark mark(this, "StubRoutines", "verify_spcw");
 510     address start = __ pc();
 511 
 512     const Address fpu_cntrl_wrd_save(rsp, 0);
 513 
 514     if (CheckJNICalls) {
 515       Label ok_ret;
 516       __ push(rax);
 517       __ subptr(rsp, wordSize);      // allocate a temp location
 518       __ fnstcw(fpu_cntrl_wrd_save);
 519       __ movl(rax, fpu_cntrl_wrd_save);
 520       __ andl(rax, FPU_CNTRL_WRD_MASK);
 521       ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());
 522       __ cmp32(rax, fpu_std);
 523       __ jcc(Assembler::equal, ok_ret);
 524 
 525       __ warn("Floating point control word changed by native JNI code.");
 526 
 527       __ fldcw(fpu_std);
 528 
 529       __ bind(ok_ret);
 530       __ addptr(rsp, wordSize);
 531       __ pop(rax);
 532     }
 533 
 534     __ ret(0);
 535 
 536     return start;
 537   }
 538 
 539   //---------------------------------------------------------------------------
 540   // Wrapper for slow-case handling of double-to-integer conversion
 541   // d2i or f2i fast case failed either because it is nan or because
 542   // of under/overflow.
 543   // Input:  FPU TOS: float value
 544   // Output: rax, (rdx): integer (long) result
 545 
 546   address generate_d2i_wrapper(BasicType t, address fcn) {
 547     StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
 548     address start = __ pc();
 549 
 550   // Capture info about frame layout
 551   enum layout { FPUState_off         = 0,
 552                 rbp_off              = FPUStateSizeInWords,
 553                 rdi_off,
 554                 rsi_off,
 555                 rcx_off,
 556                 rbx_off,
 557                 saved_argument_off,
 558                 saved_argument_off2, // 2nd half of double
 559                 framesize
 560   };
 561 
 562   assert(FPUStateSizeInWords == 27, "update stack layout");
 563 
 564     // Save outgoing argument to stack across push_FPU_state()
 565     __ subptr(rsp, wordSize * 2);
 566     __ fstp_d(Address(rsp, 0));
 567 
 568     // Save CPU & FPU state
 569     __ push(rbx);
 570     __ push(rcx);
 571     __ push(rsi);
 572     __ push(rdi);
 573     __ push(rbp);
 574     __ push_FPU_state();
 575 
 576     // push_FPU_state() resets the FP top of stack
 577     // Load original double into FP top of stack
 578     __ fld_d(Address(rsp, saved_argument_off * wordSize));
 579     // Store double into stack as outgoing argument
 580     __ subptr(rsp, wordSize*2);
 581     __ fst_d(Address(rsp, 0));
 582 
 583     // Prepare FPU for doing math in C-land
 584     __ empty_FPU_stack();
 585     // Call the C code to massage the double.  Result in EAX
 586     if (t == T_INT)
 587       { BLOCK_COMMENT("SharedRuntime::d2i"); }
 588     else if (t == T_LONG)
 589       { BLOCK_COMMENT("SharedRuntime::d2l"); }
 590     __ call_VM_leaf( fcn, 2 );
 591 
 592     // Restore CPU & FPU state
 593     __ pop_FPU_state();
 594     __ pop(rbp);
 595     __ pop(rdi);
 596     __ pop(rsi);
 597     __ pop(rcx);
 598     __ pop(rbx);
 599     __ addptr(rsp, wordSize * 2);
 600 
 601     __ ret(0);
 602 
 603     return start;
 604   }
 605 
 606 
 607   //----------------------------------------------------------------------------------------------------
 608   // Non-destructive plausibility checks for oops
 609 
 610   address generate_verify_oop() {
 611     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 612     address start = __ pc();
 613 
 614     // Incoming arguments on stack after saving rax,:
 615     //
 616     // [tos    ]: saved rdx
 617     // [tos + 1]: saved EFLAGS
 618     // [tos + 2]: return address
 619     // [tos + 3]: char* error message
 620     // [tos + 4]: oop   object to verify
 621     // [tos + 5]: saved rax, - saved by caller and bashed
 622 
 623     Label exit, error;
 624     __ pushf();
 625     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 626     __ push(rdx);                                // save rdx
 627     // make sure object is 'reasonable'
 628     __ movptr(rax, Address(rsp, 4 * wordSize));    // get object
 629     __ testptr(rax, rax);
 630     __ jcc(Assembler::zero, exit);               // if obj is NULL it is ok
 631 
 632     // Check if the oop is in the right area of memory
 633     const int oop_mask = Universe::verify_oop_mask();
 634     const int oop_bits = Universe::verify_oop_bits();
 635     __ mov(rdx, rax);
 636     __ andptr(rdx, oop_mask);
 637     __ cmpptr(rdx, oop_bits);
 638     __ jcc(Assembler::notZero, error);
 639 
 640     // make sure klass is 'reasonable', which is not zero.
 641     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
 642     __ testptr(rax, rax);
 643     __ jcc(Assembler::zero, error);              // if klass is NULL it is broken
 644 
 645     // return if everything seems ok
 646     __ bind(exit);
 647     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 648     __ pop(rdx);                                 // restore rdx
 649     __ popf();                                   // restore EFLAGS
 650     __ ret(3 * wordSize);                        // pop arguments
 651 
 652     // handle errors
 653     __ bind(error);
 654     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 655     __ pop(rdx);                                 // get saved rdx back
 656     __ popf();                                   // get saved EFLAGS off stack -- will be ignored
 657     __ pusha();                                  // push registers (eip = return address & msg are already pushed)
 658     BLOCK_COMMENT("call MacroAssembler::debug");
 659     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 660     __ popa();
 661     __ ret(3 * wordSize);                        // pop arguments
 662     return start;
 663   }
 664 
 665 
 666   // Copy 64 bytes chunks
 667   //
 668   // Inputs:
 669   //   from        - source array address
 670   //   to_from     - destination array address - from
 671   //   qword_count - 8-bytes element count, negative
 672   //
 673   void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
 674     assert( UseSSE >= 2, "supported cpu only" );
 675     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 676 
 677     // Copy 64-byte chunks
 678     __ jmpb(L_copy_64_bytes);
 679     __ align(OptoLoopAlignment);
 680   __ BIND(L_copy_64_bytes_loop);
 681 
 682     if (UseUnalignedLoadStores) {
 683       if (UseAVX > 2) {
 684         __ evmovdqul(xmm0, Address(from, 0), Assembler::AVX_512bit);
 685         __ evmovdqul(Address(from, to_from, Address::times_1, 0), xmm0, Assembler::AVX_512bit);
 686       } else if (UseAVX == 2) {
 687         __ vmovdqu(xmm0, Address(from,  0));
 688         __ vmovdqu(Address(from, to_from, Address::times_1,  0), xmm0);
 689         __ vmovdqu(xmm1, Address(from, 32));
 690         __ vmovdqu(Address(from, to_from, Address::times_1, 32), xmm1);
 691       } else {
 692         __ movdqu(xmm0, Address(from, 0));
 693         __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
 694         __ movdqu(xmm1, Address(from, 16));
 695         __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
 696         __ movdqu(xmm2, Address(from, 32));
 697         __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
 698         __ movdqu(xmm3, Address(from, 48));
 699         __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
 700       }
 701     } else {
 702       __ movq(xmm0, Address(from, 0));
 703       __ movq(Address(from, to_from, Address::times_1, 0), xmm0);
 704       __ movq(xmm1, Address(from, 8));
 705       __ movq(Address(from, to_from, Address::times_1, 8), xmm1);
 706       __ movq(xmm2, Address(from, 16));
 707       __ movq(Address(from, to_from, Address::times_1, 16), xmm2);
 708       __ movq(xmm3, Address(from, 24));
 709       __ movq(Address(from, to_from, Address::times_1, 24), xmm3);
 710       __ movq(xmm4, Address(from, 32));
 711       __ movq(Address(from, to_from, Address::times_1, 32), xmm4);
 712       __ movq(xmm5, Address(from, 40));
 713       __ movq(Address(from, to_from, Address::times_1, 40), xmm5);
 714       __ movq(xmm6, Address(from, 48));
 715       __ movq(Address(from, to_from, Address::times_1, 48), xmm6);
 716       __ movq(xmm7, Address(from, 56));
 717       __ movq(Address(from, to_from, Address::times_1, 56), xmm7);
 718     }
 719 
 720     __ addl(from, 64);
 721   __ BIND(L_copy_64_bytes);
 722     __ subl(qword_count, 8);
 723     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 724 
 725     if (UseUnalignedLoadStores && (UseAVX == 2)) {
 726       // clean upper bits of YMM registers
 727       __ vpxor(xmm0, xmm0);
 728       __ vpxor(xmm1, xmm1);
 729     }
 730     __ addl(qword_count, 8);
 731     __ jccb(Assembler::zero, L_exit);
 732     //
 733     // length is too short, just copy qwords
 734     //
 735   __ BIND(L_copy_8_bytes);
 736     __ movq(xmm0, Address(from, 0));
 737     __ movq(Address(from, to_from, Address::times_1), xmm0);
 738     __ addl(from, 8);
 739     __ decrement(qword_count);
 740     __ jcc(Assembler::greater, L_copy_8_bytes);
 741   __ BIND(L_exit);
 742   }
 743 
 744   // Copy 64 bytes chunks
 745   //
 746   // Inputs:
 747   //   from        - source array address
 748   //   to_from     - destination array address - from
 749   //   qword_count - 8-bytes element count, negative
 750   //
 751   void mmx_copy_forward(Register from, Register to_from, Register qword_count) {
 752     assert( VM_Version::supports_mmx(), "supported cpu only" );
 753     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 754     // Copy 64-byte chunks
 755     __ jmpb(L_copy_64_bytes);
 756     __ align(OptoLoopAlignment);
 757   __ BIND(L_copy_64_bytes_loop);
 758     __ movq(mmx0, Address(from, 0));
 759     __ movq(mmx1, Address(from, 8));
 760     __ movq(mmx2, Address(from, 16));
 761     __ movq(Address(from, to_from, Address::times_1, 0), mmx0);
 762     __ movq(mmx3, Address(from, 24));
 763     __ movq(Address(from, to_from, Address::times_1, 8), mmx1);
 764     __ movq(mmx4, Address(from, 32));
 765     __ movq(Address(from, to_from, Address::times_1, 16), mmx2);
 766     __ movq(mmx5, Address(from, 40));
 767     __ movq(Address(from, to_from, Address::times_1, 24), mmx3);
 768     __ movq(mmx6, Address(from, 48));
 769     __ movq(Address(from, to_from, Address::times_1, 32), mmx4);
 770     __ movq(mmx7, Address(from, 56));
 771     __ movq(Address(from, to_from, Address::times_1, 40), mmx5);
 772     __ movq(Address(from, to_from, Address::times_1, 48), mmx6);
 773     __ movq(Address(from, to_from, Address::times_1, 56), mmx7);
 774     __ addptr(from, 64);
 775   __ BIND(L_copy_64_bytes);
 776     __ subl(qword_count, 8);
 777     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 778     __ addl(qword_count, 8);
 779     __ jccb(Assembler::zero, L_exit);
 780     //
 781     // length is too short, just copy qwords
 782     //
 783   __ BIND(L_copy_8_bytes);
 784     __ movq(mmx0, Address(from, 0));
 785     __ movq(Address(from, to_from, Address::times_1), mmx0);
 786     __ addptr(from, 8);
 787     __ decrement(qword_count);
 788     __ jcc(Assembler::greater, L_copy_8_bytes);
 789   __ BIND(L_exit);
 790     __ emms();
 791   }
 792 
 793   address generate_disjoint_copy(BasicType t, bool aligned,
 794                                  Address::ScaleFactor sf,
 795                                  address* entry, const char *name,
 796                                  bool dest_uninitialized = false) {
 797     __ align(CodeEntryAlignment);
 798     StubCodeMark mark(this, "StubRoutines", name);
 799     address start = __ pc();
 800 
 801     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
 802     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
 803 
 804     int shift = Address::times_ptr - sf;
 805 
 806     const Register from     = rsi;  // source array address
 807     const Register to       = rdi;  // destination array address
 808     const Register count    = rcx;  // elements count
 809     const Register to_from  = to;   // (to - from)
 810     const Register saved_to = rdx;  // saved destination array address
 811 
 812     __ enter(); // required for proper stackwalking of RuntimeStub frame
 813     __ push(rsi);
 814     __ push(rdi);
 815     __ movptr(from , Address(rsp, 12+ 4));
 816     __ movptr(to   , Address(rsp, 12+ 8));
 817     __ movl(count, Address(rsp, 12+ 12));
 818 
 819     if (entry != NULL) {
 820       *entry = __ pc(); // Entry point from conjoint arraycopy stub.
 821       BLOCK_COMMENT("Entry:");
 822     }
 823 
 824     if (t == T_OBJECT) {
 825       __ testl(count, count);
 826       __ jcc(Assembler::zero, L_0_count);
 827     }
 828 
 829     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
 830     if (dest_uninitialized) {
 831       decorators |= IS_DEST_UNINITIALIZED;
 832     }
 833     if (aligned) {
 834       decorators |= ARRAYCOPY_ALIGNED;
 835     }
 836 
 837     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 838     bs->arraycopy_prologue(_masm, decorators, t, from, to, count);
 839 
 840     __ subptr(to, from); // to --> to_from
 841     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
 842     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
 843     if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 844       // align source address at 4 bytes address boundary
 845       if (t == T_BYTE) {
 846         // One byte misalignment happens only for byte arrays
 847         __ testl(from, 1);
 848         __ jccb(Assembler::zero, L_skip_align1);
 849         __ movb(rax, Address(from, 0));
 850         __ movb(Address(from, to_from, Address::times_1, 0), rax);
 851         __ increment(from);
 852         __ decrement(count);
 853       __ BIND(L_skip_align1);
 854       }
 855       // Two bytes misalignment happens only for byte and short (char) arrays
 856       __ testl(from, 2);
 857       __ jccb(Assembler::zero, L_skip_align2);
 858       __ movw(rax, Address(from, 0));
 859       __ movw(Address(from, to_from, Address::times_1, 0), rax);
 860       __ addptr(from, 2);
 861       __ subl(count, 1<<(shift-1));
 862     __ BIND(L_skip_align2);
 863     }
 864     if (!VM_Version::supports_mmx()) {
 865       __ mov(rax, count);      // save 'count'
 866       __ shrl(count, shift); // bytes count
 867       __ addptr(to_from, from);// restore 'to'
 868       __ rep_mov();
 869       __ subptr(to_from, from);// restore 'to_from'
 870       __ mov(count, rax);      // restore 'count'
 871       __ jmpb(L_copy_2_bytes); // all dwords were copied
 872     } else {
 873       if (!UseUnalignedLoadStores) {
 874         // align to 8 bytes, we know we are 4 byte aligned to start
 875         __ testptr(from, 4);
 876         __ jccb(Assembler::zero, L_copy_64_bytes);
 877         __ movl(rax, Address(from, 0));
 878         __ movl(Address(from, to_from, Address::times_1, 0), rax);
 879         __ addptr(from, 4);
 880         __ subl(count, 1<<shift);
 881       }
 882     __ BIND(L_copy_64_bytes);
 883       __ mov(rax, count);
 884       __ shrl(rax, shift+1);  // 8 bytes chunk count
 885       //
 886       // Copy 8-byte chunks through MMX registers, 8 per iteration of the loop
 887       //
 888       if (UseXMMForArrayCopy) {
 889         xmm_copy_forward(from, to_from, rax);
 890       } else {
 891         mmx_copy_forward(from, to_from, rax);
 892       }
 893     }
 894     // copy tailing dword
 895   __ BIND(L_copy_4_bytes);
 896     __ testl(count, 1<<shift);
 897     __ jccb(Assembler::zero, L_copy_2_bytes);
 898     __ movl(rax, Address(from, 0));
 899     __ movl(Address(from, to_from, Address::times_1, 0), rax);
 900     if (t == T_BYTE || t == T_SHORT) {
 901       __ addptr(from, 4);
 902     __ BIND(L_copy_2_bytes);
 903       // copy tailing word
 904       __ testl(count, 1<<(shift-1));
 905       __ jccb(Assembler::zero, L_copy_byte);
 906       __ movw(rax, Address(from, 0));
 907       __ movw(Address(from, to_from, Address::times_1, 0), rax);
 908       if (t == T_BYTE) {
 909         __ addptr(from, 2);
 910       __ BIND(L_copy_byte);
 911         // copy tailing byte
 912         __ testl(count, 1);
 913         __ jccb(Assembler::zero, L_exit);
 914         __ movb(rax, Address(from, 0));
 915         __ movb(Address(from, to_from, Address::times_1, 0), rax);
 916       __ BIND(L_exit);
 917       } else {
 918       __ BIND(L_copy_byte);
 919       }
 920     } else {
 921     __ BIND(L_copy_2_bytes);
 922     }
 923 
 924     __ movl(count, Address(rsp, 12+12)); // reread 'count'
 925     bs->arraycopy_epilogue(_masm, decorators, t, from, to, count);
 926 
 927     if (t == T_OBJECT) {
 928     __ BIND(L_0_count);
 929     }
 930     inc_copy_counter_np(t);
 931     __ pop(rdi);
 932     __ pop(rsi);
 933     __ leave(); // required for proper stackwalking of RuntimeStub frame
 934     __ vzeroupper();
 935     __ xorptr(rax, rax); // return 0
 936     __ ret(0);
 937     return start;
 938   }
 939 
 940 
 941   address generate_fill(BasicType t, bool aligned, const char *name) {
 942     __ align(CodeEntryAlignment);
 943     StubCodeMark mark(this, "StubRoutines", name);
 944     address start = __ pc();
 945 
 946     BLOCK_COMMENT("Entry:");
 947 
 948     const Register to       = rdi;  // source array address
 949     const Register value    = rdx;  // value
 950     const Register count    = rsi;  // elements count
 951 
 952     __ enter(); // required for proper stackwalking of RuntimeStub frame
 953     __ push(rsi);
 954     __ push(rdi);
 955     __ movptr(to   , Address(rsp, 12+ 4));
 956     __ movl(value, Address(rsp, 12+ 8));
 957     __ movl(count, Address(rsp, 12+ 12));
 958 
 959     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
 960 
 961     __ pop(rdi);
 962     __ pop(rsi);
 963     __ leave(); // required for proper stackwalking of RuntimeStub frame
 964     __ ret(0);
 965     return start;
 966   }
 967 
 968   address generate_conjoint_copy(BasicType t, bool aligned,
 969                                  Address::ScaleFactor sf,
 970                                  address nooverlap_target,
 971                                  address* entry, const char *name,
 972                                  bool dest_uninitialized = false) {
 973     __ align(CodeEntryAlignment);
 974     StubCodeMark mark(this, "StubRoutines", name);
 975     address start = __ pc();
 976 
 977     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
 978     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
 979 
 980     int shift = Address::times_ptr - sf;
 981 
 982     const Register src   = rax;  // source array address
 983     const Register dst   = rdx;  // destination array address
 984     const Register from  = rsi;  // source array address
 985     const Register to    = rdi;  // destination array address
 986     const Register count = rcx;  // elements count
 987     const Register end   = rax;  // array end address
 988 
 989     __ enter(); // required for proper stackwalking of RuntimeStub frame
 990     __ push(rsi);
 991     __ push(rdi);
 992     __ movptr(src  , Address(rsp, 12+ 4));   // from
 993     __ movptr(dst  , Address(rsp, 12+ 8));   // to
 994     __ movl2ptr(count, Address(rsp, 12+12)); // count
 995 
 996     if (entry != NULL) {
 997       *entry = __ pc(); // Entry point from generic arraycopy stub.
 998       BLOCK_COMMENT("Entry:");
 999     }
1000 
1001     // nooverlap_target expects arguments in rsi and rdi.
1002     __ mov(from, src);
1003     __ mov(to  , dst);
1004 
1005     // arrays overlap test: dispatch to disjoint stub if necessary.
1006     RuntimeAddress nooverlap(nooverlap_target);
1007     __ cmpptr(dst, src);
1008     __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
1009     __ jump_cc(Assembler::belowEqual, nooverlap);
1010     __ cmpptr(dst, end);
1011     __ jump_cc(Assembler::aboveEqual, nooverlap);
1012 
1013     if (t == T_OBJECT) {
1014       __ testl(count, count);
1015       __ jcc(Assembler::zero, L_0_count);
1016     }
1017 
1018     DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1019     if (dest_uninitialized) {
1020       decorators |= IS_DEST_UNINITIALIZED;
1021     }
1022     if (aligned) {
1023       decorators |= ARRAYCOPY_ALIGNED;
1024     }
1025 
1026     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1027     bs->arraycopy_prologue(_masm, decorators, t, from, to, count);
1028 
1029     // copy from high to low
1030     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1031     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1032     if (t == T_BYTE || t == T_SHORT) {
1033       // Align the end of destination array at 4 bytes address boundary
1034       __ lea(end, Address(dst, count, sf, 0));
1035       if (t == T_BYTE) {
1036         // One byte misalignment happens only for byte arrays
1037         __ testl(end, 1);
1038         __ jccb(Assembler::zero, L_skip_align1);
1039         __ decrement(count);
1040         __ movb(rdx, Address(from, count, sf, 0));
1041         __ movb(Address(to, count, sf, 0), rdx);
1042       __ BIND(L_skip_align1);
1043       }
1044       // Two bytes misalignment happens only for byte and short (char) arrays
1045       __ testl(end, 2);
1046       __ jccb(Assembler::zero, L_skip_align2);
1047       __ subptr(count, 1<<(shift-1));
1048       __ movw(rdx, Address(from, count, sf, 0));
1049       __ movw(Address(to, count, sf, 0), rdx);
1050     __ BIND(L_skip_align2);
1051       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1052       __ jcc(Assembler::below, L_copy_4_bytes);
1053     }
1054 
1055     if (!VM_Version::supports_mmx()) {
1056       __ std();
1057       __ mov(rax, count); // Save 'count'
1058       __ mov(rdx, to);    // Save 'to'
1059       __ lea(rsi, Address(from, count, sf, -4));
1060       __ lea(rdi, Address(to  , count, sf, -4));
1061       __ shrptr(count, shift); // bytes count
1062       __ rep_mov();
1063       __ cld();
1064       __ mov(count, rax); // restore 'count'
1065       __ andl(count, (1<<shift)-1);      // mask the number of rest elements
1066       __ movptr(from, Address(rsp, 12+4)); // reread 'from'
1067       __ mov(to, rdx);   // restore 'to'
1068       __ jmpb(L_copy_2_bytes); // all dword were copied
1069    } else {
1070       // Align to 8 bytes the end of array. It is aligned to 4 bytes already.
1071       __ testptr(end, 4);
1072       __ jccb(Assembler::zero, L_copy_8_bytes);
1073       __ subl(count, 1<<shift);
1074       __ movl(rdx, Address(from, count, sf, 0));
1075       __ movl(Address(to, count, sf, 0), rdx);
1076       __ jmpb(L_copy_8_bytes);
1077 
1078       __ align(OptoLoopAlignment);
1079       // Move 8 bytes
1080     __ BIND(L_copy_8_bytes_loop);
1081       if (UseXMMForArrayCopy) {
1082         __ movq(xmm0, Address(from, count, sf, 0));
1083         __ movq(Address(to, count, sf, 0), xmm0);
1084       } else {
1085         __ movq(mmx0, Address(from, count, sf, 0));
1086         __ movq(Address(to, count, sf, 0), mmx0);
1087       }
1088     __ BIND(L_copy_8_bytes);
1089       __ subl(count, 2<<shift);
1090       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1091       __ addl(count, 2<<shift);
1092       if (!UseXMMForArrayCopy) {
1093         __ emms();
1094       }
1095     }
1096   __ BIND(L_copy_4_bytes);
1097     // copy prefix qword
1098     __ testl(count, 1<<shift);
1099     __ jccb(Assembler::zero, L_copy_2_bytes);
1100     __ movl(rdx, Address(from, count, sf, -4));
1101     __ movl(Address(to, count, sf, -4), rdx);
1102 
1103     if (t == T_BYTE || t == T_SHORT) {
1104         __ subl(count, (1<<shift));
1105       __ BIND(L_copy_2_bytes);
1106         // copy prefix dword
1107         __ testl(count, 1<<(shift-1));
1108         __ jccb(Assembler::zero, L_copy_byte);
1109         __ movw(rdx, Address(from, count, sf, -2));
1110         __ movw(Address(to, count, sf, -2), rdx);
1111         if (t == T_BYTE) {
1112           __ subl(count, 1<<(shift-1));
1113         __ BIND(L_copy_byte);
1114           // copy prefix byte
1115           __ testl(count, 1);
1116           __ jccb(Assembler::zero, L_exit);
1117           __ movb(rdx, Address(from, 0));
1118           __ movb(Address(to, 0), rdx);
1119         __ BIND(L_exit);
1120         } else {
1121         __ BIND(L_copy_byte);
1122         }
1123     } else {
1124     __ BIND(L_copy_2_bytes);
1125     }
1126 
1127     __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1128     bs->arraycopy_epilogue(_masm, decorators, t, from, to, count);
1129 
1130     if (t == T_OBJECT) {
1131     __ BIND(L_0_count);
1132     }
1133     inc_copy_counter_np(t);
1134     __ pop(rdi);
1135     __ pop(rsi);
1136     __ leave(); // required for proper stackwalking of RuntimeStub frame
1137     __ xorptr(rax, rax); // return 0
1138     __ ret(0);
1139     return start;
1140   }
1141 
1142 
1143   address generate_disjoint_long_copy(address* entry, const char *name) {
1144     __ align(CodeEntryAlignment);
1145     StubCodeMark mark(this, "StubRoutines", name);
1146     address start = __ pc();
1147 
1148     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1149     const Register from       = rax;  // source array address
1150     const Register to         = rdx;  // destination array address
1151     const Register count      = rcx;  // elements count
1152     const Register to_from    = rdx;  // (to - from)
1153 
1154     __ enter(); // required for proper stackwalking of RuntimeStub frame
1155     __ movptr(from , Address(rsp, 8+0));       // from
1156     __ movptr(to   , Address(rsp, 8+4));       // to
1157     __ movl2ptr(count, Address(rsp, 8+8));     // count
1158 
1159     *entry = __ pc(); // Entry point from conjoint arraycopy stub.
1160     BLOCK_COMMENT("Entry:");
1161 
1162     __ subptr(to, from); // to --> to_from
1163     if (VM_Version::supports_mmx()) {
1164       if (UseXMMForArrayCopy) {
1165         xmm_copy_forward(from, to_from, count);
1166       } else {
1167         mmx_copy_forward(from, to_from, count);
1168       }
1169     } else {
1170       __ jmpb(L_copy_8_bytes);
1171       __ align(OptoLoopAlignment);
1172     __ BIND(L_copy_8_bytes_loop);
1173       __ fild_d(Address(from, 0));
1174       __ fistp_d(Address(from, to_from, Address::times_1));
1175       __ addptr(from, 8);
1176     __ BIND(L_copy_8_bytes);
1177       __ decrement(count);
1178       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1179     }
1180     inc_copy_counter_np(T_LONG);
1181     __ leave(); // required for proper stackwalking of RuntimeStub frame
1182     __ vzeroupper();
1183     __ xorptr(rax, rax); // return 0
1184     __ ret(0);
1185     return start;
1186   }
1187 
1188   address generate_conjoint_long_copy(address nooverlap_target,
1189                                       address* entry, const char *name) {
1190     __ align(CodeEntryAlignment);
1191     StubCodeMark mark(this, "StubRoutines", name);
1192     address start = __ pc();
1193 
1194     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1195     const Register from       = rax;  // source array address
1196     const Register to         = rdx;  // destination array address
1197     const Register count      = rcx;  // elements count
1198     const Register end_from   = rax;  // source array end address
1199 
1200     __ enter(); // required for proper stackwalking of RuntimeStub frame
1201     __ movptr(from , Address(rsp, 8+0));       // from
1202     __ movptr(to   , Address(rsp, 8+4));       // to
1203     __ movl2ptr(count, Address(rsp, 8+8));     // count
1204 
1205     *entry = __ pc(); // Entry point from generic arraycopy stub.
1206     BLOCK_COMMENT("Entry:");
1207 
1208     // arrays overlap test
1209     __ cmpptr(to, from);
1210     RuntimeAddress nooverlap(nooverlap_target);
1211     __ jump_cc(Assembler::belowEqual, nooverlap);
1212     __ lea(end_from, Address(from, count, Address::times_8, 0));
1213     __ cmpptr(to, end_from);
1214     __ movptr(from, Address(rsp, 8));  // from
1215     __ jump_cc(Assembler::aboveEqual, nooverlap);
1216 
1217     __ jmpb(L_copy_8_bytes);
1218 
1219     __ align(OptoLoopAlignment);
1220   __ BIND(L_copy_8_bytes_loop);
1221     if (VM_Version::supports_mmx()) {
1222       if (UseXMMForArrayCopy) {
1223         __ movq(xmm0, Address(from, count, Address::times_8));
1224         __ movq(Address(to, count, Address::times_8), xmm0);
1225       } else {
1226         __ movq(mmx0, Address(from, count, Address::times_8));
1227         __ movq(Address(to, count, Address::times_8), mmx0);
1228       }
1229     } else {
1230       __ fild_d(Address(from, count, Address::times_8));
1231       __ fistp_d(Address(to, count, Address::times_8));
1232     }
1233   __ BIND(L_copy_8_bytes);
1234     __ decrement(count);
1235     __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1236 
1237     if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
1238       __ emms();
1239     }
1240     inc_copy_counter_np(T_LONG);
1241     __ leave(); // required for proper stackwalking of RuntimeStub frame
1242     __ xorptr(rax, rax); // return 0
1243     __ ret(0);
1244     return start;
1245   }
1246 
1247 
1248   // Helper for generating a dynamic type check.
1249   // The sub_klass must be one of {rbx, rdx, rsi}.
1250   // The temp is killed.
1251   void generate_type_check(Register sub_klass,
1252                            Address& super_check_offset_addr,
1253                            Address& super_klass_addr,
1254                            Register temp,
1255                            Label* L_success, Label* L_failure) {
1256     BLOCK_COMMENT("type_check:");
1257 
1258     Label L_fallthrough;
1259 #define LOCAL_JCC(assembler_con, label_ptr)                             \
1260     if (label_ptr != NULL)  __ jcc(assembler_con, *(label_ptr));        \
1261     else                    __ jcc(assembler_con, L_fallthrough) /*omit semi*/
1262 
1263     // The following is a strange variation of the fast path which requires
1264     // one less register, because needed values are on the argument stack.
1265     // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
1266     //                                  L_success, L_failure, NULL);
1267     assert_different_registers(sub_klass, temp);
1268 
1269     int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1270 
1271     // if the pointers are equal, we are done (e.g., String[] elements)
1272     __ cmpptr(sub_klass, super_klass_addr);
1273     LOCAL_JCC(Assembler::equal, L_success);
1274 
1275     // check the supertype display:
1276     __ movl2ptr(temp, super_check_offset_addr);
1277     Address super_check_addr(sub_klass, temp, Address::times_1, 0);
1278     __ movptr(temp, super_check_addr); // load displayed supertype
1279     __ cmpptr(temp, super_klass_addr); // test the super type
1280     LOCAL_JCC(Assembler::equal, L_success);
1281 
1282     // if it was a primary super, we can just fail immediately
1283     __ cmpl(super_check_offset_addr, sc_offset);
1284     LOCAL_JCC(Assembler::notEqual, L_failure);
1285 
1286     // The repne_scan instruction uses fixed registers, which will get spilled.
1287     // We happen to know this works best when super_klass is in rax.
1288     Register super_klass = temp;
1289     __ movptr(super_klass, super_klass_addr);
1290     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
1291                                      L_success, L_failure);
1292 
1293     __ bind(L_fallthrough);
1294 
1295     if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
1296     if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
1297 
1298 #undef LOCAL_JCC
1299   }
1300 
1301   //
1302   //  Generate checkcasting array copy stub
1303   //
1304   //  Input:
1305   //    4(rsp)   - source array address
1306   //    8(rsp)   - destination array address
1307   //   12(rsp)   - element count, can be zero
1308   //   16(rsp)   - size_t ckoff (super_check_offset)
1309   //   20(rsp)   - oop ckval (super_klass)
1310   //
1311   //  Output:
1312   //    rax, ==  0  -  success
1313   //    rax, == -1^K - failure, where K is partial transfer count
1314   //
1315   address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitialized = false) {
1316     __ align(CodeEntryAlignment);
1317     StubCodeMark mark(this, "StubRoutines", name);
1318     address start = __ pc();
1319 
1320     Label L_load_element, L_store_element, L_do_card_marks, L_done;
1321 
1322     // register use:
1323     //  rax, rdx, rcx -- loop control (end_from, end_to, count)
1324     //  rdi, rsi      -- element access (oop, klass)
1325     //  rbx,           -- temp
1326     const Register from       = rax;    // source array address
1327     const Register to         = rdx;    // destination array address
1328     const Register length     = rcx;    // elements count
1329     const Register elem       = rdi;    // each oop copied
1330     const Register elem_klass = rsi;    // each elem._klass (sub_klass)
1331     const Register temp       = rbx;    // lone remaining temp
1332 
1333     __ enter(); // required for proper stackwalking of RuntimeStub frame
1334 
1335     __ push(rsi);
1336     __ push(rdi);
1337     __ push(rbx);
1338 
1339     Address   from_arg(rsp, 16+ 4);     // from
1340     Address     to_arg(rsp, 16+ 8);     // to
1341     Address length_arg(rsp, 16+12);     // elements count
1342     Address  ckoff_arg(rsp, 16+16);     // super_check_offset
1343     Address  ckval_arg(rsp, 16+20);     // super_klass
1344 
1345     // Load up:
1346     __ movptr(from,     from_arg);
1347     __ movptr(to,         to_arg);
1348     __ movl2ptr(length, length_arg);
1349 
1350     if (entry != NULL) {
1351       *entry = __ pc(); // Entry point from generic arraycopy stub.
1352       BLOCK_COMMENT("Entry:");
1353     }
1354 
1355     //---------------------------------------------------------------
1356     // Assembler stub will be used for this call to arraycopy
1357     // if the two arrays are subtypes of Object[] but the
1358     // destination array type is not equal to or a supertype
1359     // of the source type.  Each element must be separately
1360     // checked.
1361 
1362     // Loop-invariant addresses.  They are exclusive end pointers.
1363     Address end_from_addr(from, length, Address::times_ptr, 0);
1364     Address   end_to_addr(to,   length, Address::times_ptr, 0);
1365 
1366     Register end_from = from;           // re-use
1367     Register end_to   = to;             // re-use
1368     Register count    = length;         // re-use
1369 
1370     // Loop-variant addresses.  They assume post-incremented count < 0.
1371     Address from_element_addr(end_from, count, Address::times_ptr, 0);
1372     Address   to_element_addr(end_to,   count, Address::times_ptr, 0);
1373     Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
1374 
1375     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST;
1376     if (dest_uninitialized) {
1377       decorators |= IS_DEST_UNINITIALIZED;
1378     }
1379 
1380     BasicType type = T_OBJECT;
1381     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1382     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1383 
1384     // Copy from low to high addresses, indexed from the end of each array.
1385     __ lea(end_from, end_from_addr);
1386     __ lea(end_to,   end_to_addr);
1387     assert(length == count, "");        // else fix next line:
1388     __ negptr(count);                   // negate and test the length
1389     __ jccb(Assembler::notZero, L_load_element);
1390 
1391     // Empty array:  Nothing to do.
1392     __ xorptr(rax, rax);                  // return 0 on (trivial) success
1393     __ jmp(L_done);
1394 
1395     // ======== begin loop ========
1396     // (Loop is rotated; its entry is L_load_element.)
1397     // Loop control:
1398     //   for (count = -count; count != 0; count++)
1399     // Base pointers src, dst are biased by 8*count,to last element.
1400     __ align(OptoLoopAlignment);
1401 
1402     __ BIND(L_store_element);
1403     __ movptr(to_element_addr, elem);     // store the oop
1404     __ increment(count);                // increment the count toward zero
1405     __ jccb(Assembler::zero, L_do_card_marks);
1406 
1407     // ======== loop entry is here ========
1408     __ BIND(L_load_element);
1409     __ movptr(elem, from_element_addr);   // load the oop
1410     __ testptr(elem, elem);
1411     __ jccb(Assembler::zero, L_store_element);
1412 
1413     // (Could do a trick here:  Remember last successful non-null
1414     // element stored and make a quick oop equality check on it.)
1415 
1416     __ movptr(elem_klass, elem_klass_addr); // query the object klass
1417     generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
1418                         &L_store_element, NULL);
1419     // (On fall-through, we have failed the element type check.)
1420     // ======== end loop ========
1421 
1422     // It was a real error; we must depend on the caller to finish the job.
1423     // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
1424     // Emit GC store barriers for the oops we have copied (length_arg + count),
1425     // and report their number to the caller.
1426     assert_different_registers(to, count, rax);
1427     Label L_post_barrier;
1428     __ addl(count, length_arg);         // transfers = (length - remaining)
1429     __ movl2ptr(rax, count);            // save the value
1430     __ notptr(rax);                     // report (-1^K) to caller (does not affect flags)
1431     __ jccb(Assembler::notZero, L_post_barrier);
1432     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
1433 
1434     // Come here on success only.
1435     __ BIND(L_do_card_marks);
1436     __ xorptr(rax, rax);                // return 0 on success
1437     __ movl2ptr(count, length_arg);
1438 
1439     __ BIND(L_post_barrier);
1440     __ movptr(to, to_arg);              // reload
1441     bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
1442 
1443     // Common exit point (success or failure).
1444     __ BIND(L_done);
1445     __ pop(rbx);
1446     __ pop(rdi);
1447     __ pop(rsi);
1448     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1449     __ leave(); // required for proper stackwalking of RuntimeStub frame
1450     __ ret(0);
1451 
1452     return start;
1453   }
1454 
1455   //
1456   //  Generate 'unsafe' array copy stub
1457   //  Though just as safe as the other stubs, it takes an unscaled
1458   //  size_t argument instead of an element count.
1459   //
1460   //  Input:
1461   //    4(rsp)   - source array address
1462   //    8(rsp)   - destination array address
1463   //   12(rsp)   - byte count, can be zero
1464   //
1465   //  Output:
1466   //    rax, ==  0  -  success
1467   //    rax, == -1  -  need to call System.arraycopy
1468   //
1469   // Examines the alignment of the operands and dispatches
1470   // to a long, int, short, or byte copy loop.
1471   //
1472   address generate_unsafe_copy(const char *name,
1473                                address byte_copy_entry,
1474                                address short_copy_entry,
1475                                address int_copy_entry,
1476                                address long_copy_entry) {
1477 
1478     Label L_long_aligned, L_int_aligned, L_short_aligned;
1479 
1480     __ align(CodeEntryAlignment);
1481     StubCodeMark mark(this, "StubRoutines", name);
1482     address start = __ pc();
1483 
1484     const Register from       = rax;  // source array address
1485     const Register to         = rdx;  // destination array address
1486     const Register count      = rcx;  // elements count
1487 
1488     __ enter(); // required for proper stackwalking of RuntimeStub frame
1489     __ push(rsi);
1490     __ push(rdi);
1491     Address  from_arg(rsp, 12+ 4);      // from
1492     Address    to_arg(rsp, 12+ 8);      // to
1493     Address count_arg(rsp, 12+12);      // byte count
1494 
1495     // Load up:
1496     __ movptr(from ,  from_arg);
1497     __ movptr(to   ,    to_arg);
1498     __ movl2ptr(count, count_arg);
1499 
1500     // bump this on entry, not on exit:
1501     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
1502 
1503     const Register bits = rsi;
1504     __ mov(bits, from);
1505     __ orptr(bits, to);
1506     __ orptr(bits, count);
1507 
1508     __ testl(bits, BytesPerLong-1);
1509     __ jccb(Assembler::zero, L_long_aligned);
1510 
1511     __ testl(bits, BytesPerInt-1);
1512     __ jccb(Assembler::zero, L_int_aligned);
1513 
1514     __ testl(bits, BytesPerShort-1);
1515     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
1516 
1517     __ BIND(L_short_aligned);
1518     __ shrptr(count, LogBytesPerShort); // size => short_count
1519     __ movl(count_arg, count);          // update 'count'
1520     __ jump(RuntimeAddress(short_copy_entry));
1521 
1522     __ BIND(L_int_aligned);
1523     __ shrptr(count, LogBytesPerInt); // size => int_count
1524     __ movl(count_arg, count);          // update 'count'
1525     __ jump(RuntimeAddress(int_copy_entry));
1526 
1527     __ BIND(L_long_aligned);
1528     __ shrptr(count, LogBytesPerLong); // size => qword_count
1529     __ movl(count_arg, count);          // update 'count'
1530     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1531     __ pop(rsi);
1532     __ jump(RuntimeAddress(long_copy_entry));
1533 
1534     return start;
1535   }
1536 
1537 
1538   // Perform range checks on the proposed arraycopy.
1539   // Smashes src_pos and dst_pos.  (Uses them up for temps.)
1540   void arraycopy_range_checks(Register src,
1541                               Register src_pos,
1542                               Register dst,
1543                               Register dst_pos,
1544                               Address& length,
1545                               Label& L_failed) {
1546     BLOCK_COMMENT("arraycopy_range_checks:");
1547     const Register src_end = src_pos;   // source array end position
1548     const Register dst_end = dst_pos;   // destination array end position
1549     __ addl(src_end, length); // src_pos + length
1550     __ addl(dst_end, length); // dst_pos + length
1551 
1552     //  if (src_pos + length > arrayOop(src)->length() ) FAIL;
1553     __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
1554     __ jcc(Assembler::above, L_failed);
1555 
1556     //  if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
1557     __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1558     __ jcc(Assembler::above, L_failed);
1559 
1560     BLOCK_COMMENT("arraycopy_range_checks done");
1561   }
1562 
1563 
1564   //
1565   //  Generate generic array copy stubs
1566   //
1567   //  Input:
1568   //     4(rsp)    -  src oop
1569   //     8(rsp)    -  src_pos
1570   //    12(rsp)    -  dst oop
1571   //    16(rsp)    -  dst_pos
1572   //    20(rsp)    -  element count
1573   //
1574   //  Output:
1575   //    rax, ==  0  -  success
1576   //    rax, == -1^K - failure, where K is partial transfer count
1577   //
1578   address generate_generic_copy(const char *name,
1579                                 address entry_jbyte_arraycopy,
1580                                 address entry_jshort_arraycopy,
1581                                 address entry_jint_arraycopy,
1582                                 address entry_oop_arraycopy,
1583                                 address entry_jlong_arraycopy,
1584                                 address entry_checkcast_arraycopy) {
1585     Label L_failed, L_failed_0, L_objArray;
1586 
1587     { int modulus = CodeEntryAlignment;
1588       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
1589       int advance = target - (__ offset() % modulus);
1590       if (advance < 0)  advance += modulus;
1591       if (advance > 0)  __ nop(advance);
1592     }
1593     StubCodeMark mark(this, "StubRoutines", name);
1594 
1595     // Short-hop target to L_failed.  Makes for denser prologue code.
1596     __ BIND(L_failed_0);
1597     __ jmp(L_failed);
1598     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
1599 
1600     __ align(CodeEntryAlignment);
1601     address start = __ pc();
1602 
1603     __ enter(); // required for proper stackwalking of RuntimeStub frame
1604     __ push(rsi);
1605     __ push(rdi);
1606 
1607     // bump this on entry, not on exit:
1608     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
1609 
1610     // Input values
1611     Address SRC     (rsp, 12+ 4);
1612     Address SRC_POS (rsp, 12+ 8);
1613     Address DST     (rsp, 12+12);
1614     Address DST_POS (rsp, 12+16);
1615     Address LENGTH  (rsp, 12+20);
1616 
1617     //-----------------------------------------------------------------------
1618     // Assembler stub will be used for this call to arraycopy
1619     // if the following conditions are met:
1620     //
1621     // (1) src and dst must not be null.
1622     // (2) src_pos must not be negative.
1623     // (3) dst_pos must not be negative.
1624     // (4) length  must not be negative.
1625     // (5) src klass and dst klass should be the same and not NULL.
1626     // (6) src and dst should be arrays.
1627     // (7) src_pos + length must not exceed length of src.
1628     // (8) dst_pos + length must not exceed length of dst.
1629     //
1630 
1631     const Register src     = rax;       // source array oop
1632     const Register src_pos = rsi;
1633     const Register dst     = rdx;       // destination array oop
1634     const Register dst_pos = rdi;
1635     const Register length  = rcx;       // transfer count
1636 
1637     //  if (src == NULL) return -1;
1638     __ movptr(src, SRC);      // src oop
1639     __ testptr(src, src);
1640     __ jccb(Assembler::zero, L_failed_0);
1641 
1642     //  if (src_pos < 0) return -1;
1643     __ movl2ptr(src_pos, SRC_POS);  // src_pos
1644     __ testl(src_pos, src_pos);
1645     __ jccb(Assembler::negative, L_failed_0);
1646 
1647     //  if (dst == NULL) return -1;
1648     __ movptr(dst, DST);      // dst oop
1649     __ testptr(dst, dst);
1650     __ jccb(Assembler::zero, L_failed_0);
1651 
1652     //  if (dst_pos < 0) return -1;
1653     __ movl2ptr(dst_pos, DST_POS);  // dst_pos
1654     __ testl(dst_pos, dst_pos);
1655     __ jccb(Assembler::negative, L_failed_0);
1656 
1657     //  if (length < 0) return -1;
1658     __ movl2ptr(length, LENGTH);   // length
1659     __ testl(length, length);
1660     __ jccb(Assembler::negative, L_failed_0);
1661 
1662     //  if (src->klass() == NULL) return -1;
1663     Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
1664     Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
1665     const Register rcx_src_klass = rcx;    // array klass
1666     __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
1667 
1668 #ifdef ASSERT
1669     //  assert(src->klass() != NULL);
1670     BLOCK_COMMENT("assert klasses not null");
1671     { Label L1, L2;
1672       __ testptr(rcx_src_klass, rcx_src_klass);
1673       __ jccb(Assembler::notZero, L2);   // it is broken if klass is NULL
1674       __ bind(L1);
1675       __ stop("broken null klass");
1676       __ bind(L2);
1677       __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
1678       __ jccb(Assembler::equal, L1);      // this would be broken also
1679       BLOCK_COMMENT("assert done");
1680     }
1681 #endif //ASSERT
1682 
1683     // Load layout helper (32-bits)
1684     //
1685     //  |array_tag|     | header_size | element_type |     |log2_element_size|
1686     // 32        30    24            16              8     2                 0
1687     //
1688     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
1689     //
1690 
1691     int lh_offset = in_bytes(Klass::layout_helper_offset());
1692     Address src_klass_lh_addr(rcx_src_klass, lh_offset);
1693 
1694     // Handle objArrays completely differently...
1695     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
1696     __ cmpl(src_klass_lh_addr, objArray_lh);
1697     __ jcc(Assembler::equal, L_objArray);
1698 
1699     //  if (src->klass() != dst->klass()) return -1;
1700     __ cmpptr(rcx_src_klass, dst_klass_addr);
1701     __ jccb(Assembler::notEqual, L_failed_0);
1702 
1703     const Register rcx_lh = rcx;  // layout helper
1704     assert(rcx_lh == rcx_src_klass, "known alias");
1705     __ movl(rcx_lh, src_klass_lh_addr);
1706 
1707     //  if (!src->is_Array()) return -1;
1708     __ cmpl(rcx_lh, Klass::_lh_neutral_value);
1709     __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
1710 
1711     // At this point, it is known to be a typeArray (array_tag 0x3).
1712 #ifdef ASSERT
1713     { Label L;
1714       __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
1715       __ jcc(Assembler::greaterEqual, L); // signed cmp
1716       __ stop("must be a primitive array");
1717       __ bind(L);
1718     }
1719 #endif
1720 
1721     assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
1722     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1723 
1724     // TypeArrayKlass
1725     //
1726     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
1727     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
1728     //
1729     const Register rsi_offset = rsi; // array offset
1730     const Register src_array  = src; // src array offset
1731     const Register dst_array  = dst; // dst array offset
1732     const Register rdi_elsize = rdi; // log2 element size
1733 
1734     __ mov(rsi_offset, rcx_lh);
1735     __ shrptr(rsi_offset, Klass::_lh_header_size_shift);
1736     __ andptr(rsi_offset, Klass::_lh_header_size_mask);   // array_offset
1737     __ addptr(src_array, rsi_offset);  // src array offset
1738     __ addptr(dst_array, rsi_offset);  // dst array offset
1739     __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
1740 
1741     // next registers should be set before the jump to corresponding stub
1742     const Register from       = src; // source array address
1743     const Register to         = dst; // destination array address
1744     const Register count      = rcx; // elements count
1745     // some of them should be duplicated on stack
1746 #define FROM   Address(rsp, 12+ 4)
1747 #define TO     Address(rsp, 12+ 8)   // Not used now
1748 #define COUNT  Address(rsp, 12+12)   // Only for oop arraycopy
1749 
1750     BLOCK_COMMENT("scale indexes to element size");
1751     __ movl2ptr(rsi, SRC_POS);  // src_pos
1752     __ shlptr(rsi);             // src_pos << rcx (log2 elsize)
1753     assert(src_array == from, "");
1754     __ addptr(from, rsi);       // from = src_array + SRC_POS << log2 elsize
1755     __ movl2ptr(rdi, DST_POS);  // dst_pos
1756     __ shlptr(rdi);             // dst_pos << rcx (log2 elsize)
1757     assert(dst_array == to, "");
1758     __ addptr(to,  rdi);        // to   = dst_array + DST_POS << log2 elsize
1759     __ movptr(FROM, from);      // src_addr
1760     __ mov(rdi_elsize, rcx_lh); // log2 elsize
1761     __ movl2ptr(count, LENGTH); // elements count
1762 
1763     BLOCK_COMMENT("choose copy loop based on element size");
1764     __ cmpl(rdi_elsize, 0);
1765 
1766     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
1767     __ cmpl(rdi_elsize, LogBytesPerShort);
1768     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
1769     __ cmpl(rdi_elsize, LogBytesPerInt);
1770     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
1771 #ifdef ASSERT
1772     __ cmpl(rdi_elsize, LogBytesPerLong);
1773     __ jccb(Assembler::notEqual, L_failed);
1774 #endif
1775     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1776     __ pop(rsi);
1777     __ jump(RuntimeAddress(entry_jlong_arraycopy));
1778 
1779   __ BIND(L_failed);
1780     __ xorptr(rax, rax);
1781     __ notptr(rax); // return -1
1782     __ pop(rdi);
1783     __ pop(rsi);
1784     __ leave(); // required for proper stackwalking of RuntimeStub frame
1785     __ ret(0);
1786 
1787     // ObjArrayKlass
1788   __ BIND(L_objArray);
1789     // live at this point:  rcx_src_klass, src[_pos], dst[_pos]
1790 
1791     Label L_plain_copy, L_checkcast_copy;
1792     //  test array classes for subtyping
1793     __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
1794     __ jccb(Assembler::notEqual, L_checkcast_copy);
1795 
1796     // Identically typed arrays can be copied without element-wise checks.
1797     assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
1798     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1799 
1800   __ BIND(L_plain_copy);
1801     __ movl2ptr(count, LENGTH); // elements count
1802     __ movl2ptr(src_pos, SRC_POS);  // reload src_pos
1803     __ lea(from, Address(src, src_pos, Address::times_ptr,
1804                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
1805     __ movl2ptr(dst_pos, DST_POS);  // reload dst_pos
1806     __ lea(to,   Address(dst, dst_pos, Address::times_ptr,
1807                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
1808     __ movptr(FROM,  from);   // src_addr
1809     __ movptr(TO,    to);     // dst_addr
1810     __ movl(COUNT, count);  // count
1811     __ jump(RuntimeAddress(entry_oop_arraycopy));
1812 
1813   __ BIND(L_checkcast_copy);
1814     // live at this point:  rcx_src_klass, dst[_pos], src[_pos]
1815     {
1816       // Handy offsets:
1817       int  ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
1818       int sco_offset = in_bytes(Klass::super_check_offset_offset());
1819 
1820       Register rsi_dst_klass = rsi;
1821       Register rdi_temp      = rdi;
1822       assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
1823       assert(rdi_temp      == dst_pos, "expected alias w/ dst_pos");
1824       Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
1825 
1826       // Before looking at dst.length, make sure dst is also an objArray.
1827       __ movptr(rsi_dst_klass, dst_klass_addr);
1828       __ cmpl(dst_klass_lh_addr, objArray_lh);
1829       __ jccb(Assembler::notEqual, L_failed);
1830 
1831       // It is safe to examine both src.length and dst.length.
1832       __ movl2ptr(src_pos, SRC_POS);        // reload rsi
1833       arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1834       // (Now src_pos and dst_pos are killed, but not src and dst.)
1835 
1836       // We'll need this temp (don't forget to pop it after the type check).
1837       __ push(rbx);
1838       Register rbx_src_klass = rbx;
1839 
1840       __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
1841       __ movptr(rsi_dst_klass, dst_klass_addr);
1842       Address super_check_offset_addr(rsi_dst_klass, sco_offset);
1843       Label L_fail_array_check;
1844       generate_type_check(rbx_src_klass,
1845                           super_check_offset_addr, dst_klass_addr,
1846                           rdi_temp, NULL, &L_fail_array_check);
1847       // (On fall-through, we have passed the array type check.)
1848       __ pop(rbx);
1849       __ jmp(L_plain_copy);
1850 
1851       __ BIND(L_fail_array_check);
1852       // Reshuffle arguments so we can call checkcast_arraycopy:
1853 
1854       // match initial saves for checkcast_arraycopy
1855       // push(rsi);    // already done; see above
1856       // push(rdi);    // already done; see above
1857       // push(rbx);    // already done; see above
1858 
1859       // Marshal outgoing arguments now, freeing registers.
1860       Address   from_arg(rsp, 16+ 4);   // from
1861       Address     to_arg(rsp, 16+ 8);   // to
1862       Address length_arg(rsp, 16+12);   // elements count
1863       Address  ckoff_arg(rsp, 16+16);   // super_check_offset
1864       Address  ckval_arg(rsp, 16+20);   // super_klass
1865 
1866       Address SRC_POS_arg(rsp, 16+ 8);
1867       Address DST_POS_arg(rsp, 16+16);
1868       Address  LENGTH_arg(rsp, 16+20);
1869       // push rbx, changed the incoming offsets (why not just use rbp,??)
1870       // assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
1871 
1872       __ movptr(rbx, Address(rsi_dst_klass, ek_offset));
1873       __ movl2ptr(length, LENGTH_arg);    // reload elements count
1874       __ movl2ptr(src_pos, SRC_POS_arg);  // reload src_pos
1875       __ movl2ptr(dst_pos, DST_POS_arg);  // reload dst_pos
1876 
1877       __ movptr(ckval_arg, rbx);          // destination element type
1878       __ movl(rbx, Address(rbx, sco_offset));
1879       __ movl(ckoff_arg, rbx);          // corresponding class check offset
1880 
1881       __ movl(length_arg, length);      // outgoing length argument
1882 
1883       __ lea(from, Address(src, src_pos, Address::times_ptr,
1884                             arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1885       __ movptr(from_arg, from);
1886 
1887       __ lea(to, Address(dst, dst_pos, Address::times_ptr,
1888                           arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1889       __ movptr(to_arg, to);
1890       __ jump(RuntimeAddress(entry_checkcast_arraycopy));
1891     }
1892 
1893     return start;
1894   }
1895 
1896   void generate_arraycopy_stubs() {
1897     address entry;
1898     address entry_jbyte_arraycopy;
1899     address entry_jshort_arraycopy;
1900     address entry_jint_arraycopy;
1901     address entry_oop_arraycopy;
1902     address entry_jlong_arraycopy;
1903     address entry_checkcast_arraycopy;
1904 
1905     StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
1906         generate_disjoint_copy(T_BYTE,  true, Address::times_1, &entry,
1907                                "arrayof_jbyte_disjoint_arraycopy");
1908     StubRoutines::_arrayof_jbyte_arraycopy =
1909         generate_conjoint_copy(T_BYTE,  true, Address::times_1,  entry,
1910                                NULL, "arrayof_jbyte_arraycopy");
1911     StubRoutines::_jbyte_disjoint_arraycopy =
1912         generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
1913                                "jbyte_disjoint_arraycopy");
1914     StubRoutines::_jbyte_arraycopy =
1915         generate_conjoint_copy(T_BYTE, false, Address::times_1,  entry,
1916                                &entry_jbyte_arraycopy, "jbyte_arraycopy");
1917 
1918     StubRoutines::_arrayof_jshort_disjoint_arraycopy =
1919         generate_disjoint_copy(T_SHORT,  true, Address::times_2, &entry,
1920                                "arrayof_jshort_disjoint_arraycopy");
1921     StubRoutines::_arrayof_jshort_arraycopy =
1922         generate_conjoint_copy(T_SHORT,  true, Address::times_2,  entry,
1923                                NULL, "arrayof_jshort_arraycopy");
1924     StubRoutines::_jshort_disjoint_arraycopy =
1925         generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
1926                                "jshort_disjoint_arraycopy");
1927     StubRoutines::_jshort_arraycopy =
1928         generate_conjoint_copy(T_SHORT, false, Address::times_2,  entry,
1929                                &entry_jshort_arraycopy, "jshort_arraycopy");
1930 
1931     // Next arrays are always aligned on 4 bytes at least.
1932     StubRoutines::_jint_disjoint_arraycopy =
1933         generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
1934                                "jint_disjoint_arraycopy");
1935     StubRoutines::_jint_arraycopy =
1936         generate_conjoint_copy(T_INT, true, Address::times_4,  entry,
1937                                &entry_jint_arraycopy, "jint_arraycopy");
1938 
1939     StubRoutines::_oop_disjoint_arraycopy =
1940         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
1941                                "oop_disjoint_arraycopy");
1942     StubRoutines::_oop_arraycopy =
1943         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
1944                                &entry_oop_arraycopy, "oop_arraycopy");
1945 
1946     StubRoutines::_oop_disjoint_arraycopy_uninit =
1947         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
1948                                "oop_disjoint_arraycopy_uninit",
1949                                /*dest_uninitialized*/true);
1950     StubRoutines::_oop_arraycopy_uninit =
1951         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
1952                                NULL, "oop_arraycopy_uninit",
1953                                /*dest_uninitialized*/true);
1954 
1955     StubRoutines::_jlong_disjoint_arraycopy =
1956         generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
1957     StubRoutines::_jlong_arraycopy =
1958         generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
1959                                     "jlong_arraycopy");
1960 
1961     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
1962     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
1963     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
1964     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
1965     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
1966     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
1967 
1968     StubRoutines::_arrayof_jint_disjoint_arraycopy       = StubRoutines::_jint_disjoint_arraycopy;
1969     StubRoutines::_arrayof_oop_disjoint_arraycopy        = StubRoutines::_oop_disjoint_arraycopy;
1970     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;
1971     StubRoutines::_arrayof_jlong_disjoint_arraycopy      = StubRoutines::_jlong_disjoint_arraycopy;
1972 
1973     StubRoutines::_arrayof_jint_arraycopy       = StubRoutines::_jint_arraycopy;
1974     StubRoutines::_arrayof_oop_arraycopy        = StubRoutines::_oop_arraycopy;
1975     StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;
1976     StubRoutines::_arrayof_jlong_arraycopy      = StubRoutines::_jlong_arraycopy;
1977 
1978     StubRoutines::_checkcast_arraycopy =
1979         generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
1980     StubRoutines::_checkcast_arraycopy_uninit =
1981         generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, /*dest_uninitialized*/true);
1982 
1983     StubRoutines::_unsafe_arraycopy =
1984         generate_unsafe_copy("unsafe_arraycopy",
1985                                entry_jbyte_arraycopy,
1986                                entry_jshort_arraycopy,
1987                                entry_jint_arraycopy,
1988                                entry_jlong_arraycopy);
1989 
1990     StubRoutines::_generic_arraycopy =
1991         generate_generic_copy("generic_arraycopy",
1992                                entry_jbyte_arraycopy,
1993                                entry_jshort_arraycopy,
1994                                entry_jint_arraycopy,
1995                                entry_oop_arraycopy,
1996                                entry_jlong_arraycopy,
1997                                entry_checkcast_arraycopy);
1998   }
1999 
2000   // AES intrinsic stubs
2001   enum {AESBlockSize = 16};
2002 
2003   address generate_key_shuffle_mask() {
2004     __ align(16);
2005     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
2006     address start = __ pc();
2007     __ emit_data(0x00010203, relocInfo::none, 0 );
2008     __ emit_data(0x04050607, relocInfo::none, 0 );
2009     __ emit_data(0x08090a0b, relocInfo::none, 0 );
2010     __ emit_data(0x0c0d0e0f, relocInfo::none, 0 );
2011     return start;
2012   }
2013 
2014   address generate_counter_shuffle_mask() {
2015     __ align(16);
2016     StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask");
2017     address start = __ pc();
2018     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
2019     __ emit_data(0x08090a0b, relocInfo::none, 0);
2020     __ emit_data(0x04050607, relocInfo::none, 0);
2021     __ emit_data(0x00010203, relocInfo::none, 0);
2022     return start;
2023   }
2024 
2025   // Utility routine for loading a 128-bit key word in little endian format
2026   // can optionally specify that the shuffle mask is already in an xmmregister
2027   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2028     __ movdqu(xmmdst, Address(key, offset));
2029     if (xmm_shuf_mask != NULL) {
2030       __ pshufb(xmmdst, xmm_shuf_mask);
2031     } else {
2032       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2033     }
2034   }
2035 
2036   // aesenc using specified key+offset
2037   // can optionally specify that the shuffle mask is already in an xmmregister
2038   void aes_enc_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2039     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2040     __ aesenc(xmmdst, xmmtmp);
2041   }
2042 
2043   // aesdec using specified key+offset
2044   // can optionally specify that the shuffle mask is already in an xmmregister
2045   void aes_dec_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2046     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2047     __ aesdec(xmmdst, xmmtmp);
2048   }
2049 
2050   // Utility routine for increase 128bit counter (iv in CTR mode)
2051   //  XMM_128bit,  D3, D2, D1, D0
2052   void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) {
2053     __ pextrd(reg, xmmdst, 0x0);
2054     __ addl(reg, inc_delta);
2055     __ pinsrd(xmmdst, reg, 0x0);
2056     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2057 
2058     __ pextrd(reg, xmmdst, 0x01); // Carry-> D1
2059     __ addl(reg, 0x01);
2060     __ pinsrd(xmmdst, reg, 0x01);
2061     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2062 
2063     __ pextrd(reg, xmmdst, 0x02); // Carry-> D2
2064     __ addl(reg, 0x01);
2065     __ pinsrd(xmmdst, reg, 0x02);
2066     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2067 
2068     __ pextrd(reg, xmmdst, 0x03); // Carry -> D3
2069     __ addl(reg, 0x01);
2070     __ pinsrd(xmmdst, reg, 0x03);
2071 
2072     __ BIND(next_block);          // next instruction
2073   }
2074 
2075 
2076   // Arguments:
2077   //
2078   // Inputs:
2079   //   c_rarg0   - source byte array address
2080   //   c_rarg1   - destination byte array address
2081   //   c_rarg2   - K (key) in little endian int array
2082   //
2083   address generate_aescrypt_encryptBlock() {
2084     assert(UseAES, "need AES instructions and misaligned SSE support");
2085     __ align(CodeEntryAlignment);
2086     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2087     Label L_doLast;
2088     address start = __ pc();
2089 
2090     const Register from        = rdx;      // source array address
2091     const Register to          = rdx;      // destination array address
2092     const Register key         = rcx;      // key array address
2093     const Register keylen      = rax;
2094     const Address  from_param(rbp, 8+0);
2095     const Address  to_param  (rbp, 8+4);
2096     const Address  key_param (rbp, 8+8);
2097 
2098     const XMMRegister xmm_result = xmm0;
2099     const XMMRegister xmm_key_shuf_mask = xmm1;
2100     const XMMRegister xmm_temp1  = xmm2;
2101     const XMMRegister xmm_temp2  = xmm3;
2102     const XMMRegister xmm_temp3  = xmm4;
2103     const XMMRegister xmm_temp4  = xmm5;
2104 
2105     __ enter();   // required for proper stackwalking of RuntimeStub frame
2106 
2107     __ movptr(from, from_param);
2108     __ movptr(key, key_param);
2109 
2110     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2111     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2112 
2113     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2114     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
2115     __ movptr(to, to_param);
2116 
2117     // For encryption, the java expanded key ordering is just what we need
2118 
2119     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
2120     __ pxor(xmm_result, xmm_temp1);
2121 
2122     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2123     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2124     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2125     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2126 
2127     __ aesenc(xmm_result, xmm_temp1);
2128     __ aesenc(xmm_result, xmm_temp2);
2129     __ aesenc(xmm_result, xmm_temp3);
2130     __ aesenc(xmm_result, xmm_temp4);
2131 
2132     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2133     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2134     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2135     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2136 
2137     __ aesenc(xmm_result, xmm_temp1);
2138     __ aesenc(xmm_result, xmm_temp2);
2139     __ aesenc(xmm_result, xmm_temp3);
2140     __ aesenc(xmm_result, xmm_temp4);
2141 
2142     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2143     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2144 
2145     __ cmpl(keylen, 44);
2146     __ jccb(Assembler::equal, L_doLast);
2147 
2148     __ aesenc(xmm_result, xmm_temp1);
2149     __ aesenc(xmm_result, xmm_temp2);
2150 
2151     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2152     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2153 
2154     __ cmpl(keylen, 52);
2155     __ jccb(Assembler::equal, L_doLast);
2156 
2157     __ aesenc(xmm_result, xmm_temp1);
2158     __ aesenc(xmm_result, xmm_temp2);
2159 
2160     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2161     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2162 
2163     __ BIND(L_doLast);
2164     __ aesenc(xmm_result, xmm_temp1);
2165     __ aesenclast(xmm_result, xmm_temp2);
2166     __ movdqu(Address(to, 0), xmm_result);        // store the result
2167     __ xorptr(rax, rax); // return 0
2168     __ leave(); // required for proper stackwalking of RuntimeStub frame
2169     __ ret(0);
2170 
2171     return start;
2172   }
2173 
2174 
2175   // Arguments:
2176   //
2177   // Inputs:
2178   //   c_rarg0   - source byte array address
2179   //   c_rarg1   - destination byte array address
2180   //   c_rarg2   - K (key) in little endian int array
2181   //
2182   address generate_aescrypt_decryptBlock() {
2183     assert(UseAES, "need AES instructions and misaligned SSE support");
2184     __ align(CodeEntryAlignment);
2185     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
2186     Label L_doLast;
2187     address start = __ pc();
2188 
2189     const Register from        = rdx;      // source array address
2190     const Register to          = rdx;      // destination array address
2191     const Register key         = rcx;      // key array address
2192     const Register keylen      = rax;
2193     const Address  from_param(rbp, 8+0);
2194     const Address  to_param  (rbp, 8+4);
2195     const Address  key_param (rbp, 8+8);
2196 
2197     const XMMRegister xmm_result = xmm0;
2198     const XMMRegister xmm_key_shuf_mask = xmm1;
2199     const XMMRegister xmm_temp1  = xmm2;
2200     const XMMRegister xmm_temp2  = xmm3;
2201     const XMMRegister xmm_temp3  = xmm4;
2202     const XMMRegister xmm_temp4  = xmm5;
2203 
2204     __ enter(); // required for proper stackwalking of RuntimeStub frame
2205 
2206     __ movptr(from, from_param);
2207     __ movptr(key, key_param);
2208 
2209     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2210     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2211 
2212     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2213     __ movdqu(xmm_result, Address(from, 0));
2214     __ movptr(to, to_param);
2215 
2216     // for decryption java expanded key ordering is rotated one position from what we want
2217     // so we start from 0x10 here and hit 0x00 last
2218     // we don't know if the key is aligned, hence not using load-execute form
2219     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2220     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2221     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2222     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2223 
2224     __ pxor  (xmm_result, xmm_temp1);
2225     __ aesdec(xmm_result, xmm_temp2);
2226     __ aesdec(xmm_result, xmm_temp3);
2227     __ aesdec(xmm_result, xmm_temp4);
2228 
2229     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2230     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2231     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2232     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2233 
2234     __ aesdec(xmm_result, xmm_temp1);
2235     __ aesdec(xmm_result, xmm_temp2);
2236     __ aesdec(xmm_result, xmm_temp3);
2237     __ aesdec(xmm_result, xmm_temp4);
2238 
2239     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2240     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2241     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
2242 
2243     __ cmpl(keylen, 44);
2244     __ jccb(Assembler::equal, L_doLast);
2245 
2246     __ aesdec(xmm_result, xmm_temp1);
2247     __ aesdec(xmm_result, xmm_temp2);
2248 
2249     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2250     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2251 
2252     __ cmpl(keylen, 52);
2253     __ jccb(Assembler::equal, L_doLast);
2254 
2255     __ aesdec(xmm_result, xmm_temp1);
2256     __ aesdec(xmm_result, xmm_temp2);
2257 
2258     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2259     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2260 
2261     __ BIND(L_doLast);
2262     __ aesdec(xmm_result, xmm_temp1);
2263     __ aesdec(xmm_result, xmm_temp2);
2264 
2265     // for decryption the aesdeclast operation is always on key+0x00
2266     __ aesdeclast(xmm_result, xmm_temp3);
2267     __ movdqu(Address(to, 0), xmm_result);  // store the result
2268     __ xorptr(rax, rax); // return 0
2269     __ leave(); // required for proper stackwalking of RuntimeStub frame
2270     __ ret(0);
2271 
2272     return start;
2273   }
2274 
2275   void handleSOERegisters(bool saving) {
2276     const int saveFrameSizeInBytes = 4 * wordSize;
2277     const Address saved_rbx     (rbp, -3 * wordSize);
2278     const Address saved_rsi     (rbp, -2 * wordSize);
2279     const Address saved_rdi     (rbp, -1 * wordSize);
2280 
2281     if (saving) {
2282       __ subptr(rsp, saveFrameSizeInBytes);
2283       __ movptr(saved_rsi, rsi);
2284       __ movptr(saved_rdi, rdi);
2285       __ movptr(saved_rbx, rbx);
2286     } else {
2287       // restoring
2288       __ movptr(rsi, saved_rsi);
2289       __ movptr(rdi, saved_rdi);
2290       __ movptr(rbx, saved_rbx);
2291     }
2292   }
2293 
2294   // Arguments:
2295   //
2296   // Inputs:
2297   //   c_rarg0   - source byte array address
2298   //   c_rarg1   - destination byte array address
2299   //   c_rarg2   - K (key) in little endian int array
2300   //   c_rarg3   - r vector byte array address
2301   //   c_rarg4   - input length
2302   //
2303   // Output:
2304   //   rax       - input length
2305   //
2306   address generate_cipherBlockChaining_encryptAESCrypt() {
2307     assert(UseAES, "need AES instructions and misaligned SSE support");
2308     __ align(CodeEntryAlignment);
2309     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
2310     address start = __ pc();
2311 
2312     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
2313     const Register from        = rsi;      // source array address
2314     const Register to          = rdx;      // destination array address
2315     const Register key         = rcx;      // key array address
2316     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2317                                            // and left with the results of the last encryption block
2318     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2319     const Register pos         = rax;
2320 
2321     // xmm register assignments for the loops below
2322     const XMMRegister xmm_result = xmm0;
2323     const XMMRegister xmm_temp   = xmm1;
2324     // first 6 keys preloaded into xmm2-xmm7
2325     const int XMM_REG_NUM_KEY_FIRST = 2;
2326     const int XMM_REG_NUM_KEY_LAST  = 7;
2327     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
2328 
2329     __ enter(); // required for proper stackwalking of RuntimeStub frame
2330     handleSOERegisters(true /*saving*/);
2331 
2332     // load registers from incoming parameters
2333     const Address  from_param(rbp, 8+0);
2334     const Address  to_param  (rbp, 8+4);
2335     const Address  key_param (rbp, 8+8);
2336     const Address  rvec_param (rbp, 8+12);
2337     const Address  len_param  (rbp, 8+16);
2338     __ movptr(from , from_param);
2339     __ movptr(to   , to_param);
2340     __ movptr(key  , key_param);
2341     __ movptr(rvec , rvec_param);
2342     __ movptr(len_reg , len_param);
2343 
2344     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
2345     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2346     // load up xmm regs 2 thru 7 with keys 0-5
2347     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2348       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
2349       offset += 0x10;
2350     }
2351 
2352     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
2353 
2354     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2355     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2356     __ cmpl(rax, 44);
2357     __ jcc(Assembler::notEqual, L_key_192_256);
2358 
2359     // 128 bit code follows here
2360     __ movl(pos, 0);
2361     __ align(OptoLoopAlignment);
2362     __ BIND(L_loopTop_128);
2363     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2364     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2365 
2366     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2367     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2368       __ aesenc(xmm_result, as_XMMRegister(rnum));
2369     }
2370     for (int key_offset = 0x60; key_offset <= 0x90; key_offset += 0x10) {
2371       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2372     }
2373     load_key(xmm_temp, key, 0xa0);
2374     __ aesenclast(xmm_result, xmm_temp);
2375 
2376     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
2377     // no need to store r to memory until we exit
2378     __ addptr(pos, AESBlockSize);
2379     __ subptr(len_reg, AESBlockSize);
2380     __ jcc(Assembler::notEqual, L_loopTop_128);
2381 
2382     __ BIND(L_exit);
2383     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
2384 
2385     handleSOERegisters(false /*restoring*/);
2386     __ movptr(rax, len_param); // return length
2387     __ leave();                                  // required for proper stackwalking of RuntimeStub frame
2388     __ ret(0);
2389 
2390     __ BIND(L_key_192_256);
2391     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
2392     __ cmpl(rax, 52);
2393     __ jcc(Assembler::notEqual, L_key_256);
2394 
2395     // 192-bit code follows here (could be changed to use more xmm registers)
2396     __ movl(pos, 0);
2397     __ align(OptoLoopAlignment);
2398     __ BIND(L_loopTop_192);
2399     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2400     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2401 
2402     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2403     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2404       __ aesenc(xmm_result, as_XMMRegister(rnum));
2405     }
2406     for (int key_offset = 0x60; key_offset <= 0xb0; key_offset += 0x10) {
2407       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2408     }
2409     load_key(xmm_temp, key, 0xc0);
2410     __ aesenclast(xmm_result, xmm_temp);
2411 
2412     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2413     // no need to store r to memory until we exit
2414     __ addptr(pos, AESBlockSize);
2415     __ subptr(len_reg, AESBlockSize);
2416     __ jcc(Assembler::notEqual, L_loopTop_192);
2417     __ jmp(L_exit);
2418 
2419     __ BIND(L_key_256);
2420     // 256-bit code follows here (could be changed to use more xmm registers)
2421     __ movl(pos, 0);
2422     __ align(OptoLoopAlignment);
2423     __ BIND(L_loopTop_256);
2424     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2425     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2426 
2427     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2428     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2429       __ aesenc(xmm_result, as_XMMRegister(rnum));
2430     }
2431     for (int key_offset = 0x60; key_offset <= 0xd0; key_offset += 0x10) {
2432       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2433     }
2434     load_key(xmm_temp, key, 0xe0);
2435     __ aesenclast(xmm_result, xmm_temp);
2436 
2437     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2438     // no need to store r to memory until we exit
2439     __ addptr(pos, AESBlockSize);
2440     __ subptr(len_reg, AESBlockSize);
2441     __ jcc(Assembler::notEqual, L_loopTop_256);
2442     __ jmp(L_exit);
2443 
2444     return start;
2445   }
2446 
2447 
2448   // CBC AES Decryption.
2449   // In 32-bit stub, because of lack of registers we do not try to parallelize 4 blocks at a time.
2450   //
2451   // Arguments:
2452   //
2453   // Inputs:
2454   //   c_rarg0   - source byte array address
2455   //   c_rarg1   - destination byte array address
2456   //   c_rarg2   - K (key) in little endian int array
2457   //   c_rarg3   - r vector byte array address
2458   //   c_rarg4   - input length
2459   //
2460   // Output:
2461   //   rax       - input length
2462   //
2463 
2464   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
2465     assert(UseAES, "need AES instructions and misaligned SSE support");
2466     __ align(CodeEntryAlignment);
2467     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
2468     address start = __ pc();
2469 
2470     const Register from        = rsi;      // source array address
2471     const Register to          = rdx;      // destination array address
2472     const Register key         = rcx;      // key array address
2473     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2474                                            // and left with the results of the last encryption block
2475     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2476     const Register pos         = rax;
2477 
2478     const int PARALLEL_FACTOR = 4;
2479     const int ROUNDS[3] = { 10, 12, 14 }; //aes rounds for key128, key192, key256
2480 
2481     Label L_exit;
2482     Label L_singleBlock_loopTop[3]; //128, 192, 256
2483     Label L_multiBlock_loopTop[3]; //128, 192, 256
2484 
2485     const XMMRegister xmm_prev_block_cipher = xmm0; // holds cipher of previous block
2486     const XMMRegister xmm_key_shuf_mask = xmm1;
2487 
2488     const XMMRegister xmm_key_tmp0 = xmm2;
2489     const XMMRegister xmm_key_tmp1 = xmm3;
2490 
2491     // registers holding the six results in the parallelized loop
2492     const XMMRegister xmm_result0 = xmm4;
2493     const XMMRegister xmm_result1 = xmm5;
2494     const XMMRegister xmm_result2 = xmm6;
2495     const XMMRegister xmm_result3 = xmm7;
2496 
2497     __ enter(); // required for proper stackwalking of RuntimeStub frame
2498     handleSOERegisters(true /*saving*/);
2499 
2500     // load registers from incoming parameters
2501     const Address  from_param(rbp, 8+0);
2502     const Address  to_param  (rbp, 8+4);
2503     const Address  key_param (rbp, 8+8);
2504     const Address  rvec_param (rbp, 8+12);
2505     const Address  len_param  (rbp, 8+16);
2506 
2507     __ movptr(from , from_param);
2508     __ movptr(to   , to_param);
2509     __ movptr(key  , key_param);
2510     __ movptr(rvec , rvec_param);
2511     __ movptr(len_reg , len_param);
2512 
2513     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2514     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // initialize with initial rvec
2515 
2516     __ xorptr(pos, pos);
2517 
2518     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2519     // rvec is reused
2520     __ movl(rvec, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2521     __ cmpl(rvec, 52);
2522     __ jcc(Assembler::equal, L_multiBlock_loopTop[1]);
2523     __ cmpl(rvec, 60);
2524     __ jcc(Assembler::equal, L_multiBlock_loopTop[2]);
2525 
2526 #define DoFour(opc, src_reg)           \
2527   __ opc(xmm_result0, src_reg);         \
2528   __ opc(xmm_result1, src_reg);         \
2529   __ opc(xmm_result2, src_reg);         \
2530   __ opc(xmm_result3, src_reg);         \
2531 
2532     for (int k = 0; k < 3; ++k) {
2533       __ align(OptoLoopAlignment);
2534       __ BIND(L_multiBlock_loopTop[k]);
2535       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
2536       __ jcc(Assembler::less, L_singleBlock_loopTop[k]);
2537 
2538       __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers
2539       __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2540       __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2541       __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2542 
2543       // the java expanded key ordering is rotated one position from what we want
2544       // so we start from 0x10 here and hit 0x00 last
2545       load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2546       DoFour(pxor, xmm_key_tmp0); //xor with first key
2547       // do the aes dec rounds
2548       for (int rnum = 1; rnum <= ROUNDS[k];) {
2549         //load two keys at a time
2550         //k1->0x20, ..., k9->0xa0, k10->0x00
2551         load_key(xmm_key_tmp1, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2552         load_key(xmm_key_tmp0, key, ((rnum + 2) % (ROUNDS[k] + 1)) * 0x10, xmm_key_shuf_mask); // hit 0x00 last!
2553         DoFour(aesdec, xmm_key_tmp1);
2554         rnum++;
2555         if (rnum != ROUNDS[k]) {
2556           DoFour(aesdec, xmm_key_tmp0);
2557         }
2558         else {
2559           DoFour(aesdeclast, xmm_key_tmp0);
2560         }
2561         rnum++;
2562       }
2563 
2564       // for each result, xor with the r vector of previous cipher block
2565       __ pxor(xmm_result0, xmm_prev_block_cipher);
2566       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2567       __ pxor(xmm_result1, xmm_prev_block_cipher);
2568       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2569       __ pxor(xmm_result2, xmm_prev_block_cipher);
2570       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2571       __ pxor(xmm_result3, xmm_prev_block_cipher);
2572       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize)); // this will carry over to next set of blocks
2573 
2574             // store 4 results into the next 64 bytes of output
2575        __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2576        __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2577        __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2578        __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2579 
2580        __ addptr(pos, 4 * AESBlockSize);
2581        __ subptr(len_reg, 4 * AESBlockSize);
2582        __ jmp(L_multiBlock_loopTop[k]);
2583 
2584        //singleBlock starts here
2585        __ align(OptoLoopAlignment);
2586        __ BIND(L_singleBlock_loopTop[k]);
2587        __ cmpptr(len_reg, 0); // any blocks left?
2588        __ jcc(Assembler::equal, L_exit);
2589        __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
2590        __ movdqa(xmm_result1, xmm_result0);
2591 
2592        load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2593        __ pxor(xmm_result0, xmm_key_tmp0);
2594        // do the aes dec rounds
2595        for (int rnum = 1; rnum < ROUNDS[k]; rnum++) {
2596          // the java expanded key ordering is rotated one position from what we want
2597          load_key(xmm_key_tmp0, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2598          __ aesdec(xmm_result0, xmm_key_tmp0);
2599        }
2600        load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
2601        __ aesdeclast(xmm_result0, xmm_key_tmp0);
2602        __ pxor(xmm_result0, xmm_prev_block_cipher); // xor with the current r vector
2603        __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result0); // store into the next 16 bytes of output
2604        // no need to store r to memory until we exit
2605        __ movdqa(xmm_prev_block_cipher, xmm_result1); // set up next r vector with cipher input from this block
2606 
2607        __ addptr(pos, AESBlockSize);
2608        __ subptr(len_reg, AESBlockSize);
2609        __ jmp(L_singleBlock_loopTop[k]);
2610     }//for 128/192/256
2611 
2612     __ BIND(L_exit);
2613     __ movptr(rvec, rvec_param);                        // restore this since reused earlier
2614     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher); // final value of r stored in rvec of CipherBlockChaining object
2615     handleSOERegisters(false /*restoring*/);
2616     __ movptr(rax, len_param);                          // return length
2617     __ leave();                                         // required for proper stackwalking of RuntimeStub frame
2618     __ ret(0);
2619 
2620     return start;
2621   }
2622 
2623   // CTR AES crypt.
2624   // In 32-bit stub, parallelize 4 blocks at a time
2625   // Arguments:
2626   //
2627   // Inputs:
2628   //   c_rarg0   - source byte array address
2629   //   c_rarg1   - destination byte array address
2630   //   c_rarg2   - K (key) in little endian int array
2631   //   c_rarg3   - counter vector byte array address
2632   //   c_rarg4   - input length
2633   //
2634   // Output:
2635   //   rax       - input length
2636   //
2637   address generate_counterMode_AESCrypt_Parallel() {
2638     assert(UseAES, "need AES instructions and misaligned SSE support");
2639     __ align(CodeEntryAlignment);
2640     StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
2641     address start = __ pc();
2642     const Register from        = rsi;      // source array address
2643     const Register to          = rdx;      // destination array address
2644     const Register key         = rcx;      // key array address
2645     const Register counter     = rdi;      // counter byte array initialized from initvector array address
2646                                            // and updated with the incremented counter in the end
2647     const Register len_reg     = rbx;
2648     const Register pos         = rax;
2649 
2650     __ enter(); // required for proper stackwalking of RuntimeStub frame
2651     handleSOERegisters(true /*saving*/); // save rbx, rsi, rdi
2652 
2653     // load registers from incoming parameters
2654     const Address  from_param(rbp, 8+0);
2655     const Address  to_param  (rbp, 8+4);
2656     const Address  key_param (rbp, 8+8);
2657     const Address  rvec_param (rbp, 8+12);
2658     const Address  len_param  (rbp, 8+16);
2659     const Address  saved_counter_param(rbp, 8 + 20);
2660     const Address  used_addr_param(rbp, 8 + 24);
2661 
2662     __ movptr(from , from_param);
2663     __ movptr(to   , to_param);
2664     __ movptr(len_reg , len_param);
2665 
2666     // Use the partially used encrpyted counter from last invocation
2667     Label L_exit_preLoop, L_preLoop_start;
2668 
2669     // Use the registers 'counter' and 'key' here in this preloop
2670     // to hold of last 2 params 'used' and 'saved_encCounter_start'
2671     Register used = counter;
2672     Register saved_encCounter_start = key;
2673     Register used_addr = saved_encCounter_start;
2674 
2675     __ movptr(used_addr, used_addr_param);
2676     __ movptr(used, Address(used_addr, 0));
2677     __ movptr(saved_encCounter_start, saved_counter_param);
2678 
2679     __ BIND(L_preLoop_start);
2680     __ cmpptr(used, 16);
2681     __ jcc(Assembler::aboveEqual, L_exit_preLoop);
2682     __ cmpptr(len_reg, 0);
2683     __ jcc(Assembler::lessEqual, L_exit_preLoop);
2684     __ movb(rax, Address(saved_encCounter_start, used));
2685     __ xorb(rax, Address(from, 0));
2686     __ movb(Address(to, 0), rax);
2687     __ addptr(from, 1);
2688     __ addptr(to, 1);
2689     __ addptr(used, 1);
2690     __ subptr(len_reg, 1);
2691 
2692     __ jmp(L_preLoop_start);
2693 
2694     __ BIND(L_exit_preLoop);
2695     __ movptr(used_addr, used_addr_param);
2696     __ movptr(used_addr, used_addr_param);
2697     __ movl(Address(used_addr, 0), used);
2698 
2699     // load the parameters 'key' and 'counter'
2700     __ movptr(key, key_param);
2701     __ movptr(counter, rvec_param);
2702 
2703     // xmm register assignments for the loops below
2704     const XMMRegister xmm_curr_counter      = xmm0;
2705     const XMMRegister xmm_counter_shuf_mask = xmm1;  // need to be reloaded
2706     const XMMRegister xmm_key_shuf_mask     = xmm2;  // need to be reloaded
2707     const XMMRegister xmm_key               = xmm3;
2708     const XMMRegister xmm_result0           = xmm4;
2709     const XMMRegister xmm_result1           = xmm5;
2710     const XMMRegister xmm_result2           = xmm6;
2711     const XMMRegister xmm_result3           = xmm7;
2712     const XMMRegister xmm_from0             = xmm1;   //reuse XMM register
2713     const XMMRegister xmm_from1             = xmm2;
2714     const XMMRegister xmm_from2             = xmm3;
2715     const XMMRegister xmm_from3             = xmm4;
2716 
2717     //for key_128, key_192, key_256
2718     const int rounds[3] = {10, 12, 14};
2719     Label L_singleBlockLoopTop[3];
2720     Label L_multiBlock_loopTop[3];
2721     Label L_key192_top, L_key256_top;
2722     Label L_incCounter[3][4]; // 3: different key length,  4: 4 blocks at a time
2723     Label L_incCounter_single[3]; //for single block, key128, key192, key256
2724     Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3];
2725     Label L_processTail_extr[3], L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3];
2726 
2727     Label L_exit;
2728     const int PARALLEL_FACTOR = 4;  //because of the limited register number
2729 
2730     // initialize counter with initial counter
2731     __ movdqu(xmm_curr_counter, Address(counter, 0x00));
2732     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2733     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled for increase
2734 
2735     // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
2736     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2737     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2738     __ cmpl(rax, 52);
2739     __ jcc(Assembler::equal, L_key192_top);
2740     __ cmpl(rax, 60);
2741     __ jcc(Assembler::equal, L_key256_top);
2742 
2743     //key128 begins here
2744     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2745 
2746 #define CTR_DoFour(opc, src_reg)               \
2747     __ opc(xmm_result0, src_reg);              \
2748     __ opc(xmm_result1, src_reg);              \
2749     __ opc(xmm_result2, src_reg);              \
2750     __ opc(xmm_result3, src_reg);
2751 
2752     // k == 0 :  generate code for key_128
2753     // k == 1 :  generate code for key_192
2754     // k == 2 :  generate code for key_256
2755     for (int k = 0; k < 3; ++k) {
2756       //multi blocks starts here
2757       __ align(OptoLoopAlignment);
2758       __ BIND(L_multiBlock_loopTop[k]);
2759       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left
2760       __ jcc(Assembler::less, L_singleBlockLoopTop[k]);
2761 
2762       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2763       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2764 
2765       //load, then increase counters
2766       CTR_DoFour(movdqa, xmm_curr_counter);
2767       __ push(rbx);
2768       inc_counter(rbx, xmm_result1, 0x01, L_incCounter[k][0]);
2769       inc_counter(rbx, xmm_result2, 0x02, L_incCounter[k][1]);
2770       inc_counter(rbx, xmm_result3, 0x03, L_incCounter[k][2]);
2771       inc_counter(rbx, xmm_curr_counter, 0x04, L_incCounter[k][3]);
2772       __ pop (rbx);
2773 
2774       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask); // load Round 0 key. interleaving for better performance
2775 
2776       CTR_DoFour(pshufb, xmm_counter_shuf_mask); // after increased, shuffled counters back for PXOR
2777       CTR_DoFour(pxor, xmm_key);   //PXOR with Round 0 key
2778 
2779       for (int i = 1; i < rounds[k]; ++i) {
2780         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2781         CTR_DoFour(aesenc, xmm_key);
2782       }
2783       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2784       CTR_DoFour(aesenclast, xmm_key);
2785 
2786       // get next PARALLEL_FACTOR blocks into xmm_from registers
2787       __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2788       __ movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2789       __ movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2790 
2791       // PXOR with input text
2792       __ pxor(xmm_result0, xmm_from0); //result0 is xmm4
2793       __ pxor(xmm_result1, xmm_from1);
2794       __ pxor(xmm_result2, xmm_from2);
2795 
2796       // store PARALLEL_FACTOR results into the next 64 bytes of output
2797       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2798       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2799       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2800 
2801       // do it here after xmm_result0 is saved, because xmm_from3 reuse the same register of xmm_result0.
2802       __ movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2803       __ pxor(xmm_result3, xmm_from3);
2804       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2805 
2806       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text
2807       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length
2808       __ jmp(L_multiBlock_loopTop[k]);
2809 
2810       // singleBlock starts here
2811       __ align(OptoLoopAlignment);
2812       __ BIND(L_singleBlockLoopTop[k]);
2813       __ cmpptr(len_reg, 0);
2814       __ jcc(Assembler::equal, L_exit);
2815       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2816       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2817       __ movdqa(xmm_result0, xmm_curr_counter);
2818       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask);
2819       __ push(rbx);//rbx is used for increasing counter
2820       inc_counter(rbx, xmm_curr_counter, 0x01, L_incCounter_single[k]);
2821       __ pop (rbx);
2822       __ pshufb(xmm_result0, xmm_counter_shuf_mask);
2823       __ pxor(xmm_result0, xmm_key);
2824       for (int i = 1; i < rounds[k]; i++) {
2825         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2826         __ aesenc(xmm_result0, xmm_key);
2827       }
2828       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2829       __ aesenclast(xmm_result0, xmm_key);
2830       __ cmpptr(len_reg, AESBlockSize);
2831       __ jcc(Assembler::less, L_processTail_insr[k]);
2832         __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2833         __ pxor(xmm_result0, xmm_from0);
2834         __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2835         __ addptr(pos, AESBlockSize);
2836         __ subptr(len_reg, AESBlockSize);
2837         __ jmp(L_singleBlockLoopTop[k]);
2838 
2839       __ BIND(L_processTail_insr[k]);                                               // Process the tail part of the input array
2840         __ addptr(pos, len_reg);                                                    // 1. Insert bytes from src array into xmm_from0 register
2841         __ testptr(len_reg, 8);
2842         __ jcc(Assembler::zero, L_processTail_4_insr[k]);
2843           __ subptr(pos,8);
2844           __ pinsrd(xmm_from0, Address(from, pos), 0);
2845           __ pinsrd(xmm_from0, Address(from, pos, Address::times_1, 4), 1);
2846         __ BIND(L_processTail_4_insr[k]);
2847         __ testptr(len_reg, 4);
2848         __ jcc(Assembler::zero, L_processTail_2_insr[k]);
2849           __ subptr(pos,4);
2850           __ pslldq(xmm_from0, 4);
2851           __ pinsrd(xmm_from0, Address(from, pos), 0);
2852         __ BIND(L_processTail_2_insr[k]);
2853         __ testptr(len_reg, 2);
2854         __ jcc(Assembler::zero, L_processTail_1_insr[k]);
2855           __ subptr(pos, 2);
2856           __ pslldq(xmm_from0, 2);
2857           __ pinsrw(xmm_from0, Address(from, pos), 0);
2858         __ BIND(L_processTail_1_insr[k]);
2859         __ testptr(len_reg, 1);
2860         __ jcc(Assembler::zero, L_processTail_exit_insr[k]);
2861           __ subptr(pos, 1);
2862           __ pslldq(xmm_from0, 1);
2863           __ pinsrb(xmm_from0, Address(from, pos), 0);
2864         __ BIND(L_processTail_exit_insr[k]);
2865 
2866         __ movptr(saved_encCounter_start, saved_counter_param);
2867         __ movdqu(Address(saved_encCounter_start, 0), xmm_result0);               // 2. Perform pxor of the encrypted counter and plaintext Bytes.
2868         __ pxor(xmm_result0, xmm_from0);                                          //    Also the encrypted counter is saved for next invocation.
2869 
2870         __ testptr(len_reg, 8);
2871         __ jcc(Assembler::zero, L_processTail_4_extr[k]);                        // 3. Extract bytes from xmm_result0 into the dest. array
2872           __ pextrd(Address(to, pos), xmm_result0, 0);
2873           __ pextrd(Address(to, pos, Address::times_1, 4), xmm_result0, 1);
2874           __ psrldq(xmm_result0, 8);
2875           __ addptr(pos, 8);
2876         __ BIND(L_processTail_4_extr[k]);
2877         __ testptr(len_reg, 4);
2878         __ jcc(Assembler::zero, L_processTail_2_extr[k]);
2879           __ pextrd(Address(to, pos), xmm_result0, 0);
2880           __ psrldq(xmm_result0, 4);
2881           __ addptr(pos, 4);
2882         __ BIND(L_processTail_2_extr[k]);
2883         __ testptr(len_reg, 2);
2884         __ jcc(Assembler::zero, L_processTail_1_extr[k]);
2885           __ pextrb(Address(to, pos), xmm_result0, 0);
2886           __ pextrb(Address(to, pos, Address::times_1, 1), xmm_result0, 1);
2887           __ psrldq(xmm_result0, 2);
2888           __ addptr(pos, 2);
2889         __ BIND(L_processTail_1_extr[k]);
2890         __ testptr(len_reg, 1);
2891         __ jcc(Assembler::zero, L_processTail_exit_extr[k]);
2892           __ pextrb(Address(to, pos), xmm_result0, 0);
2893 
2894         __ BIND(L_processTail_exit_extr[k]);
2895         __ movptr(used_addr, used_addr_param);
2896         __ movl(Address(used_addr, 0), len_reg);
2897         __ jmp(L_exit);
2898     }
2899 
2900     __ BIND(L_exit);
2901     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2902     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back.
2903     __ movdqu(Address(counter, 0), xmm_curr_counter); //save counter back
2904     handleSOERegisters(false /*restoring*/);
2905     __ movptr(rax, len_param); // return length
2906     __ leave();                // required for proper stackwalking of RuntimeStub frame
2907     __ ret(0);
2908 
2909     __ BIND (L_key192_top);
2910     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2911     __ jmp(L_multiBlock_loopTop[1]); //key192
2912 
2913     __ BIND (L_key256_top);
2914     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2915     __ jmp(L_multiBlock_loopTop[2]); //key192
2916 
2917     return start;
2918   }
2919 
2920   address generate_upper_word_mask() {
2921     __ align(64);
2922     StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
2923     address start = __ pc();
2924     __ emit_data(0x00000000, relocInfo::none, 0);
2925     __ emit_data(0x00000000, relocInfo::none, 0);
2926     __ emit_data(0x00000000, relocInfo::none, 0);
2927     __ emit_data(0xFFFFFFFF, relocInfo::none, 0);
2928     return start;
2929   }
2930 
2931   address generate_shuffle_byte_flip_mask() {
2932     __ align(64);
2933     StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask");
2934     address start = __ pc();
2935     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
2936     __ emit_data(0x08090a0b, relocInfo::none, 0);
2937     __ emit_data(0x04050607, relocInfo::none, 0);
2938     __ emit_data(0x00010203, relocInfo::none, 0);
2939     return start;
2940   }
2941 
2942   // ofs and limit are use for multi-block byte array.
2943   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
2944   address generate_sha1_implCompress(bool multi_block, const char *name) {
2945     __ align(CodeEntryAlignment);
2946     StubCodeMark mark(this, "StubRoutines", name);
2947     address start = __ pc();
2948 
2949     Register buf   = rax;
2950     Register state = rdx;
2951     Register ofs   = rcx;
2952     Register limit = rdi;
2953 
2954     const Address  buf_param(rbp, 8 + 0);
2955     const Address  state_param(rbp, 8 + 4);
2956     const Address  ofs_param(rbp, 8 + 8);
2957     const Address  limit_param(rbp, 8 + 12);
2958 
2959     const XMMRegister abcd = xmm0;
2960     const XMMRegister e0 = xmm1;
2961     const XMMRegister e1 = xmm2;
2962     const XMMRegister msg0 = xmm3;
2963 
2964     const XMMRegister msg1 = xmm4;
2965     const XMMRegister msg2 = xmm5;
2966     const XMMRegister msg3 = xmm6;
2967     const XMMRegister shuf_mask = xmm7;
2968 
2969     __ enter();
2970     __ subptr(rsp, 8 * wordSize);
2971     if (multi_block) {
2972       __ push(limit);
2973     }
2974     __ movptr(buf, buf_param);
2975     __ movptr(state, state_param);
2976     if (multi_block) {
2977       __ movptr(ofs, ofs_param);
2978       __ movptr(limit, limit_param);
2979     }
2980 
2981     __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask,
2982       buf, state, ofs, limit, rsp, multi_block);
2983 
2984     if (multi_block) {
2985       __ pop(limit);
2986     }
2987     __ addptr(rsp, 8 * wordSize);
2988     __ leave();
2989     __ ret(0);
2990     return start;
2991   }
2992 
2993   address generate_pshuffle_byte_flip_mask() {
2994     __ align(64);
2995     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask");
2996     address start = __ pc();
2997     __ emit_data(0x00010203, relocInfo::none, 0);
2998     __ emit_data(0x04050607, relocInfo::none, 0);
2999     __ emit_data(0x08090a0b, relocInfo::none, 0);
3000     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3001     return start;
3002   }
3003 
3004   // ofs and limit are use for multi-block byte array.
3005   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3006  address generate_sha256_implCompress(bool multi_block, const char *name) {
3007     __ align(CodeEntryAlignment);
3008     StubCodeMark mark(this, "StubRoutines", name);
3009     address start = __ pc();
3010 
3011     Register buf = rbx;
3012     Register state = rsi;
3013     Register ofs = rdx;
3014     Register limit = rcx;
3015 
3016     const Address  buf_param(rbp, 8 + 0);
3017     const Address  state_param(rbp, 8 + 4);
3018     const Address  ofs_param(rbp, 8 + 8);
3019     const Address  limit_param(rbp, 8 + 12);
3020 
3021     const XMMRegister msg = xmm0;
3022     const XMMRegister state0 = xmm1;
3023     const XMMRegister state1 = xmm2;
3024     const XMMRegister msgtmp0 = xmm3;
3025 
3026     const XMMRegister msgtmp1 = xmm4;
3027     const XMMRegister msgtmp2 = xmm5;
3028     const XMMRegister msgtmp3 = xmm6;
3029     const XMMRegister msgtmp4 = xmm7;
3030 
3031     __ enter();
3032     __ subptr(rsp, 8 * wordSize);
3033     handleSOERegisters(true /*saving*/);
3034     __ movptr(buf, buf_param);
3035     __ movptr(state, state_param);
3036     if (multi_block) {
3037      __ movptr(ofs, ofs_param);
3038      __ movptr(limit, limit_param);
3039     }
3040 
3041     __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3042       buf, state, ofs, limit, rsp, multi_block);
3043 
3044     handleSOERegisters(false);
3045     __ addptr(rsp, 8 * wordSize);
3046     __ leave();
3047     __ ret(0);
3048     return start;
3049   }
3050 
3051   // byte swap x86 long
3052   address generate_ghash_long_swap_mask() {
3053     __ align(CodeEntryAlignment);
3054     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
3055     address start = __ pc();
3056     __ emit_data(0x0b0a0908, relocInfo::none, 0);
3057     __ emit_data(0x0f0e0d0c, relocInfo::none, 0);
3058     __ emit_data(0x03020100, relocInfo::none, 0);
3059     __ emit_data(0x07060504, relocInfo::none, 0);
3060 
3061   return start;
3062   }
3063 
3064   // byte swap x86 byte array
3065   address generate_ghash_byte_swap_mask() {
3066     __ align(CodeEntryAlignment);
3067     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
3068     address start = __ pc();
3069     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3070     __ emit_data(0x08090a0b, relocInfo::none, 0);
3071     __ emit_data(0x04050607, relocInfo::none, 0);
3072     __ emit_data(0x00010203, relocInfo::none, 0);
3073   return start;
3074   }
3075 
3076   /* Single and multi-block ghash operations */
3077   address generate_ghash_processBlocks() {
3078     assert(UseGHASHIntrinsics, "need GHASH intrinsics and CLMUL support");
3079     __ align(CodeEntryAlignment);
3080     Label L_ghash_loop, L_exit;
3081     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
3082     address start = __ pc();
3083 
3084     const Register state        = rdi;
3085     const Register subkeyH      = rsi;
3086     const Register data         = rdx;
3087     const Register blocks       = rcx;
3088 
3089     const Address  state_param(rbp, 8+0);
3090     const Address  subkeyH_param(rbp, 8+4);
3091     const Address  data_param(rbp, 8+8);
3092     const Address  blocks_param(rbp, 8+12);
3093 
3094     const XMMRegister xmm_temp0 = xmm0;
3095     const XMMRegister xmm_temp1 = xmm1;
3096     const XMMRegister xmm_temp2 = xmm2;
3097     const XMMRegister xmm_temp3 = xmm3;
3098     const XMMRegister xmm_temp4 = xmm4;
3099     const XMMRegister xmm_temp5 = xmm5;
3100     const XMMRegister xmm_temp6 = xmm6;
3101     const XMMRegister xmm_temp7 = xmm7;
3102 
3103     __ enter();
3104     handleSOERegisters(true);  // Save registers
3105 
3106     __ movptr(state, state_param);
3107     __ movptr(subkeyH, subkeyH_param);
3108     __ movptr(data, data_param);
3109     __ movptr(blocks, blocks_param);
3110 
3111     __ movdqu(xmm_temp0, Address(state, 0));
3112     __ pshufb(xmm_temp0, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3113 
3114     __ movdqu(xmm_temp1, Address(subkeyH, 0));
3115     __ pshufb(xmm_temp1, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3116 
3117     __ BIND(L_ghash_loop);
3118     __ movdqu(xmm_temp2, Address(data, 0));
3119     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
3120 
3121     __ pxor(xmm_temp0, xmm_temp2);
3122 
3123     //
3124     // Multiply with the hash key
3125     //
3126     __ movdqu(xmm_temp3, xmm_temp0);
3127     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
3128     __ movdqu(xmm_temp4, xmm_temp0);
3129     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
3130 
3131     __ movdqu(xmm_temp5, xmm_temp0);
3132     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
3133     __ movdqu(xmm_temp6, xmm_temp0);
3134     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
3135 
3136     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
3137 
3138     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
3139     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
3140     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
3141     __ pxor(xmm_temp3, xmm_temp5);
3142     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
3143                                         // of the carry-less multiplication of
3144                                         // xmm0 by xmm1.
3145 
3146     // We shift the result of the multiplication by one bit position
3147     // to the left to cope for the fact that the bits are reversed.
3148     __ movdqu(xmm_temp7, xmm_temp3);
3149     __ movdqu(xmm_temp4, xmm_temp6);
3150     __ pslld (xmm_temp3, 1);
3151     __ pslld(xmm_temp6, 1);
3152     __ psrld(xmm_temp7, 31);
3153     __ psrld(xmm_temp4, 31);
3154     __ movdqu(xmm_temp5, xmm_temp7);
3155     __ pslldq(xmm_temp4, 4);
3156     __ pslldq(xmm_temp7, 4);
3157     __ psrldq(xmm_temp5, 12);
3158     __ por(xmm_temp3, xmm_temp7);
3159     __ por(xmm_temp6, xmm_temp4);
3160     __ por(xmm_temp6, xmm_temp5);
3161 
3162     //
3163     // First phase of the reduction
3164     //
3165     // Move xmm3 into xmm4, xmm5, xmm7 in order to perform the shifts
3166     // independently.
3167     __ movdqu(xmm_temp7, xmm_temp3);
3168     __ movdqu(xmm_temp4, xmm_temp3);
3169     __ movdqu(xmm_temp5, xmm_temp3);
3170     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
3171     __ pslld(xmm_temp4, 30);    // packed right shift shifting << 30
3172     __ pslld(xmm_temp5, 25);    // packed right shift shifting << 25
3173     __ pxor(xmm_temp7, xmm_temp4);      // xor the shifted versions
3174     __ pxor(xmm_temp7, xmm_temp5);
3175     __ movdqu(xmm_temp4, xmm_temp7);
3176     __ pslldq(xmm_temp7, 12);
3177     __ psrldq(xmm_temp4, 4);
3178     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
3179 
3180     //
3181     // Second phase of the reduction
3182     //
3183     // Make 3 copies of xmm3 in xmm2, xmm5, xmm7 for doing these
3184     // shift operations.
3185     __ movdqu(xmm_temp2, xmm_temp3);
3186     __ movdqu(xmm_temp7, xmm_temp3);
3187     __ movdqu(xmm_temp5, xmm_temp3);
3188     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
3189     __ psrld(xmm_temp7, 2);     // packed left shifting >> 2
3190     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
3191     __ pxor(xmm_temp2, xmm_temp7);      // xor the shifted versions
3192     __ pxor(xmm_temp2, xmm_temp5);
3193     __ pxor(xmm_temp2, xmm_temp4);
3194     __ pxor(xmm_temp3, xmm_temp2);
3195     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
3196 
3197     __ decrement(blocks);
3198     __ jcc(Assembler::zero, L_exit);
3199     __ movdqu(xmm_temp0, xmm_temp6);
3200     __ addptr(data, 16);
3201     __ jmp(L_ghash_loop);
3202 
3203     __ BIND(L_exit);
3204        // Byte swap 16-byte result
3205     __ pshufb(xmm_temp6, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3206     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
3207 
3208     handleSOERegisters(false);  // restore registers
3209     __ leave();
3210     __ ret(0);
3211     return start;
3212   }
3213 
3214   /**
3215    *  Arguments:
3216    *
3217    * Inputs:
3218    *   rsp(4)   - int crc
3219    *   rsp(8)   - byte* buf
3220    *   rsp(12)  - int length
3221    *
3222    * Ouput:
3223    *       rax   - int crc result
3224    */
3225   address generate_updateBytesCRC32() {
3226     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
3227 
3228     __ align(CodeEntryAlignment);
3229     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
3230 
3231     address start = __ pc();
3232 
3233     const Register crc   = rdx;  // crc
3234     const Register buf   = rsi;  // source java byte array address
3235     const Register len   = rcx;  // length
3236     const Register table = rdi;  // crc_table address (reuse register)
3237     const Register tmp   = rbx;
3238     assert_different_registers(crc, buf, len, table, tmp, rax);
3239 
3240     BLOCK_COMMENT("Entry:");
3241     __ enter(); // required for proper stackwalking of RuntimeStub frame
3242     __ push(rsi);
3243     __ push(rdi);
3244     __ push(rbx);
3245 
3246     Address crc_arg(rbp, 8 + 0);
3247     Address buf_arg(rbp, 8 + 4);
3248     Address len_arg(rbp, 8 + 8);
3249 
3250     // Load up:
3251     __ movl(crc,   crc_arg);
3252     __ movptr(buf, buf_arg);
3253     __ movl(len,   len_arg);
3254 
3255     __ kernel_crc32(crc, buf, len, table, tmp);
3256 
3257     __ movl(rax, crc);
3258     __ pop(rbx);
3259     __ pop(rdi);
3260     __ pop(rsi);
3261     __ vzeroupper();
3262     __ leave(); // required for proper stackwalking of RuntimeStub frame
3263     __ ret(0);
3264 
3265     return start;
3266   }
3267 
3268   /**
3269   *  Arguments:
3270   *
3271   * Inputs:
3272   *   rsp(4)   - int crc
3273   *   rsp(8)   - byte* buf
3274   *   rsp(12)  - int length
3275   *   rsp(16)  - table_start - optional (present only when doing a library_calll,
3276   *              not used by x86 algorithm)
3277   *
3278   * Ouput:
3279   *       rax  - int crc result
3280   */
3281   address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
3282     assert(UseCRC32CIntrinsics, "need SSE4_2");
3283     __ align(CodeEntryAlignment);
3284     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
3285     address start = __ pc();
3286     const Register crc = rax;  // crc
3287     const Register buf = rcx;  // source java byte array address
3288     const Register len = rdx;  // length
3289     const Register d = rbx;
3290     const Register g = rsi;
3291     const Register h = rdi;
3292     const Register empty = 0; // will never be used, in order not
3293                               // to change a signature for crc32c_IPL_Alg2_Alt2
3294                               // between 64/32 I'm just keeping it here
3295     assert_different_registers(crc, buf, len, d, g, h);
3296 
3297     BLOCK_COMMENT("Entry:");
3298     __ enter(); // required for proper stackwalking of RuntimeStub frame
3299     Address crc_arg(rsp, 4 + 4 + 0); // ESP+4 +
3300                                      // we need to add additional 4 because __ enter
3301                                      // have just pushed ebp on a stack
3302     Address buf_arg(rsp, 4 + 4 + 4);
3303     Address len_arg(rsp, 4 + 4 + 8);
3304       // Load up:
3305       __ movl(crc, crc_arg);
3306       __ movl(buf, buf_arg);
3307       __ movl(len, len_arg);
3308       __ push(d);
3309       __ push(g);
3310       __ push(h);
3311       __ crc32c_ipl_alg2_alt2(crc, buf, len,
3312                               d, g, h,
3313                               empty, empty, empty,
3314                               xmm0, xmm1, xmm2,
3315                               is_pclmulqdq_supported);
3316       __ pop(h);
3317       __ pop(g);
3318       __ pop(d);
3319     __ vzeroupper();
3320     __ leave(); // required for proper stackwalking of RuntimeStub frame
3321     __ ret(0);
3322 
3323     return start;
3324   }
3325 
3326  address generate_libmExp() {
3327     StubCodeMark mark(this, "StubRoutines", "libmExp");
3328 
3329     address start = __ pc();
3330 
3331     const XMMRegister x0  = xmm0;
3332     const XMMRegister x1  = xmm1;
3333     const XMMRegister x2  = xmm2;
3334     const XMMRegister x3  = xmm3;
3335 
3336     const XMMRegister x4  = xmm4;
3337     const XMMRegister x5  = xmm5;
3338     const XMMRegister x6  = xmm6;
3339     const XMMRegister x7  = xmm7;
3340 
3341     const Register tmp   = rbx;
3342 
3343     BLOCK_COMMENT("Entry:");
3344     __ enter(); // required for proper stackwalking of RuntimeStub frame
3345     __ fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3346     __ leave(); // required for proper stackwalking of RuntimeStub frame
3347     __ ret(0);
3348 
3349     return start;
3350 
3351   }
3352 
3353  address generate_libmLog() {
3354    StubCodeMark mark(this, "StubRoutines", "libmLog");
3355 
3356    address start = __ pc();
3357 
3358    const XMMRegister x0 = xmm0;
3359    const XMMRegister x1 = xmm1;
3360    const XMMRegister x2 = xmm2;
3361    const XMMRegister x3 = xmm3;
3362 
3363    const XMMRegister x4 = xmm4;
3364    const XMMRegister x5 = xmm5;
3365    const XMMRegister x6 = xmm6;
3366    const XMMRegister x7 = xmm7;
3367 
3368    const Register tmp = rbx;
3369 
3370    BLOCK_COMMENT("Entry:");
3371    __ enter(); // required for proper stackwalking of RuntimeStub frame
3372    __ fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3373    __ leave(); // required for proper stackwalking of RuntimeStub frame
3374    __ ret(0);
3375 
3376    return start;
3377 
3378  }
3379 
3380  address generate_libmLog10() {
3381    StubCodeMark mark(this, "StubRoutines", "libmLog10");
3382 
3383    address start = __ pc();
3384 
3385    const XMMRegister x0 = xmm0;
3386    const XMMRegister x1 = xmm1;
3387    const XMMRegister x2 = xmm2;
3388    const XMMRegister x3 = xmm3;
3389 
3390    const XMMRegister x4 = xmm4;
3391    const XMMRegister x5 = xmm5;
3392    const XMMRegister x6 = xmm6;
3393    const XMMRegister x7 = xmm7;
3394 
3395    const Register tmp = rbx;
3396 
3397    BLOCK_COMMENT("Entry:");
3398    __ enter(); // required for proper stackwalking of RuntimeStub frame
3399    __ fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3400    __ leave(); // required for proper stackwalking of RuntimeStub frame
3401    __ ret(0);
3402 
3403    return start;
3404 
3405  }
3406 
3407  address generate_libmPow() {
3408    StubCodeMark mark(this, "StubRoutines", "libmPow");
3409 
3410    address start = __ pc();
3411 
3412    const XMMRegister x0 = xmm0;
3413    const XMMRegister x1 = xmm1;
3414    const XMMRegister x2 = xmm2;
3415    const XMMRegister x3 = xmm3;
3416 
3417    const XMMRegister x4 = xmm4;
3418    const XMMRegister x5 = xmm5;
3419    const XMMRegister x6 = xmm6;
3420    const XMMRegister x7 = xmm7;
3421 
3422    const Register tmp = rbx;
3423 
3424    BLOCK_COMMENT("Entry:");
3425    __ enter(); // required for proper stackwalking of RuntimeStub frame
3426    __ fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3427    __ leave(); // required for proper stackwalking of RuntimeStub frame
3428    __ ret(0);
3429 
3430    return start;
3431 
3432  }
3433 
3434  address generate_libm_reduce_pi04l() {
3435    StubCodeMark mark(this, "StubRoutines", "libm_reduce_pi04l");
3436 
3437    address start = __ pc();
3438 
3439    BLOCK_COMMENT("Entry:");
3440    __ libm_reduce_pi04l(rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3441 
3442    return start;
3443 
3444  }
3445 
3446  address generate_libm_sin_cos_huge() {
3447    StubCodeMark mark(this, "StubRoutines", "libm_sin_cos_huge");
3448 
3449    address start = __ pc();
3450 
3451    const XMMRegister x0 = xmm0;
3452    const XMMRegister x1 = xmm1;
3453 
3454    BLOCK_COMMENT("Entry:");
3455    __ libm_sincos_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3456 
3457    return start;
3458 
3459  }
3460 
3461  address generate_libmSin() {
3462    StubCodeMark mark(this, "StubRoutines", "libmSin");
3463 
3464    address start = __ pc();
3465 
3466    const XMMRegister x0 = xmm0;
3467    const XMMRegister x1 = xmm1;
3468    const XMMRegister x2 = xmm2;
3469    const XMMRegister x3 = xmm3;
3470 
3471    const XMMRegister x4 = xmm4;
3472    const XMMRegister x5 = xmm5;
3473    const XMMRegister x6 = xmm6;
3474    const XMMRegister x7 = xmm7;
3475 
3476    BLOCK_COMMENT("Entry:");
3477    __ enter(); // required for proper stackwalking of RuntimeStub frame
3478    __ fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rdx);
3479    __ leave(); // required for proper stackwalking of RuntimeStub frame
3480    __ ret(0);
3481 
3482    return start;
3483 
3484  }
3485 
3486  address generate_libmCos() {
3487    StubCodeMark mark(this, "StubRoutines", "libmCos");
3488 
3489    address start = __ pc();
3490 
3491    const XMMRegister x0 = xmm0;
3492    const XMMRegister x1 = xmm1;
3493    const XMMRegister x2 = xmm2;
3494    const XMMRegister x3 = xmm3;
3495 
3496    const XMMRegister x4 = xmm4;
3497    const XMMRegister x5 = xmm5;
3498    const XMMRegister x6 = xmm6;
3499    const XMMRegister x7 = xmm7;
3500 
3501    const Register tmp = rbx;
3502 
3503    BLOCK_COMMENT("Entry:");
3504    __ enter(); // required for proper stackwalking of RuntimeStub frame
3505    __ fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3506    __ leave(); // required for proper stackwalking of RuntimeStub frame
3507    __ ret(0);
3508 
3509    return start;
3510 
3511  }
3512 
3513  address generate_libm_tan_cot_huge() {
3514    StubCodeMark mark(this, "StubRoutines", "libm_tan_cot_huge");
3515 
3516    address start = __ pc();
3517 
3518    const XMMRegister x0 = xmm0;
3519    const XMMRegister x1 = xmm1;
3520 
3521    BLOCK_COMMENT("Entry:");
3522    __ libm_tancot_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3523 
3524    return start;
3525 
3526  }
3527 
3528  address generate_libmTan() {
3529    StubCodeMark mark(this, "StubRoutines", "libmTan");
3530 
3531    address start = __ pc();
3532 
3533    const XMMRegister x0 = xmm0;
3534    const XMMRegister x1 = xmm1;
3535    const XMMRegister x2 = xmm2;
3536    const XMMRegister x3 = xmm3;
3537 
3538    const XMMRegister x4 = xmm4;
3539    const XMMRegister x5 = xmm5;
3540    const XMMRegister x6 = xmm6;
3541    const XMMRegister x7 = xmm7;
3542 
3543    const Register tmp = rbx;
3544 
3545    BLOCK_COMMENT("Entry:");
3546    __ enter(); // required for proper stackwalking of RuntimeStub frame
3547    __ fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3548    __ leave(); // required for proper stackwalking of RuntimeStub frame
3549    __ ret(0);
3550 
3551    return start;
3552 
3553  }
3554 
3555   // Safefetch stubs.
3556   void generate_safefetch(const char* name, int size, address* entry,
3557                           address* fault_pc, address* continuation_pc) {
3558     // safefetch signatures:
3559     //   int      SafeFetch32(int*      adr, int      errValue);
3560     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3561 
3562     StubCodeMark mark(this, "StubRoutines", name);
3563 
3564     // Entry point, pc or function descriptor.
3565     *entry = __ pc();
3566 
3567     __ movl(rax, Address(rsp, 0x8));
3568     __ movl(rcx, Address(rsp, 0x4));
3569     // Load *adr into eax, may fault.
3570     *fault_pc = __ pc();
3571     switch (size) {
3572       case 4:
3573         // int32_t
3574         __ movl(rax, Address(rcx, 0));
3575         break;
3576       case 8:
3577         // int64_t
3578         Unimplemented();
3579         break;
3580       default:
3581         ShouldNotReachHere();
3582     }
3583 
3584     // Return errValue or *adr.
3585     *continuation_pc = __ pc();
3586     __ ret(0);
3587   }
3588 
3589  public:
3590   // Information about frame layout at time of blocking runtime call.
3591   // Note that we only have to preserve callee-saved registers since
3592   // the compilers are responsible for supplying a continuation point
3593   // if they expect all registers to be preserved.
3594   enum layout {
3595     thread_off,    // last_java_sp
3596     arg1_off,
3597     arg2_off,
3598     rbp_off,       // callee saved register
3599     ret_pc,
3600     framesize
3601   };
3602 
3603  private:
3604 
3605 #undef  __
3606 #define __ masm->
3607 
3608   //------------------------------------------------------------------------------------------------------------------------
3609   // Continuation point for throwing of implicit exceptions that are not handled in
3610   // the current activation. Fabricates an exception oop and initiates normal
3611   // exception dispatching in this frame.
3612   //
3613   // Previously the compiler (c2) allowed for callee save registers on Java calls.
3614   // This is no longer true after adapter frames were removed but could possibly
3615   // be brought back in the future if the interpreter code was reworked and it
3616   // was deemed worthwhile. The comment below was left to describe what must
3617   // happen here if callee saves were resurrected. As it stands now this stub
3618   // could actually be a vanilla BufferBlob and have now oopMap at all.
3619   // Since it doesn't make much difference we've chosen to leave it the
3620   // way it was in the callee save days and keep the comment.
3621 
3622   // If we need to preserve callee-saved values we need a callee-saved oop map and
3623   // therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
3624   // If the compiler needs all registers to be preserved between the fault
3625   // point and the exception handler then it must assume responsibility for that in
3626   // AbstractCompiler::continuation_for_implicit_null_exception or
3627   // continuation_for_implicit_division_by_zero_exception. All other implicit
3628   // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
3629   // either at call sites or otherwise assume that stack unwinding will be initiated,
3630   // so caller saved registers were assumed volatile in the compiler.
3631   address generate_throw_exception(const char* name, address runtime_entry,
3632                                    Register arg1 = noreg, Register arg2 = noreg) {
3633 
3634     int insts_size = 256;
3635     int locs_size  = 32;
3636 
3637     CodeBuffer code(name, insts_size, locs_size);
3638     OopMapSet* oop_maps  = new OopMapSet();
3639     MacroAssembler* masm = new MacroAssembler(&code);
3640 
3641     address start = __ pc();
3642 
3643     // This is an inlined and slightly modified version of call_VM
3644     // which has the ability to fetch the return PC out of
3645     // thread-local storage and also sets up last_Java_sp slightly
3646     // differently than the real call_VM
3647     Register java_thread = rbx;
3648     __ get_thread(java_thread);
3649 
3650     __ enter(); // required for proper stackwalking of RuntimeStub frame
3651 
3652     // pc and rbp, already pushed
3653     __ subptr(rsp, (framesize-2) * wordSize); // prolog
3654 
3655     // Frame is now completed as far as size and linkage.
3656 
3657     int frame_complete = __ pc() - start;
3658 
3659     // push java thread (becomes first argument of C function)
3660     __ movptr(Address(rsp, thread_off * wordSize), java_thread);
3661     if (arg1 != noreg) {
3662       __ movptr(Address(rsp, arg1_off * wordSize), arg1);
3663     }
3664     if (arg2 != noreg) {
3665       assert(arg1 != noreg, "missing reg arg");
3666       __ movptr(Address(rsp, arg2_off * wordSize), arg2);
3667     }
3668 
3669     // Set up last_Java_sp and last_Java_fp
3670     __ set_last_Java_frame(java_thread, rsp, rbp, NULL);
3671 
3672     // Call runtime
3673     BLOCK_COMMENT("call runtime_entry");
3674     __ call(RuntimeAddress(runtime_entry));
3675     // Generate oop map
3676     OopMap* map =  new OopMap(framesize, 0);
3677     oop_maps->add_gc_map(__ pc() - start, map);
3678 
3679     // restore the thread (cannot use the pushed argument since arguments
3680     // may be overwritten by C code generated by an optimizing compiler);
3681     // however can use the register value directly if it is callee saved.
3682     __ get_thread(java_thread);
3683 
3684     __ reset_last_Java_frame(java_thread, true);
3685 
3686     __ leave(); // required for proper stackwalking of RuntimeStub frame
3687 
3688     // check for pending exceptions
3689 #ifdef ASSERT
3690     Label L;
3691     __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3692     __ jcc(Assembler::notEqual, L);
3693     __ should_not_reach_here();
3694     __ bind(L);
3695 #endif /* ASSERT */
3696     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3697 
3698 
3699     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
3700     return stub->entry_point();
3701   }
3702 
3703 
3704   void create_control_words() {
3705     // Round to nearest, 53-bit mode, exceptions masked
3706     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
3707     // Round to zero, 53-bit mode, exception mased
3708     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
3709     // Round to nearest, 24-bit mode, exceptions masked
3710     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
3711     // Round to nearest, 64-bit mode, exceptions masked
3712     StubRoutines::_mxcsr_std           = 0x1F80;
3713     // Note: the following two constants are 80-bit values
3714     //       layout is critical for correct loading by FPU.
3715     // Bias for strict fp multiply/divide
3716     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
3717     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
3718     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
3719     // Un-Bias for strict fp multiply/divide
3720     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
3721     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
3722     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
3723   }
3724 
3725   //---------------------------------------------------------------------------
3726   // Initialization
3727 
3728   void generate_initial() {
3729     // Generates all stubs and initializes the entry points
3730 
3731     //------------------------------------------------------------------------------------------------------------------------
3732     // entry points that exist in all platforms
3733     // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
3734     //       the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
3735     StubRoutines::_forward_exception_entry      = generate_forward_exception();
3736 
3737     StubRoutines::_call_stub_entry              =
3738       generate_call_stub(StubRoutines::_call_stub_return_address);
3739     // is referenced by megamorphic call
3740     StubRoutines::_catch_exception_entry        = generate_catch_exception();
3741 
3742     // These are currently used by Solaris/Intel
3743     StubRoutines::_atomic_xchg_entry            = generate_atomic_xchg();
3744 
3745     // platform dependent
3746     create_control_words();
3747 
3748     StubRoutines::x86::_verify_mxcsr_entry                 = generate_verify_mxcsr();
3749     StubRoutines::x86::_verify_fpu_cntrl_wrd_entry         = generate_verify_fpu_cntrl_wrd();
3750     StubRoutines::_d2i_wrapper                              = generate_d2i_wrapper(T_INT,
3751                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
3752     StubRoutines::_d2l_wrapper                              = generate_d2i_wrapper(T_LONG,
3753                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
3754 
3755     // Build this early so it's available for the interpreter
3756     StubRoutines::_throw_StackOverflowError_entry          = generate_throw_exception("StackOverflowError throw_exception",
3757                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
3758     StubRoutines::_throw_delayed_StackOverflowError_entry  = generate_throw_exception("delayed StackOverflowError throw_exception",
3759                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_delayed_StackOverflowError));
3760 
3761     if (UseCRC32Intrinsics) {
3762       // set table address before stub generation which use it
3763       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
3764       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
3765     }
3766 
3767     if (UseCRC32CIntrinsics) {
3768       bool supports_clmul = VM_Version::supports_clmul();
3769       StubRoutines::x86::generate_CRC32C_table(supports_clmul);
3770       StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table;
3771       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
3772     }
3773     if (VM_Version::supports_sse2() && UseLibmIntrinsic && InlineIntrinsics) {
3774       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3775           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3776           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3777         StubRoutines::x86::_L_2il0floatpacket_0_adr = (address)StubRoutines::x86::_L_2il0floatpacket_0;
3778         StubRoutines::x86::_Pi4Inv_adr = (address)StubRoutines::x86::_Pi4Inv;
3779         StubRoutines::x86::_Pi4x3_adr = (address)StubRoutines::x86::_Pi4x3;
3780         StubRoutines::x86::_Pi4x4_adr = (address)StubRoutines::x86::_Pi4x4;
3781         StubRoutines::x86::_ones_adr = (address)StubRoutines::x86::_ones;
3782       }
3783       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) {
3784         StubRoutines::_dexp = generate_libmExp();
3785       }
3786       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
3787         StubRoutines::_dlog = generate_libmLog();
3788       }
3789       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) {
3790         StubRoutines::_dlog10 = generate_libmLog10();
3791       }
3792       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) {
3793         StubRoutines::_dpow = generate_libmPow();
3794       }
3795       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3796         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3797         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3798         StubRoutines::_dlibm_reduce_pi04l = generate_libm_reduce_pi04l();
3799       }
3800       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3801         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3802         StubRoutines::_dlibm_sin_cos_huge = generate_libm_sin_cos_huge();
3803       }
3804       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
3805         StubRoutines::_dsin = generate_libmSin();
3806       }
3807       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3808         StubRoutines::_dcos = generate_libmCos();
3809       }
3810       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3811         StubRoutines::_dlibm_tan_cot_huge = generate_libm_tan_cot_huge();
3812         StubRoutines::_dtan = generate_libmTan();
3813       }
3814     }
3815   }
3816 
3817   void generate_all() {
3818     // Generates all stubs and initializes the entry points
3819 
3820     // These entry points require SharedInfo::stack0 to be set up in non-core builds
3821     // and need to be relocatable, so they each fabricate a RuntimeStub internally.
3822     StubRoutines::_throw_AbstractMethodError_entry         = generate_throw_exception("AbstractMethodError throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
3823     StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
3824     StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
3825 
3826     //------------------------------------------------------------------------------------------------------------------------
3827     // entry points that are platform specific
3828 
3829     // support for verify_oop (must happen after universe_init)
3830     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
3831 
3832     // arraycopy stubs used by compilers
3833     generate_arraycopy_stubs();
3834 
3835     // don't bother generating these AES intrinsic stubs unless global flag is set
3836     if (UseAESIntrinsics) {
3837       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // might be needed by the others
3838 
3839       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
3840       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
3841       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
3842       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
3843     }
3844 
3845     if (UseAESCTRIntrinsics) {
3846       StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
3847       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
3848     }
3849 
3850     if (UseSHA1Intrinsics) {
3851       StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask();
3852       StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask();
3853       StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
3854       StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
3855     }
3856     if (UseSHA256Intrinsics) {
3857       StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
3858       StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
3859       StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
3860       StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
3861     }
3862 
3863     // Generate GHASH intrinsics code
3864     if (UseGHASHIntrinsics) {
3865       StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
3866       StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
3867       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
3868     }
3869 
3870     // Safefetch stubs.
3871     generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
3872                                                    &StubRoutines::_safefetch32_fault_pc,
3873                                                    &StubRoutines::_safefetch32_continuation_pc);
3874     StubRoutines::_safefetchN_entry           = StubRoutines::_safefetch32_entry;
3875     StubRoutines::_safefetchN_fault_pc        = StubRoutines::_safefetch32_fault_pc;
3876     StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc;
3877   }
3878 
3879 
3880  public:
3881   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
3882     if (all) {
3883       generate_all();
3884     } else {
3885       generate_initial();
3886     }
3887   }
3888 }; // end class declaration
3889 
3890 
3891 void StubGenerator_generate(CodeBuffer* code, bool all) {
3892   StubGenerator g(code, all);
3893 }