1 /*
   2  * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "compiler/oopMap.hpp"
  29 #include "gc/shared/barrierSet.hpp"
  30 #include "gc/shared/barrierSetAssembler.hpp"
  31 #include "gc/shared/barrierSetNMethod.hpp"
  32 #include "interpreter/interpreter.hpp"
  33 #include "memory/universe.hpp"
  34 #include "nativeInst_x86.hpp"
  35 #include "oops/instanceOop.hpp"
  36 #include "oops/method.hpp"
  37 #include "oops/objArrayKlass.hpp"
  38 #include "oops/oop.inline.hpp"
  39 #include "prims/methodHandles.hpp"
  40 #include "runtime/frame.inline.hpp"
  41 #include "runtime/handles.inline.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "runtime/stubCodeGenerator.hpp"
  44 #include "runtime/stubRoutines.hpp"
  45 #include "runtime/thread.inline.hpp"
  46 #ifdef COMPILER2
  47 #include "opto/runtime.hpp"
  48 #endif
  49 
  50 // Declaration and definition of StubGenerator (no .hpp file).
  51 // For a more detailed description of the stub routine structure
  52 // see the comment in stubRoutines.hpp
  53 
  54 #define __ _masm->
  55 #define a__ ((Assembler*)_masm)->
  56 
  57 #ifdef PRODUCT
  58 #define BLOCK_COMMENT(str) /* nothing */
  59 #else
  60 #define BLOCK_COMMENT(str) __ block_comment(str)
  61 #endif
  62 
  63 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  64 
  65 const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions
  66 const int FPU_CNTRL_WRD_MASK = 0xFFFF;
  67 
  68 // -------------------------------------------------------------------------------------------------------------------------
  69 // Stub Code definitions
  70 
  71 class StubGenerator: public StubCodeGenerator {
  72  private:
  73 
  74 #ifdef PRODUCT
  75 #define inc_counter_np(counter) ((void)0)
  76 #else
  77   void inc_counter_np_(int& counter) {
  78     __ incrementl(ExternalAddress((address)&counter));
  79   }
  80 #define inc_counter_np(counter) \
  81   BLOCK_COMMENT("inc_counter " #counter); \
  82   inc_counter_np_(counter);
  83 #endif //PRODUCT
  84 
  85   void inc_copy_counter_np(BasicType t) {
  86 #ifndef PRODUCT
  87     switch (t) {
  88     case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
  89     case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
  90     case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
  91     case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
  92     case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
  93     default:        ShouldNotReachHere();
  94     }
  95 #endif //PRODUCT
  96   }
  97 
  98   //------------------------------------------------------------------------------------------------------------------------
  99   // Call stubs are used to call Java from C
 100   //
 101   //    [ return_from_Java     ] <--- rsp
 102   //    [ argument word n      ]
 103   //      ...
 104   // -N [ argument word 1      ]
 105   // -7 [ Possible padding for stack alignment ]
 106   // -6 [ Possible padding for stack alignment ]
 107   // -5 [ Possible padding for stack alignment ]
 108   // -4 [ mxcsr save           ] <--- rsp_after_call
 109   // -3 [ saved rbx,            ]
 110   // -2 [ saved rsi            ]
 111   // -1 [ saved rdi            ]
 112   //  0 [ saved rbp,            ] <--- rbp,
 113   //  1 [ return address       ]
 114   //  2 [ ptr. to call wrapper ]
 115   //  3 [ result               ]
 116   //  4 [ result_type          ]
 117   //  5 [ method               ]
 118   //  6 [ entry_point          ]
 119   //  7 [ parameters           ]
 120   //  8 [ parameter_size       ]
 121   //  9 [ thread               ]
 122 
 123 
 124   address generate_call_stub(address& return_address) {
 125     StubCodeMark mark(this, "StubRoutines", "call_stub");
 126     address start = __ pc();
 127 
 128     // stub code parameters / addresses
 129     assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
 130     bool  sse_save = false;
 131     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
 132     const int     locals_count_in_bytes  (4*wordSize);
 133     const Address mxcsr_save    (rbp, -4 * wordSize);
 134     const Address saved_rbx     (rbp, -3 * wordSize);
 135     const Address saved_rsi     (rbp, -2 * wordSize);
 136     const Address saved_rdi     (rbp, -1 * wordSize);
 137     const Address result        (rbp,  3 * wordSize);
 138     const Address result_type   (rbp,  4 * wordSize);
 139     const Address method        (rbp,  5 * wordSize);
 140     const Address entry_point   (rbp,  6 * wordSize);
 141     const Address parameters    (rbp,  7 * wordSize);
 142     const Address parameter_size(rbp,  8 * wordSize);
 143     const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!
 144     sse_save =  UseSSE > 0;
 145 
 146     // stub code
 147     __ enter();
 148     __ movptr(rcx, parameter_size);              // parameter counter
 149     __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes
 150     __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves
 151     __ subptr(rsp, rcx);
 152     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
 153 
 154     // save rdi, rsi, & rbx, according to C calling conventions
 155     __ movptr(saved_rdi, rdi);
 156     __ movptr(saved_rsi, rsi);
 157     __ movptr(saved_rbx, rbx);
 158 
 159     // save and initialize %mxcsr
 160     if (sse_save) {
 161       Label skip_ldmx;
 162       __ stmxcsr(mxcsr_save);
 163       __ movl(rax, mxcsr_save);
 164       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 165       ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
 166       __ cmp32(rax, mxcsr_std);
 167       __ jcc(Assembler::equal, skip_ldmx);
 168       __ ldmxcsr(mxcsr_std);
 169       __ bind(skip_ldmx);
 170     }
 171 
 172     // make sure the control word is correct.
 173     __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std()));
 174 
 175 #ifdef ASSERT
 176     // make sure we have no pending exceptions
 177     { Label L;
 178       __ movptr(rcx, thread);
 179       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 180       __ jcc(Assembler::equal, L);
 181       __ stop("StubRoutines::call_stub: entered with pending exception");
 182       __ bind(L);
 183     }
 184 #endif
 185 
 186     // pass parameters if any
 187     BLOCK_COMMENT("pass parameters if any");
 188     Label parameters_done;
 189     __ movl(rcx, parameter_size);  // parameter counter
 190     __ testl(rcx, rcx);
 191     __ jcc(Assembler::zero, parameters_done);
 192 
 193     // parameter passing loop
 194 
 195     Label loop;
 196     // Copy Java parameters in reverse order (receiver last)
 197     // Note that the argument order is inverted in the process
 198     // source is rdx[rcx: N-1..0]
 199     // dest   is rsp[rbx: 0..N-1]
 200 
 201     __ movptr(rdx, parameters);          // parameter pointer
 202     __ xorptr(rbx, rbx);
 203 
 204     __ BIND(loop);
 205 
 206     // get parameter
 207     __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
 208     __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
 209                     Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter
 210     __ increment(rbx);
 211     __ decrement(rcx);
 212     __ jcc(Assembler::notZero, loop);
 213 
 214     // call Java function
 215     __ BIND(parameters_done);
 216     __ movptr(rbx, method);           // get Method*
 217     __ movptr(rax, entry_point);      // get entry_point
 218     __ mov(rsi, rsp);                 // set sender sp
 219     BLOCK_COMMENT("call Java function");
 220     __ call(rax);
 221 
 222     BLOCK_COMMENT("call_stub_return_address:");
 223     return_address = __ pc();
 224 
 225 #ifdef COMPILER2
 226     {
 227       Label L_skip;
 228       if (UseSSE >= 2) {
 229         __ verify_FPU(0, "call_stub_return");
 230       } else {
 231         for (int i = 1; i < 8; i++) {
 232           __ ffree(i);
 233         }
 234 
 235         // UseSSE <= 1 so double result should be left on TOS
 236         __ movl(rsi, result_type);
 237         __ cmpl(rsi, T_DOUBLE);
 238         __ jcc(Assembler::equal, L_skip);
 239         if (UseSSE == 0) {
 240           // UseSSE == 0 so float result should be left on TOS
 241           __ cmpl(rsi, T_FLOAT);
 242           __ jcc(Assembler::equal, L_skip);
 243         }
 244         __ ffree(0);
 245       }
 246       __ BIND(L_skip);
 247     }
 248 #endif // COMPILER2
 249 
 250     // store result depending on type
 251     // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 252     __ movptr(rdi, result);
 253     Label is_long, is_float, is_double, exit;
 254     __ movl(rsi, result_type);
 255     __ cmpl(rsi, T_LONG);
 256     __ jcc(Assembler::equal, is_long);
 257     __ cmpl(rsi, T_FLOAT);
 258     __ jcc(Assembler::equal, is_float);
 259     __ cmpl(rsi, T_DOUBLE);
 260     __ jcc(Assembler::equal, is_double);
 261 
 262     // handle T_INT case
 263     __ movl(Address(rdi, 0), rax);
 264     __ BIND(exit);
 265 
 266     // check that FPU stack is empty
 267     __ verify_FPU(0, "generate_call_stub");
 268 
 269     // pop parameters
 270     __ lea(rsp, rsp_after_call);
 271 
 272     // restore %mxcsr
 273     if (sse_save) {
 274       __ ldmxcsr(mxcsr_save);
 275     }
 276 
 277     // restore rdi, rsi and rbx,
 278     __ movptr(rbx, saved_rbx);
 279     __ movptr(rsi, saved_rsi);
 280     __ movptr(rdi, saved_rdi);
 281     __ addptr(rsp, 4*wordSize);
 282 
 283     // return
 284     __ pop(rbp);
 285     __ ret(0);
 286 
 287     // handle return types different from T_INT
 288     __ BIND(is_long);
 289     __ movl(Address(rdi, 0 * wordSize), rax);
 290     __ movl(Address(rdi, 1 * wordSize), rdx);
 291     __ jmp(exit);
 292 
 293     __ BIND(is_float);
 294     // interpreter uses xmm0 for return values
 295     if (UseSSE >= 1) {
 296       __ movflt(Address(rdi, 0), xmm0);
 297     } else {
 298       __ fstp_s(Address(rdi, 0));
 299     }
 300     __ jmp(exit);
 301 
 302     __ BIND(is_double);
 303     // interpreter uses xmm0 for return values
 304     if (UseSSE >= 2) {
 305       __ movdbl(Address(rdi, 0), xmm0);
 306     } else {
 307       __ fstp_d(Address(rdi, 0));
 308     }
 309     __ jmp(exit);
 310 
 311     return start;
 312   }
 313 
 314 
 315   //------------------------------------------------------------------------------------------------------------------------
 316   // Return point for a Java call if there's an exception thrown in Java code.
 317   // The exception is caught and transformed into a pending exception stored in
 318   // JavaThread that can be tested from within the VM.
 319   //
 320   // Note: Usually the parameters are removed by the callee. In case of an exception
 321   //       crossing an activation frame boundary, that is not the case if the callee
 322   //       is compiled code => need to setup the rsp.
 323   //
 324   // rax,: exception oop
 325 
 326   address generate_catch_exception() {
 327     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 328     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
 329     const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!
 330     address start = __ pc();
 331 
 332     // get thread directly
 333     __ movptr(rcx, thread);
 334 #ifdef ASSERT
 335     // verify that threads correspond
 336     { Label L;
 337       __ get_thread(rbx);
 338       __ cmpptr(rbx, rcx);
 339       __ jcc(Assembler::equal, L);
 340       __ stop("StubRoutines::catch_exception: threads must correspond");
 341       __ bind(L);
 342     }
 343 #endif
 344     // set pending exception
 345     __ verify_oop(rax);
 346     __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );
 347     __ lea(Address(rcx, Thread::exception_file_offset   ()),
 348            ExternalAddress((address)__FILE__));
 349     __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );
 350     // complete return to VM
 351     assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
 352     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 353 
 354     return start;
 355   }
 356 
 357 
 358   //------------------------------------------------------------------------------------------------------------------------
 359   // Continuation point for runtime calls returning with a pending exception.
 360   // The pending exception check happened in the runtime or native call stub.
 361   // The pending exception in Thread is converted into a Java-level exception.
 362   //
 363   // Contract with Java-level exception handlers:
 364   // rax: exception
 365   // rdx: throwing pc
 366   //
 367   // NOTE: At entry of this stub, exception-pc must be on stack !!
 368 
 369   address generate_forward_exception() {
 370     StubCodeMark mark(this, "StubRoutines", "forward exception");
 371     address start = __ pc();
 372     const Register thread = rcx;
 373 
 374     // other registers used in this stub
 375     const Register exception_oop = rax;
 376     const Register handler_addr  = rbx;
 377     const Register exception_pc  = rdx;
 378 
 379     // Upon entry, the sp points to the return address returning into Java
 380     // (interpreted or compiled) code; i.e., the return address becomes the
 381     // throwing pc.
 382     //
 383     // Arguments pushed before the runtime call are still on the stack but
 384     // the exception handler will reset the stack pointer -> ignore them.
 385     // A potential result in registers can be ignored as well.
 386 
 387 #ifdef ASSERT
 388     // make sure this code is only executed if there is a pending exception
 389     { Label L;
 390       __ get_thread(thread);
 391       __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 392       __ jcc(Assembler::notEqual, L);
 393       __ stop("StubRoutines::forward exception: no pending exception (1)");
 394       __ bind(L);
 395     }
 396 #endif
 397 
 398     // compute exception handler into rbx,
 399     __ get_thread(thread);
 400     __ movptr(exception_pc, Address(rsp, 0));
 401     BLOCK_COMMENT("call exception_handler_for_return_address");
 402     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
 403     __ mov(handler_addr, rax);
 404 
 405     // setup rax & rdx, remove return address & clear pending exception
 406     __ get_thread(thread);
 407     __ pop(exception_pc);
 408     __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
 409     __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
 410 
 411 #ifdef ASSERT
 412     // make sure exception is set
 413     { Label L;
 414       __ testptr(exception_oop, exception_oop);
 415       __ jcc(Assembler::notEqual, L);
 416       __ stop("StubRoutines::forward exception: no pending exception (2)");
 417       __ bind(L);
 418     }
 419 #endif
 420 
 421     // Verify that there is really a valid exception in RAX.
 422     __ verify_oop(exception_oop);
 423 
 424     // continue at exception handler (return address removed)
 425     // rax: exception
 426     // rbx: exception handler
 427     // rdx: throwing pc
 428     __ jmp(handler_addr);
 429 
 430     return start;
 431   }
 432 
 433   //----------------------------------------------------------------------------------------------------
 434   // Support for void verify_mxcsr()
 435   //
 436   // This routine is used with -Xcheck:jni to verify that native
 437   // JNI code does not return to Java code without restoring the
 438   // MXCSR register to our expected state.
 439 
 440 
 441   address generate_verify_mxcsr() {
 442     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 443     address start = __ pc();
 444 
 445     const Address mxcsr_save(rsp, 0);
 446 
 447     if (CheckJNICalls && UseSSE > 0 ) {
 448       Label ok_ret;
 449       ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
 450       __ push(rax);
 451       __ subptr(rsp, wordSize);      // allocate a temp location
 452       __ stmxcsr(mxcsr_save);
 453       __ movl(rax, mxcsr_save);
 454       __ andl(rax, MXCSR_MASK);
 455       __ cmp32(rax, mxcsr_std);
 456       __ jcc(Assembler::equal, ok_ret);
 457 
 458       __ warn("MXCSR changed by native JNI code.");
 459 
 460       __ ldmxcsr(mxcsr_std);
 461 
 462       __ bind(ok_ret);
 463       __ addptr(rsp, wordSize);
 464       __ pop(rax);
 465     }
 466 
 467     __ ret(0);
 468 
 469     return start;
 470   }
 471 
 472 
 473   //---------------------------------------------------------------------------
 474   // Support for void verify_fpu_cntrl_wrd()
 475   //
 476   // This routine is used with -Xcheck:jni to verify that native
 477   // JNI code does not return to Java code without restoring the
 478   // FP control word to our expected state.
 479 
 480   address generate_verify_fpu_cntrl_wrd() {
 481     StubCodeMark mark(this, "StubRoutines", "verify_spcw");
 482     address start = __ pc();
 483 
 484     const Address fpu_cntrl_wrd_save(rsp, 0);
 485 
 486     if (CheckJNICalls) {
 487       Label ok_ret;
 488       __ push(rax);
 489       __ subptr(rsp, wordSize);      // allocate a temp location
 490       __ fnstcw(fpu_cntrl_wrd_save);
 491       __ movl(rax, fpu_cntrl_wrd_save);
 492       __ andl(rax, FPU_CNTRL_WRD_MASK);
 493       ExternalAddress fpu_std(StubRoutines::x86::addr_fpu_cntrl_wrd_std());
 494       __ cmp32(rax, fpu_std);
 495       __ jcc(Assembler::equal, ok_ret);
 496 
 497       __ warn("Floating point control word changed by native JNI code.");
 498 
 499       __ fldcw(fpu_std);
 500 
 501       __ bind(ok_ret);
 502       __ addptr(rsp, wordSize);
 503       __ pop(rax);
 504     }
 505 
 506     __ ret(0);
 507 
 508     return start;
 509   }
 510 
 511   //---------------------------------------------------------------------------
 512   // Wrapper for slow-case handling of double-to-integer conversion
 513   // d2i or f2i fast case failed either because it is nan or because
 514   // of under/overflow.
 515   // Input:  FPU TOS: float value
 516   // Output: rax, (rdx): integer (long) result
 517 
 518   address generate_d2i_wrapper(BasicType t, address fcn) {
 519     StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
 520     address start = __ pc();
 521 
 522   // Capture info about frame layout
 523   enum layout { FPUState_off         = 0,
 524                 rbp_off              = FPUStateSizeInWords,
 525                 rdi_off,
 526                 rsi_off,
 527                 rcx_off,
 528                 rbx_off,
 529                 saved_argument_off,
 530                 saved_argument_off2, // 2nd half of double
 531                 framesize
 532   };
 533 
 534   assert(FPUStateSizeInWords == 27, "update stack layout");
 535 
 536     // Save outgoing argument to stack across push_FPU_state()
 537     __ subptr(rsp, wordSize * 2);
 538     __ fstp_d(Address(rsp, 0));
 539 
 540     // Save CPU & FPU state
 541     __ push(rbx);
 542     __ push(rcx);
 543     __ push(rsi);
 544     __ push(rdi);
 545     __ push(rbp);
 546     __ push_FPU_state();
 547 
 548     // push_FPU_state() resets the FP top of stack
 549     // Load original double into FP top of stack
 550     __ fld_d(Address(rsp, saved_argument_off * wordSize));
 551     // Store double into stack as outgoing argument
 552     __ subptr(rsp, wordSize*2);
 553     __ fst_d(Address(rsp, 0));
 554 
 555     // Prepare FPU for doing math in C-land
 556     __ empty_FPU_stack();
 557     // Call the C code to massage the double.  Result in EAX
 558     if (t == T_INT)
 559       { BLOCK_COMMENT("SharedRuntime::d2i"); }
 560     else if (t == T_LONG)
 561       { BLOCK_COMMENT("SharedRuntime::d2l"); }
 562     __ call_VM_leaf( fcn, 2 );
 563 
 564     // Restore CPU & FPU state
 565     __ pop_FPU_state();
 566     __ pop(rbp);
 567     __ pop(rdi);
 568     __ pop(rsi);
 569     __ pop(rcx);
 570     __ pop(rbx);
 571     __ addptr(rsp, wordSize * 2);
 572 
 573     __ ret(0);
 574 
 575     return start;
 576   }
 577   //---------------------------------------------------------------------------------------------------
 578 
 579   address generate_vector_mask(const char *stub_name, int32_t mask) {
 580     __ align(CodeEntryAlignment);
 581     StubCodeMark mark(this, "StubRoutines", stub_name);
 582     address start = __ pc();
 583 
 584     for (int i = 0; i < 16; i++) {
 585       __ emit_data(mask, relocInfo::none, 0);
 586     }
 587 
 588     return start;
 589   }
 590 
 591   address generate_iota_indices(const char *stub_name) {
 592     __ align(CodeEntryAlignment);
 593     StubCodeMark mark(this, "StubRoutines", stub_name);
 594     address start = __ pc();
 595     __ emit_data(0x03020100, relocInfo::none, 0);
 596     __ emit_data(0x07060504, relocInfo::none, 0);
 597     __ emit_data(0x0B0A0908, relocInfo::none, 0);
 598     __ emit_data(0x0F0E0D0C, relocInfo::none, 0);
 599     __ emit_data(0x13121110, relocInfo::none, 0);
 600     __ emit_data(0x17161514, relocInfo::none, 0);
 601     __ emit_data(0x1B1A1918, relocInfo::none, 0);
 602     __ emit_data(0x1F1E1D1C, relocInfo::none, 0);
 603     __ emit_data(0x23222120, relocInfo::none, 0);
 604     __ emit_data(0x27262524, relocInfo::none, 0);
 605     __ emit_data(0x2B2A2928, relocInfo::none, 0);
 606     __ emit_data(0x2F2E2D2C, relocInfo::none, 0);
 607     __ emit_data(0x33323130, relocInfo::none, 0);
 608     __ emit_data(0x37363534, relocInfo::none, 0);
 609     __ emit_data(0x3B3A3938, relocInfo::none, 0);
 610     __ emit_data(0x3F3E3D3C, relocInfo::none, 0);
 611     return start;
 612   }
 613 
 614   address generate_vector_byte_shuffle_mask(const char *stub_name) {
 615     __ align(CodeEntryAlignment);
 616     StubCodeMark mark(this, "StubRoutines", stub_name);
 617     address start = __ pc();
 618     __ emit_data(0x70707070, relocInfo::none, 0);
 619     __ emit_data(0x70707070, relocInfo::none, 0);
 620     __ emit_data(0x70707070, relocInfo::none, 0);
 621     __ emit_data(0x70707070, relocInfo::none, 0);
 622     __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
 623     __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
 624     __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
 625     __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
 626     return start;
 627   }
 628 
 629   address generate_vector_mask_long_double(const char *stub_name, int32_t maskhi, int32_t masklo) {
 630     __ align(CodeEntryAlignment);
 631     StubCodeMark mark(this, "StubRoutines", stub_name);
 632     address start = __ pc();
 633 
 634     for (int i = 0; i < 8; i++) {
 635       __ emit_data(masklo, relocInfo::none, 0);
 636       __ emit_data(maskhi, relocInfo::none, 0);
 637     }
 638 
 639     return start;
 640   }
 641 
 642   //----------------------------------------------------------------------------------------------------
 643 
 644   address generate_vector_byte_perm_mask(const char *stub_name) {
 645     __ align(CodeEntryAlignment);
 646     StubCodeMark mark(this, "StubRoutines", stub_name);
 647     address start = __ pc();
 648 
 649     __ emit_data(0x00000001, relocInfo::none, 0);
 650     __ emit_data(0x00000000, relocInfo::none, 0);
 651     __ emit_data(0x00000003, relocInfo::none, 0);
 652     __ emit_data(0x00000000, relocInfo::none, 0);
 653     __ emit_data(0x00000005, relocInfo::none, 0);
 654     __ emit_data(0x00000000, relocInfo::none, 0);
 655     __ emit_data(0x00000007, relocInfo::none, 0);
 656     __ emit_data(0x00000000, relocInfo::none, 0);
 657     __ emit_data(0x00000000, relocInfo::none, 0);
 658     __ emit_data(0x00000000, relocInfo::none, 0);
 659     __ emit_data(0x00000002, relocInfo::none, 0);
 660     __ emit_data(0x00000000, relocInfo::none, 0);
 661     __ emit_data(0x00000004, relocInfo::none, 0);
 662     __ emit_data(0x00000000, relocInfo::none, 0);
 663     __ emit_data(0x00000006, relocInfo::none, 0);
 664     __ emit_data(0x00000000, relocInfo::none, 0);
 665 
 666     return start;
 667   }
 668 
 669   address generate_vector_custom_i32(const char *stub_name, Assembler::AvxVectorLen len,
 670                                      int32_t val0, int32_t val1, int32_t val2, int32_t val3,
 671                                      int32_t val4 = 0, int32_t val5 = 0, int32_t val6 = 0, int32_t val7 = 0,
 672                                      int32_t val8 = 0, int32_t val9 = 0, int32_t val10 = 0, int32_t val11 = 0,
 673                                      int32_t val12 = 0, int32_t val13 = 0, int32_t val14 = 0, int32_t val15 = 0) {
 674     __ align(CodeEntryAlignment);
 675     StubCodeMark mark(this, "StubRoutines", stub_name);
 676     address start = __ pc();
 677 
 678     assert(len != Assembler::AVX_NoVec, "vector len must be specified");
 679     __ emit_data(val0, relocInfo::none, 0);
 680     __ emit_data(val1, relocInfo::none, 0);
 681     __ emit_data(val2, relocInfo::none, 0);
 682     __ emit_data(val3, relocInfo::none, 0);
 683     if (len >= Assembler::AVX_256bit) {
 684       __ emit_data(val4, relocInfo::none, 0);
 685       __ emit_data(val5, relocInfo::none, 0);
 686       __ emit_data(val6, relocInfo::none, 0);
 687       __ emit_data(val7, relocInfo::none, 0);
 688       if (len >= Assembler::AVX_512bit) {
 689         __ emit_data(val8, relocInfo::none, 0);
 690         __ emit_data(val9, relocInfo::none, 0);
 691         __ emit_data(val10, relocInfo::none, 0);
 692         __ emit_data(val11, relocInfo::none, 0);
 693         __ emit_data(val12, relocInfo::none, 0);
 694         __ emit_data(val13, relocInfo::none, 0);
 695         __ emit_data(val14, relocInfo::none, 0);
 696         __ emit_data(val15, relocInfo::none, 0);
 697       }
 698     }
 699 
 700     return start;
 701   }
 702 
 703   //----------------------------------------------------------------------------------------------------
 704   // Non-destructive plausibility checks for oops
 705 
 706   address generate_verify_oop() {
 707     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 708     address start = __ pc();
 709 
 710     // Incoming arguments on stack after saving rax,:
 711     //
 712     // [tos    ]: saved rdx
 713     // [tos + 1]: saved EFLAGS
 714     // [tos + 2]: return address
 715     // [tos + 3]: char* error message
 716     // [tos + 4]: oop   object to verify
 717     // [tos + 5]: saved rax, - saved by caller and bashed
 718 
 719     Label exit, error;
 720     __ pushf();
 721     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 722     __ push(rdx);                                // save rdx
 723     // make sure object is 'reasonable'
 724     __ movptr(rax, Address(rsp, 4 * wordSize));    // get object
 725     __ testptr(rax, rax);
 726     __ jcc(Assembler::zero, exit);               // if obj is NULL it is ok
 727 
 728     // Check if the oop is in the right area of memory
 729     const int oop_mask = Universe::verify_oop_mask();
 730     const int oop_bits = Universe::verify_oop_bits();
 731     __ mov(rdx, rax);
 732     __ andptr(rdx, oop_mask);
 733     __ cmpptr(rdx, oop_bits);
 734     __ jcc(Assembler::notZero, error);
 735 
 736     // make sure klass is 'reasonable', which is not zero.
 737     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
 738     __ testptr(rax, rax);
 739     __ jcc(Assembler::zero, error);              // if klass is NULL it is broken
 740 
 741     // return if everything seems ok
 742     __ bind(exit);
 743     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 744     __ pop(rdx);                                 // restore rdx
 745     __ popf();                                   // restore EFLAGS
 746     __ ret(3 * wordSize);                        // pop arguments
 747 
 748     // handle errors
 749     __ bind(error);
 750     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 751     __ pop(rdx);                                 // get saved rdx back
 752     __ popf();                                   // get saved EFLAGS off stack -- will be ignored
 753     __ pusha();                                  // push registers (eip = return address & msg are already pushed)
 754     BLOCK_COMMENT("call MacroAssembler::debug");
 755     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 756     __ hlt();
 757     return start;
 758   }
 759 
 760 
 761   // Copy 64 bytes chunks
 762   //
 763   // Inputs:
 764   //   from        - source array address
 765   //   to_from     - destination array address - from
 766   //   qword_count - 8-bytes element count, negative
 767   //
 768   void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
 769     assert( UseSSE >= 2, "supported cpu only" );
 770     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 771 
 772     // Copy 64-byte chunks
 773     __ jmpb(L_copy_64_bytes);
 774     __ align(OptoLoopAlignment);
 775   __ BIND(L_copy_64_bytes_loop);
 776 
 777     if (UseUnalignedLoadStores) {
 778       if (UseAVX > 2) {
 779         __ evmovdqul(xmm0, Address(from, 0), Assembler::AVX_512bit);
 780         __ evmovdqul(Address(from, to_from, Address::times_1, 0), xmm0, Assembler::AVX_512bit);
 781       } else if (UseAVX == 2) {
 782         __ vmovdqu(xmm0, Address(from,  0));
 783         __ vmovdqu(Address(from, to_from, Address::times_1,  0), xmm0);
 784         __ vmovdqu(xmm1, Address(from, 32));
 785         __ vmovdqu(Address(from, to_from, Address::times_1, 32), xmm1);
 786       } else {
 787         __ movdqu(xmm0, Address(from, 0));
 788         __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
 789         __ movdqu(xmm1, Address(from, 16));
 790         __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
 791         __ movdqu(xmm2, Address(from, 32));
 792         __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
 793         __ movdqu(xmm3, Address(from, 48));
 794         __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
 795       }
 796     } else {
 797       __ movq(xmm0, Address(from, 0));
 798       __ movq(Address(from, to_from, Address::times_1, 0), xmm0);
 799       __ movq(xmm1, Address(from, 8));
 800       __ movq(Address(from, to_from, Address::times_1, 8), xmm1);
 801       __ movq(xmm2, Address(from, 16));
 802       __ movq(Address(from, to_from, Address::times_1, 16), xmm2);
 803       __ movq(xmm3, Address(from, 24));
 804       __ movq(Address(from, to_from, Address::times_1, 24), xmm3);
 805       __ movq(xmm4, Address(from, 32));
 806       __ movq(Address(from, to_from, Address::times_1, 32), xmm4);
 807       __ movq(xmm5, Address(from, 40));
 808       __ movq(Address(from, to_from, Address::times_1, 40), xmm5);
 809       __ movq(xmm6, Address(from, 48));
 810       __ movq(Address(from, to_from, Address::times_1, 48), xmm6);
 811       __ movq(xmm7, Address(from, 56));
 812       __ movq(Address(from, to_from, Address::times_1, 56), xmm7);
 813     }
 814 
 815     __ addl(from, 64);
 816   __ BIND(L_copy_64_bytes);
 817     __ subl(qword_count, 8);
 818     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 819 
 820     if (UseUnalignedLoadStores && (UseAVX == 2)) {
 821       // clean upper bits of YMM registers
 822       __ vpxor(xmm0, xmm0);
 823       __ vpxor(xmm1, xmm1);
 824     }
 825     __ addl(qword_count, 8);
 826     __ jccb(Assembler::zero, L_exit);
 827     //
 828     // length is too short, just copy qwords
 829     //
 830   __ BIND(L_copy_8_bytes);
 831     __ movq(xmm0, Address(from, 0));
 832     __ movq(Address(from, to_from, Address::times_1), xmm0);
 833     __ addl(from, 8);
 834     __ decrement(qword_count);
 835     __ jcc(Assembler::greater, L_copy_8_bytes);
 836   __ BIND(L_exit);
 837   }
 838 
 839   address generate_disjoint_copy(BasicType t, bool aligned,
 840                                  Address::ScaleFactor sf,
 841                                  address* entry, const char *name,
 842                                  bool dest_uninitialized = false) {
 843     __ align(CodeEntryAlignment);
 844     StubCodeMark mark(this, "StubRoutines", name);
 845     address start = __ pc();
 846 
 847     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
 848     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
 849 
 850     int shift = Address::times_ptr - sf;
 851 
 852     const Register from     = rsi;  // source array address
 853     const Register to       = rdi;  // destination array address
 854     const Register count    = rcx;  // elements count
 855     const Register to_from  = to;   // (to - from)
 856     const Register saved_to = rdx;  // saved destination array address
 857 
 858     __ enter(); // required for proper stackwalking of RuntimeStub frame
 859     __ push(rsi);
 860     __ push(rdi);
 861     __ movptr(from , Address(rsp, 12+ 4));
 862     __ movptr(to   , Address(rsp, 12+ 8));
 863     __ movl(count, Address(rsp, 12+ 12));
 864 
 865     if (entry != NULL) {
 866       *entry = __ pc(); // Entry point from conjoint arraycopy stub.
 867       BLOCK_COMMENT("Entry:");
 868     }
 869 
 870     if (t == T_OBJECT) {
 871       __ testl(count, count);
 872       __ jcc(Assembler::zero, L_0_count);
 873     }
 874 
 875     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
 876     if (dest_uninitialized) {
 877       decorators |= IS_DEST_UNINITIALIZED;
 878     }
 879     if (aligned) {
 880       decorators |= ARRAYCOPY_ALIGNED;
 881     }
 882 
 883     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 884     bs->arraycopy_prologue(_masm, decorators, t, from, to, count);
 885     {
 886       bool add_entry = (t != T_OBJECT && (!aligned || t == T_INT));
 887       // UnsafeCopyMemory page error: continue after ucm
 888       UnsafeCopyMemoryMark ucmm(this, add_entry, true);
 889       __ subptr(to, from); // to --> to_from
 890       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
 891       __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
 892       if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 893         // align source address at 4 bytes address boundary
 894         if (t == T_BYTE) {
 895           // One byte misalignment happens only for byte arrays
 896           __ testl(from, 1);
 897           __ jccb(Assembler::zero, L_skip_align1);
 898           __ movb(rax, Address(from, 0));
 899           __ movb(Address(from, to_from, Address::times_1, 0), rax);
 900           __ increment(from);
 901           __ decrement(count);
 902         __ BIND(L_skip_align1);
 903         }
 904         // Two bytes misalignment happens only for byte and short (char) arrays
 905         __ testl(from, 2);
 906         __ jccb(Assembler::zero, L_skip_align2);
 907         __ movw(rax, Address(from, 0));
 908         __ movw(Address(from, to_from, Address::times_1, 0), rax);
 909         __ addptr(from, 2);
 910         __ subl(count, 1<<(shift-1));
 911       __ BIND(L_skip_align2);
 912       }
 913       if (!UseXMMForArrayCopy) {
 914         __ mov(rax, count);      // save 'count'
 915         __ shrl(count, shift); // bytes count
 916         __ addptr(to_from, from);// restore 'to'
 917         __ rep_mov();
 918         __ subptr(to_from, from);// restore 'to_from'
 919         __ mov(count, rax);      // restore 'count'
 920         __ jmpb(L_copy_2_bytes); // all dwords were copied
 921       } else {
 922         if (!UseUnalignedLoadStores) {
 923           // align to 8 bytes, we know we are 4 byte aligned to start
 924           __ testptr(from, 4);
 925           __ jccb(Assembler::zero, L_copy_64_bytes);
 926           __ movl(rax, Address(from, 0));
 927           __ movl(Address(from, to_from, Address::times_1, 0), rax);
 928           __ addptr(from, 4);
 929           __ subl(count, 1<<shift);
 930         }
 931       __ BIND(L_copy_64_bytes);
 932         __ mov(rax, count);
 933         __ shrl(rax, shift+1);  // 8 bytes chunk count
 934         //
 935         // Copy 8-byte chunks through XMM registers, 8 per iteration of the loop
 936         //
 937         xmm_copy_forward(from, to_from, rax);
 938       }
 939       // copy tailing dword
 940     __ BIND(L_copy_4_bytes);
 941       __ testl(count, 1<<shift);
 942       __ jccb(Assembler::zero, L_copy_2_bytes);
 943       __ movl(rax, Address(from, 0));
 944       __ movl(Address(from, to_from, Address::times_1, 0), rax);
 945       if (t == T_BYTE || t == T_SHORT) {
 946         __ addptr(from, 4);
 947       __ BIND(L_copy_2_bytes);
 948         // copy tailing word
 949         __ testl(count, 1<<(shift-1));
 950         __ jccb(Assembler::zero, L_copy_byte);
 951         __ movw(rax, Address(from, 0));
 952         __ movw(Address(from, to_from, Address::times_1, 0), rax);
 953         if (t == T_BYTE) {
 954           __ addptr(from, 2);
 955         __ BIND(L_copy_byte);
 956           // copy tailing byte
 957           __ testl(count, 1);
 958           __ jccb(Assembler::zero, L_exit);
 959           __ movb(rax, Address(from, 0));
 960           __ movb(Address(from, to_from, Address::times_1, 0), rax);
 961         __ BIND(L_exit);
 962         } else {
 963         __ BIND(L_copy_byte);
 964         }
 965       } else {
 966       __ BIND(L_copy_2_bytes);
 967       }
 968     }
 969 
 970     __ movl(count, Address(rsp, 12+12)); // reread 'count'
 971     bs->arraycopy_epilogue(_masm, decorators, t, from, to, count);
 972 
 973     if (t == T_OBJECT) {
 974     __ BIND(L_0_count);
 975     }
 976     inc_copy_counter_np(t);
 977     __ pop(rdi);
 978     __ pop(rsi);
 979     __ leave(); // required for proper stackwalking of RuntimeStub frame
 980     __ vzeroupper();
 981     __ xorptr(rax, rax); // return 0
 982     __ ret(0);
 983     return start;
 984   }
 985 
 986 
 987   address generate_fill(BasicType t, bool aligned, const char *name) {
 988     __ align(CodeEntryAlignment);
 989     StubCodeMark mark(this, "StubRoutines", name);
 990     address start = __ pc();
 991 
 992     BLOCK_COMMENT("Entry:");
 993 
 994     const Register to       = rdi;  // source array address
 995     const Register value    = rdx;  // value
 996     const Register count    = rsi;  // elements count
 997 
 998     __ enter(); // required for proper stackwalking of RuntimeStub frame
 999     __ push(rsi);
1000     __ push(rdi);
1001     __ movptr(to   , Address(rsp, 12+ 4));
1002     __ movl(value, Address(rsp, 12+ 8));
1003     __ movl(count, Address(rsp, 12+ 12));
1004 
1005     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1006 
1007     __ pop(rdi);
1008     __ pop(rsi);
1009     __ leave(); // required for proper stackwalking of RuntimeStub frame
1010     __ ret(0);
1011     return start;
1012   }
1013 
1014   address generate_conjoint_copy(BasicType t, bool aligned,
1015                                  Address::ScaleFactor sf,
1016                                  address nooverlap_target,
1017                                  address* entry, const char *name,
1018                                  bool dest_uninitialized = false) {
1019     __ align(CodeEntryAlignment);
1020     StubCodeMark mark(this, "StubRoutines", name);
1021     address start = __ pc();
1022 
1023     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
1024     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
1025 
1026     int shift = Address::times_ptr - sf;
1027 
1028     const Register src   = rax;  // source array address
1029     const Register dst   = rdx;  // destination array address
1030     const Register from  = rsi;  // source array address
1031     const Register to    = rdi;  // destination array address
1032     const Register count = rcx;  // elements count
1033     const Register end   = rax;  // array end address
1034 
1035     __ enter(); // required for proper stackwalking of RuntimeStub frame
1036     __ push(rsi);
1037     __ push(rdi);
1038     __ movptr(src  , Address(rsp, 12+ 4));   // from
1039     __ movptr(dst  , Address(rsp, 12+ 8));   // to
1040     __ movl2ptr(count, Address(rsp, 12+12)); // count
1041 
1042     if (entry != NULL) {
1043       *entry = __ pc(); // Entry point from generic arraycopy stub.
1044       BLOCK_COMMENT("Entry:");
1045     }
1046 
1047     // nooverlap_target expects arguments in rsi and rdi.
1048     __ mov(from, src);
1049     __ mov(to  , dst);
1050 
1051     // arrays overlap test: dispatch to disjoint stub if necessary.
1052     RuntimeAddress nooverlap(nooverlap_target);
1053     __ cmpptr(dst, src);
1054     __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
1055     __ jump_cc(Assembler::belowEqual, nooverlap);
1056     __ cmpptr(dst, end);
1057     __ jump_cc(Assembler::aboveEqual, nooverlap);
1058 
1059     if (t == T_OBJECT) {
1060       __ testl(count, count);
1061       __ jcc(Assembler::zero, L_0_count);
1062     }
1063 
1064     DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1065     if (dest_uninitialized) {
1066       decorators |= IS_DEST_UNINITIALIZED;
1067     }
1068     if (aligned) {
1069       decorators |= ARRAYCOPY_ALIGNED;
1070     }
1071 
1072     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1073     bs->arraycopy_prologue(_masm, decorators, t, from, to, count);
1074 
1075     {
1076       bool add_entry = (t != T_OBJECT && (!aligned || t == T_INT));
1077       // UnsafeCopyMemory page error: continue after ucm
1078       UnsafeCopyMemoryMark ucmm(this, add_entry, true);
1079       // copy from high to low
1080       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1081       __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1082       if (t == T_BYTE || t == T_SHORT) {
1083         // Align the end of destination array at 4 bytes address boundary
1084         __ lea(end, Address(dst, count, sf, 0));
1085         if (t == T_BYTE) {
1086           // One byte misalignment happens only for byte arrays
1087           __ testl(end, 1);
1088           __ jccb(Assembler::zero, L_skip_align1);
1089           __ decrement(count);
1090           __ movb(rdx, Address(from, count, sf, 0));
1091           __ movb(Address(to, count, sf, 0), rdx);
1092         __ BIND(L_skip_align1);
1093         }
1094         // Two bytes misalignment happens only for byte and short (char) arrays
1095         __ testl(end, 2);
1096         __ jccb(Assembler::zero, L_skip_align2);
1097         __ subptr(count, 1<<(shift-1));
1098         __ movw(rdx, Address(from, count, sf, 0));
1099         __ movw(Address(to, count, sf, 0), rdx);
1100       __ BIND(L_skip_align2);
1101         __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1102         __ jcc(Assembler::below, L_copy_4_bytes);
1103       }
1104 
1105       if (!UseXMMForArrayCopy) {
1106         __ std();
1107         __ mov(rax, count); // Save 'count'
1108         __ mov(rdx, to);    // Save 'to'
1109         __ lea(rsi, Address(from, count, sf, -4));
1110         __ lea(rdi, Address(to  , count, sf, -4));
1111         __ shrptr(count, shift); // bytes count
1112         __ rep_mov();
1113         __ cld();
1114         __ mov(count, rax); // restore 'count'
1115         __ andl(count, (1<<shift)-1);      // mask the number of rest elements
1116         __ movptr(from, Address(rsp, 12+4)); // reread 'from'
1117         __ mov(to, rdx);   // restore 'to'
1118         __ jmpb(L_copy_2_bytes); // all dword were copied
1119       } else {
1120         // Align to 8 bytes the end of array. It is aligned to 4 bytes already.
1121         __ testptr(end, 4);
1122         __ jccb(Assembler::zero, L_copy_8_bytes);
1123         __ subl(count, 1<<shift);
1124         __ movl(rdx, Address(from, count, sf, 0));
1125         __ movl(Address(to, count, sf, 0), rdx);
1126         __ jmpb(L_copy_8_bytes);
1127 
1128         __ align(OptoLoopAlignment);
1129         // Move 8 bytes
1130       __ BIND(L_copy_8_bytes_loop);
1131         __ movq(xmm0, Address(from, count, sf, 0));
1132         __ movq(Address(to, count, sf, 0), xmm0);
1133       __ BIND(L_copy_8_bytes);
1134         __ subl(count, 2<<shift);
1135         __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1136         __ addl(count, 2<<shift);
1137       }
1138     __ BIND(L_copy_4_bytes);
1139       // copy prefix qword
1140       __ testl(count, 1<<shift);
1141       __ jccb(Assembler::zero, L_copy_2_bytes);
1142       __ movl(rdx, Address(from, count, sf, -4));
1143       __ movl(Address(to, count, sf, -4), rdx);
1144 
1145       if (t == T_BYTE || t == T_SHORT) {
1146           __ subl(count, (1<<shift));
1147         __ BIND(L_copy_2_bytes);
1148           // copy prefix dword
1149           __ testl(count, 1<<(shift-1));
1150           __ jccb(Assembler::zero, L_copy_byte);
1151           __ movw(rdx, Address(from, count, sf, -2));
1152           __ movw(Address(to, count, sf, -2), rdx);
1153           if (t == T_BYTE) {
1154             __ subl(count, 1<<(shift-1));
1155           __ BIND(L_copy_byte);
1156             // copy prefix byte
1157             __ testl(count, 1);
1158             __ jccb(Assembler::zero, L_exit);
1159             __ movb(rdx, Address(from, 0));
1160             __ movb(Address(to, 0), rdx);
1161           __ BIND(L_exit);
1162           } else {
1163           __ BIND(L_copy_byte);
1164           }
1165       } else {
1166       __ BIND(L_copy_2_bytes);
1167       }
1168     }
1169 
1170     __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1171     bs->arraycopy_epilogue(_masm, decorators, t, from, to, count);
1172 
1173     if (t == T_OBJECT) {
1174     __ BIND(L_0_count);
1175     }
1176     inc_copy_counter_np(t);
1177     __ pop(rdi);
1178     __ pop(rsi);
1179     __ leave(); // required for proper stackwalking of RuntimeStub frame
1180     __ xorptr(rax, rax); // return 0
1181     __ ret(0);
1182     return start;
1183   }
1184 
1185 
1186   address generate_disjoint_long_copy(address* entry, const char *name) {
1187     __ align(CodeEntryAlignment);
1188     StubCodeMark mark(this, "StubRoutines", name);
1189     address start = __ pc();
1190 
1191     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1192     const Register from       = rax;  // source array address
1193     const Register to         = rdx;  // destination array address
1194     const Register count      = rcx;  // elements count
1195     const Register to_from    = rdx;  // (to - from)
1196 
1197     __ enter(); // required for proper stackwalking of RuntimeStub frame
1198     __ movptr(from , Address(rsp, 8+0));       // from
1199     __ movptr(to   , Address(rsp, 8+4));       // to
1200     __ movl2ptr(count, Address(rsp, 8+8));     // count
1201 
1202     *entry = __ pc(); // Entry point from conjoint arraycopy stub.
1203     BLOCK_COMMENT("Entry:");
1204 
1205     {
1206       // UnsafeCopyMemory page error: continue after ucm
1207       UnsafeCopyMemoryMark ucmm(this, true, true);
1208       __ subptr(to, from); // to --> to_from
1209       if (UseXMMForArrayCopy) {
1210         xmm_copy_forward(from, to_from, count);
1211       } else {
1212         __ jmpb(L_copy_8_bytes);
1213         __ align(OptoLoopAlignment);
1214       __ BIND(L_copy_8_bytes_loop);
1215         __ fild_d(Address(from, 0));
1216         __ fistp_d(Address(from, to_from, Address::times_1));
1217         __ addptr(from, 8);
1218       __ BIND(L_copy_8_bytes);
1219         __ decrement(count);
1220         __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1221       }
1222     }
1223     inc_copy_counter_np(T_LONG);
1224     __ leave(); // required for proper stackwalking of RuntimeStub frame
1225     __ vzeroupper();
1226     __ xorptr(rax, rax); // return 0
1227     __ ret(0);
1228     return start;
1229   }
1230 
1231   address generate_conjoint_long_copy(address nooverlap_target,
1232                                       address* entry, const char *name) {
1233     __ align(CodeEntryAlignment);
1234     StubCodeMark mark(this, "StubRoutines", name);
1235     address start = __ pc();
1236 
1237     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1238     const Register from       = rax;  // source array address
1239     const Register to         = rdx;  // destination array address
1240     const Register count      = rcx;  // elements count
1241     const Register end_from   = rax;  // source array end address
1242 
1243     __ enter(); // required for proper stackwalking of RuntimeStub frame
1244     __ movptr(from , Address(rsp, 8+0));       // from
1245     __ movptr(to   , Address(rsp, 8+4));       // to
1246     __ movl2ptr(count, Address(rsp, 8+8));     // count
1247 
1248     *entry = __ pc(); // Entry point from generic arraycopy stub.
1249     BLOCK_COMMENT("Entry:");
1250 
1251     // arrays overlap test
1252     __ cmpptr(to, from);
1253     RuntimeAddress nooverlap(nooverlap_target);
1254     __ jump_cc(Assembler::belowEqual, nooverlap);
1255     __ lea(end_from, Address(from, count, Address::times_8, 0));
1256     __ cmpptr(to, end_from);
1257     __ movptr(from, Address(rsp, 8));  // from
1258     __ jump_cc(Assembler::aboveEqual, nooverlap);
1259 
1260     {
1261       // UnsafeCopyMemory page error: continue after ucm
1262       UnsafeCopyMemoryMark ucmm(this, true, true);
1263 
1264       __ jmpb(L_copy_8_bytes);
1265 
1266       __ align(OptoLoopAlignment);
1267     __ BIND(L_copy_8_bytes_loop);
1268       if (UseXMMForArrayCopy) {
1269         __ movq(xmm0, Address(from, count, Address::times_8));
1270         __ movq(Address(to, count, Address::times_8), xmm0);
1271       } else {
1272         __ fild_d(Address(from, count, Address::times_8));
1273         __ fistp_d(Address(to, count, Address::times_8));
1274       }
1275     __ BIND(L_copy_8_bytes);
1276       __ decrement(count);
1277       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1278 
1279     }
1280     inc_copy_counter_np(T_LONG);
1281     __ leave(); // required for proper stackwalking of RuntimeStub frame
1282     __ xorptr(rax, rax); // return 0
1283     __ ret(0);
1284     return start;
1285   }
1286 
1287 
1288   // Helper for generating a dynamic type check.
1289   // The sub_klass must be one of {rbx, rdx, rsi}.
1290   // The temp is killed.
1291   void generate_type_check(Register sub_klass,
1292                            Address& super_check_offset_addr,
1293                            Address& super_klass_addr,
1294                            Register temp,
1295                            Label* L_success, Label* L_failure) {
1296     BLOCK_COMMENT("type_check:");
1297 
1298     Label L_fallthrough;
1299 #define LOCAL_JCC(assembler_con, label_ptr)                             \
1300     if (label_ptr != NULL)  __ jcc(assembler_con, *(label_ptr));        \
1301     else                    __ jcc(assembler_con, L_fallthrough) /*omit semi*/
1302 
1303     // The following is a strange variation of the fast path which requires
1304     // one less register, because needed values are on the argument stack.
1305     // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
1306     //                                  L_success, L_failure, NULL);
1307     assert_different_registers(sub_klass, temp);
1308 
1309     int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1310 
1311     // if the pointers are equal, we are done (e.g., String[] elements)
1312     __ cmpptr(sub_klass, super_klass_addr);
1313     LOCAL_JCC(Assembler::equal, L_success);
1314 
1315     // check the supertype display:
1316     __ movl2ptr(temp, super_check_offset_addr);
1317     Address super_check_addr(sub_klass, temp, Address::times_1, 0);
1318     __ movptr(temp, super_check_addr); // load displayed supertype
1319     __ cmpptr(temp, super_klass_addr); // test the super type
1320     LOCAL_JCC(Assembler::equal, L_success);
1321 
1322     // if it was a primary super, we can just fail immediately
1323     __ cmpl(super_check_offset_addr, sc_offset);
1324     LOCAL_JCC(Assembler::notEqual, L_failure);
1325 
1326     // The repne_scan instruction uses fixed registers, which will get spilled.
1327     // We happen to know this works best when super_klass is in rax.
1328     Register super_klass = temp;
1329     __ movptr(super_klass, super_klass_addr);
1330     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
1331                                      L_success, L_failure);
1332 
1333     __ bind(L_fallthrough);
1334 
1335     if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
1336     if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
1337 
1338 #undef LOCAL_JCC
1339   }
1340 
1341   //
1342   //  Generate checkcasting array copy stub
1343   //
1344   //  Input:
1345   //    4(rsp)   - source array address
1346   //    8(rsp)   - destination array address
1347   //   12(rsp)   - element count, can be zero
1348   //   16(rsp)   - size_t ckoff (super_check_offset)
1349   //   20(rsp)   - oop ckval (super_klass)
1350   //
1351   //  Output:
1352   //    rax, ==  0  -  success
1353   //    rax, == -1^K - failure, where K is partial transfer count
1354   //
1355   address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitialized = false) {
1356     __ align(CodeEntryAlignment);
1357     StubCodeMark mark(this, "StubRoutines", name);
1358     address start = __ pc();
1359 
1360     Label L_load_element, L_store_element, L_do_card_marks, L_done;
1361 
1362     // register use:
1363     //  rax, rdx, rcx -- loop control (end_from, end_to, count)
1364     //  rdi, rsi      -- element access (oop, klass)
1365     //  rbx,           -- temp
1366     const Register from       = rax;    // source array address
1367     const Register to         = rdx;    // destination array address
1368     const Register length     = rcx;    // elements count
1369     const Register elem       = rdi;    // each oop copied
1370     const Register elem_klass = rsi;    // each elem._klass (sub_klass)
1371     const Register temp       = rbx;    // lone remaining temp
1372 
1373     __ enter(); // required for proper stackwalking of RuntimeStub frame
1374 
1375     __ push(rsi);
1376     __ push(rdi);
1377     __ push(rbx);
1378 
1379     Address   from_arg(rsp, 16+ 4);     // from
1380     Address     to_arg(rsp, 16+ 8);     // to
1381     Address length_arg(rsp, 16+12);     // elements count
1382     Address  ckoff_arg(rsp, 16+16);     // super_check_offset
1383     Address  ckval_arg(rsp, 16+20);     // super_klass
1384 
1385     // Load up:
1386     __ movptr(from,     from_arg);
1387     __ movptr(to,         to_arg);
1388     __ movl2ptr(length, length_arg);
1389 
1390     if (entry != NULL) {
1391       *entry = __ pc(); // Entry point from generic arraycopy stub.
1392       BLOCK_COMMENT("Entry:");
1393     }
1394 
1395     //---------------------------------------------------------------
1396     // Assembler stub will be used for this call to arraycopy
1397     // if the two arrays are subtypes of Object[] but the
1398     // destination array type is not equal to or a supertype
1399     // of the source type.  Each element must be separately
1400     // checked.
1401 
1402     // Loop-invariant addresses.  They are exclusive end pointers.
1403     Address end_from_addr(from, length, Address::times_ptr, 0);
1404     Address   end_to_addr(to,   length, Address::times_ptr, 0);
1405 
1406     Register end_from = from;           // re-use
1407     Register end_to   = to;             // re-use
1408     Register count    = length;         // re-use
1409 
1410     // Loop-variant addresses.  They assume post-incremented count < 0.
1411     Address from_element_addr(end_from, count, Address::times_ptr, 0);
1412     Address   to_element_addr(end_to,   count, Address::times_ptr, 0);
1413     Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
1414 
1415     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST;
1416     if (dest_uninitialized) {
1417       decorators |= IS_DEST_UNINITIALIZED;
1418     }
1419 
1420     BasicType type = T_OBJECT;
1421     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1422     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1423 
1424     // Copy from low to high addresses, indexed from the end of each array.
1425     __ lea(end_from, end_from_addr);
1426     __ lea(end_to,   end_to_addr);
1427     assert(length == count, "");        // else fix next line:
1428     __ negptr(count);                   // negate and test the length
1429     __ jccb(Assembler::notZero, L_load_element);
1430 
1431     // Empty array:  Nothing to do.
1432     __ xorptr(rax, rax);                  // return 0 on (trivial) success
1433     __ jmp(L_done);
1434 
1435     // ======== begin loop ========
1436     // (Loop is rotated; its entry is L_load_element.)
1437     // Loop control:
1438     //   for (count = -count; count != 0; count++)
1439     // Base pointers src, dst are biased by 8*count,to last element.
1440     __ align(OptoLoopAlignment);
1441 
1442     __ BIND(L_store_element);
1443     __ movptr(to_element_addr, elem);     // store the oop
1444     __ increment(count);                // increment the count toward zero
1445     __ jccb(Assembler::zero, L_do_card_marks);
1446 
1447     // ======== loop entry is here ========
1448     __ BIND(L_load_element);
1449     __ movptr(elem, from_element_addr);   // load the oop
1450     __ testptr(elem, elem);
1451     __ jccb(Assembler::zero, L_store_element);
1452 
1453     // (Could do a trick here:  Remember last successful non-null
1454     // element stored and make a quick oop equality check on it.)
1455 
1456     __ movptr(elem_klass, elem_klass_addr); // query the object klass
1457     generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
1458                         &L_store_element, NULL);
1459     // (On fall-through, we have failed the element type check.)
1460     // ======== end loop ========
1461 
1462     // It was a real error; we must depend on the caller to finish the job.
1463     // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
1464     // Emit GC store barriers for the oops we have copied (length_arg + count),
1465     // and report their number to the caller.
1466     assert_different_registers(to, count, rax);
1467     Label L_post_barrier;
1468     __ addl(count, length_arg);         // transfers = (length - remaining)
1469     __ movl2ptr(rax, count);            // save the value
1470     __ notptr(rax);                     // report (-1^K) to caller (does not affect flags)
1471     __ jccb(Assembler::notZero, L_post_barrier);
1472     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
1473 
1474     // Come here on success only.
1475     __ BIND(L_do_card_marks);
1476     __ xorptr(rax, rax);                // return 0 on success
1477     __ movl2ptr(count, length_arg);
1478 
1479     __ BIND(L_post_barrier);
1480     __ movptr(to, to_arg);              // reload
1481     bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
1482 
1483     // Common exit point (success or failure).
1484     __ BIND(L_done);
1485     __ pop(rbx);
1486     __ pop(rdi);
1487     __ pop(rsi);
1488     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1489     __ leave(); // required for proper stackwalking of RuntimeStub frame
1490     __ ret(0);
1491 
1492     return start;
1493   }
1494 
1495   //
1496   //  Generate 'unsafe' array copy stub
1497   //  Though just as safe as the other stubs, it takes an unscaled
1498   //  size_t argument instead of an element count.
1499   //
1500   //  Input:
1501   //    4(rsp)   - source array address
1502   //    8(rsp)   - destination array address
1503   //   12(rsp)   - byte count, can be zero
1504   //
1505   //  Output:
1506   //    rax, ==  0  -  success
1507   //    rax, == -1  -  need to call System.arraycopy
1508   //
1509   // Examines the alignment of the operands and dispatches
1510   // to a long, int, short, or byte copy loop.
1511   //
1512   address generate_unsafe_copy(const char *name,
1513                                address byte_copy_entry,
1514                                address short_copy_entry,
1515                                address int_copy_entry,
1516                                address long_copy_entry) {
1517 
1518     Label L_long_aligned, L_int_aligned, L_short_aligned;
1519 
1520     __ align(CodeEntryAlignment);
1521     StubCodeMark mark(this, "StubRoutines", name);
1522     address start = __ pc();
1523 
1524     const Register from       = rax;  // source array address
1525     const Register to         = rdx;  // destination array address
1526     const Register count      = rcx;  // elements count
1527 
1528     __ enter(); // required for proper stackwalking of RuntimeStub frame
1529     __ push(rsi);
1530     __ push(rdi);
1531     Address  from_arg(rsp, 12+ 4);      // from
1532     Address    to_arg(rsp, 12+ 8);      // to
1533     Address count_arg(rsp, 12+12);      // byte count
1534 
1535     // Load up:
1536     __ movptr(from ,  from_arg);
1537     __ movptr(to   ,    to_arg);
1538     __ movl2ptr(count, count_arg);
1539 
1540     // bump this on entry, not on exit:
1541     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
1542 
1543     const Register bits = rsi;
1544     __ mov(bits, from);
1545     __ orptr(bits, to);
1546     __ orptr(bits, count);
1547 
1548     __ testl(bits, BytesPerLong-1);
1549     __ jccb(Assembler::zero, L_long_aligned);
1550 
1551     __ testl(bits, BytesPerInt-1);
1552     __ jccb(Assembler::zero, L_int_aligned);
1553 
1554     __ testl(bits, BytesPerShort-1);
1555     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
1556 
1557     __ BIND(L_short_aligned);
1558     __ shrptr(count, LogBytesPerShort); // size => short_count
1559     __ movl(count_arg, count);          // update 'count'
1560     __ jump(RuntimeAddress(short_copy_entry));
1561 
1562     __ BIND(L_int_aligned);
1563     __ shrptr(count, LogBytesPerInt); // size => int_count
1564     __ movl(count_arg, count);          // update 'count'
1565     __ jump(RuntimeAddress(int_copy_entry));
1566 
1567     __ BIND(L_long_aligned);
1568     __ shrptr(count, LogBytesPerLong); // size => qword_count
1569     __ movl(count_arg, count);          // update 'count'
1570     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1571     __ pop(rsi);
1572     __ jump(RuntimeAddress(long_copy_entry));
1573 
1574     return start;
1575   }
1576 
1577 
1578   // Perform range checks on the proposed arraycopy.
1579   // Smashes src_pos and dst_pos.  (Uses them up for temps.)
1580   void arraycopy_range_checks(Register src,
1581                               Register src_pos,
1582                               Register dst,
1583                               Register dst_pos,
1584                               Address& length,
1585                               Label& L_failed) {
1586     BLOCK_COMMENT("arraycopy_range_checks:");
1587     const Register src_end = src_pos;   // source array end position
1588     const Register dst_end = dst_pos;   // destination array end position
1589     __ addl(src_end, length); // src_pos + length
1590     __ addl(dst_end, length); // dst_pos + length
1591 
1592     //  if (src_pos + length > arrayOop(src)->length() ) FAIL;
1593     __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
1594     __ jcc(Assembler::above, L_failed);
1595 
1596     //  if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
1597     __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1598     __ jcc(Assembler::above, L_failed);
1599 
1600     BLOCK_COMMENT("arraycopy_range_checks done");
1601   }
1602 
1603 
1604   //
1605   //  Generate generic array copy stubs
1606   //
1607   //  Input:
1608   //     4(rsp)    -  src oop
1609   //     8(rsp)    -  src_pos
1610   //    12(rsp)    -  dst oop
1611   //    16(rsp)    -  dst_pos
1612   //    20(rsp)    -  element count
1613   //
1614   //  Output:
1615   //    rax, ==  0  -  success
1616   //    rax, == -1^K - failure, where K is partial transfer count
1617   //
1618   address generate_generic_copy(const char *name,
1619                                 address entry_jbyte_arraycopy,
1620                                 address entry_jshort_arraycopy,
1621                                 address entry_jint_arraycopy,
1622                                 address entry_oop_arraycopy,
1623                                 address entry_jlong_arraycopy,
1624                                 address entry_checkcast_arraycopy) {
1625     Label L_failed, L_failed_0, L_objArray;
1626 
1627     { int modulus = CodeEntryAlignment;
1628       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
1629       int advance = target - (__ offset() % modulus);
1630       if (advance < 0)  advance += modulus;
1631       if (advance > 0)  __ nop(advance);
1632     }
1633     StubCodeMark mark(this, "StubRoutines", name);
1634 
1635     // Short-hop target to L_failed.  Makes for denser prologue code.
1636     __ BIND(L_failed_0);
1637     __ jmp(L_failed);
1638     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
1639 
1640     __ align(CodeEntryAlignment);
1641     address start = __ pc();
1642 
1643     __ enter(); // required for proper stackwalking of RuntimeStub frame
1644     __ push(rsi);
1645     __ push(rdi);
1646 
1647     // bump this on entry, not on exit:
1648     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
1649 
1650     // Input values
1651     Address SRC     (rsp, 12+ 4);
1652     Address SRC_POS (rsp, 12+ 8);
1653     Address DST     (rsp, 12+12);
1654     Address DST_POS (rsp, 12+16);
1655     Address LENGTH  (rsp, 12+20);
1656 
1657     //-----------------------------------------------------------------------
1658     // Assembler stub will be used for this call to arraycopy
1659     // if the following conditions are met:
1660     //
1661     // (1) src and dst must not be null.
1662     // (2) src_pos must not be negative.
1663     // (3) dst_pos must not be negative.
1664     // (4) length  must not be negative.
1665     // (5) src klass and dst klass should be the same and not NULL.
1666     // (6) src and dst should be arrays.
1667     // (7) src_pos + length must not exceed length of src.
1668     // (8) dst_pos + length must not exceed length of dst.
1669     //
1670 
1671     const Register src     = rax;       // source array oop
1672     const Register src_pos = rsi;
1673     const Register dst     = rdx;       // destination array oop
1674     const Register dst_pos = rdi;
1675     const Register length  = rcx;       // transfer count
1676 
1677     //  if (src == NULL) return -1;
1678     __ movptr(src, SRC);      // src oop
1679     __ testptr(src, src);
1680     __ jccb(Assembler::zero, L_failed_0);
1681 
1682     //  if (src_pos < 0) return -1;
1683     __ movl2ptr(src_pos, SRC_POS);  // src_pos
1684     __ testl(src_pos, src_pos);
1685     __ jccb(Assembler::negative, L_failed_0);
1686 
1687     //  if (dst == NULL) return -1;
1688     __ movptr(dst, DST);      // dst oop
1689     __ testptr(dst, dst);
1690     __ jccb(Assembler::zero, L_failed_0);
1691 
1692     //  if (dst_pos < 0) return -1;
1693     __ movl2ptr(dst_pos, DST_POS);  // dst_pos
1694     __ testl(dst_pos, dst_pos);
1695     __ jccb(Assembler::negative, L_failed_0);
1696 
1697     //  if (length < 0) return -1;
1698     __ movl2ptr(length, LENGTH);   // length
1699     __ testl(length, length);
1700     __ jccb(Assembler::negative, L_failed_0);
1701 
1702     //  if (src->klass() == NULL) return -1;
1703     Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
1704     Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
1705     const Register rcx_src_klass = rcx;    // array klass
1706     __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
1707 
1708 #ifdef ASSERT
1709     //  assert(src->klass() != NULL);
1710     BLOCK_COMMENT("assert klasses not null");
1711     { Label L1, L2;
1712       __ testptr(rcx_src_klass, rcx_src_klass);
1713       __ jccb(Assembler::notZero, L2);   // it is broken if klass is NULL
1714       __ bind(L1);
1715       __ stop("broken null klass");
1716       __ bind(L2);
1717       __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
1718       __ jccb(Assembler::equal, L1);      // this would be broken also
1719       BLOCK_COMMENT("assert done");
1720     }
1721 #endif //ASSERT
1722 
1723     // Load layout helper (32-bits)
1724     //
1725     //  |array_tag|     | header_size | element_type |     |log2_element_size|
1726     // 32        30    24            16              8     2                 0
1727     //
1728     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
1729     //
1730 
1731     int lh_offset = in_bytes(Klass::layout_helper_offset());
1732     Address src_klass_lh_addr(rcx_src_klass, lh_offset);
1733 
1734     // Handle objArrays completely differently...
1735     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
1736     __ cmpl(src_klass_lh_addr, objArray_lh);
1737     __ jcc(Assembler::equal, L_objArray);
1738 
1739     //  if (src->klass() != dst->klass()) return -1;
1740     __ cmpptr(rcx_src_klass, dst_klass_addr);
1741     __ jccb(Assembler::notEqual, L_failed_0);
1742 
1743     const Register rcx_lh = rcx;  // layout helper
1744     assert(rcx_lh == rcx_src_klass, "known alias");
1745     __ movl(rcx_lh, src_klass_lh_addr);
1746 
1747     //  if (!src->is_Array()) return -1;
1748     __ cmpl(rcx_lh, Klass::_lh_neutral_value);
1749     __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
1750 
1751     // At this point, it is known to be a typeArray (array_tag 0x3).
1752 #ifdef ASSERT
1753     { Label L;
1754       __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
1755       __ jcc(Assembler::greaterEqual, L); // signed cmp
1756       __ stop("must be a primitive array");
1757       __ bind(L);
1758     }
1759 #endif
1760 
1761     assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
1762     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1763 
1764     // TypeArrayKlass
1765     //
1766     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
1767     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
1768     //
1769     const Register rsi_offset = rsi; // array offset
1770     const Register src_array  = src; // src array offset
1771     const Register dst_array  = dst; // dst array offset
1772     const Register rdi_elsize = rdi; // log2 element size
1773 
1774     __ mov(rsi_offset, rcx_lh);
1775     __ shrptr(rsi_offset, Klass::_lh_header_size_shift);
1776     __ andptr(rsi_offset, Klass::_lh_header_size_mask);   // array_offset
1777     __ addptr(src_array, rsi_offset);  // src array offset
1778     __ addptr(dst_array, rsi_offset);  // dst array offset
1779     __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
1780 
1781     // next registers should be set before the jump to corresponding stub
1782     const Register from       = src; // source array address
1783     const Register to         = dst; // destination array address
1784     const Register count      = rcx; // elements count
1785     // some of them should be duplicated on stack
1786 #define FROM   Address(rsp, 12+ 4)
1787 #define TO     Address(rsp, 12+ 8)   // Not used now
1788 #define COUNT  Address(rsp, 12+12)   // Only for oop arraycopy
1789 
1790     BLOCK_COMMENT("scale indexes to element size");
1791     __ movl2ptr(rsi, SRC_POS);  // src_pos
1792     __ shlptr(rsi);             // src_pos << rcx (log2 elsize)
1793     assert(src_array == from, "");
1794     __ addptr(from, rsi);       // from = src_array + SRC_POS << log2 elsize
1795     __ movl2ptr(rdi, DST_POS);  // dst_pos
1796     __ shlptr(rdi);             // dst_pos << rcx (log2 elsize)
1797     assert(dst_array == to, "");
1798     __ addptr(to,  rdi);        // to   = dst_array + DST_POS << log2 elsize
1799     __ movptr(FROM, from);      // src_addr
1800     __ mov(rdi_elsize, rcx_lh); // log2 elsize
1801     __ movl2ptr(count, LENGTH); // elements count
1802 
1803     BLOCK_COMMENT("choose copy loop based on element size");
1804     __ cmpl(rdi_elsize, 0);
1805 
1806     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
1807     __ cmpl(rdi_elsize, LogBytesPerShort);
1808     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
1809     __ cmpl(rdi_elsize, LogBytesPerInt);
1810     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
1811 #ifdef ASSERT
1812     __ cmpl(rdi_elsize, LogBytesPerLong);
1813     __ jccb(Assembler::notEqual, L_failed);
1814 #endif
1815     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1816     __ pop(rsi);
1817     __ jump(RuntimeAddress(entry_jlong_arraycopy));
1818 
1819   __ BIND(L_failed);
1820     __ xorptr(rax, rax);
1821     __ notptr(rax); // return -1
1822     __ pop(rdi);
1823     __ pop(rsi);
1824     __ leave(); // required for proper stackwalking of RuntimeStub frame
1825     __ ret(0);
1826 
1827     // ObjArrayKlass
1828   __ BIND(L_objArray);
1829     // live at this point:  rcx_src_klass, src[_pos], dst[_pos]
1830 
1831     Label L_plain_copy, L_checkcast_copy;
1832     //  test array classes for subtyping
1833     __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
1834     __ jccb(Assembler::notEqual, L_checkcast_copy);
1835 
1836     // Identically typed arrays can be copied without element-wise checks.
1837     assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
1838     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1839 
1840   __ BIND(L_plain_copy);
1841     __ movl2ptr(count, LENGTH); // elements count
1842     __ movl2ptr(src_pos, SRC_POS);  // reload src_pos
1843     __ lea(from, Address(src, src_pos, Address::times_ptr,
1844                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
1845     __ movl2ptr(dst_pos, DST_POS);  // reload dst_pos
1846     __ lea(to,   Address(dst, dst_pos, Address::times_ptr,
1847                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
1848     __ movptr(FROM,  from);   // src_addr
1849     __ movptr(TO,    to);     // dst_addr
1850     __ movl(COUNT, count);  // count
1851     __ jump(RuntimeAddress(entry_oop_arraycopy));
1852 
1853   __ BIND(L_checkcast_copy);
1854     // live at this point:  rcx_src_klass, dst[_pos], src[_pos]
1855     {
1856       // Handy offsets:
1857       int  ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
1858       int sco_offset = in_bytes(Klass::super_check_offset_offset());
1859 
1860       Register rsi_dst_klass = rsi;
1861       Register rdi_temp      = rdi;
1862       assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
1863       assert(rdi_temp      == dst_pos, "expected alias w/ dst_pos");
1864       Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
1865 
1866       // Before looking at dst.length, make sure dst is also an objArray.
1867       __ movptr(rsi_dst_klass, dst_klass_addr);
1868       __ cmpl(dst_klass_lh_addr, objArray_lh);
1869       __ jccb(Assembler::notEqual, L_failed);
1870 
1871       // It is safe to examine both src.length and dst.length.
1872       __ movl2ptr(src_pos, SRC_POS);        // reload rsi
1873       arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1874       // (Now src_pos and dst_pos are killed, but not src and dst.)
1875 
1876       // We'll need this temp (don't forget to pop it after the type check).
1877       __ push(rbx);
1878       Register rbx_src_klass = rbx;
1879 
1880       __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
1881       __ movptr(rsi_dst_klass, dst_klass_addr);
1882       Address super_check_offset_addr(rsi_dst_klass, sco_offset);
1883       Label L_fail_array_check;
1884       generate_type_check(rbx_src_klass,
1885                           super_check_offset_addr, dst_klass_addr,
1886                           rdi_temp, NULL, &L_fail_array_check);
1887       // (On fall-through, we have passed the array type check.)
1888       __ pop(rbx);
1889       __ jmp(L_plain_copy);
1890 
1891       __ BIND(L_fail_array_check);
1892       // Reshuffle arguments so we can call checkcast_arraycopy:
1893 
1894       // match initial saves for checkcast_arraycopy
1895       // push(rsi);    // already done; see above
1896       // push(rdi);    // already done; see above
1897       // push(rbx);    // already done; see above
1898 
1899       // Marshal outgoing arguments now, freeing registers.
1900       Address   from_arg(rsp, 16+ 4);   // from
1901       Address     to_arg(rsp, 16+ 8);   // to
1902       Address length_arg(rsp, 16+12);   // elements count
1903       Address  ckoff_arg(rsp, 16+16);   // super_check_offset
1904       Address  ckval_arg(rsp, 16+20);   // super_klass
1905 
1906       Address SRC_POS_arg(rsp, 16+ 8);
1907       Address DST_POS_arg(rsp, 16+16);
1908       Address  LENGTH_arg(rsp, 16+20);
1909       // push rbx, changed the incoming offsets (why not just use rbp,??)
1910       // assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
1911 
1912       __ movptr(rbx, Address(rsi_dst_klass, ek_offset));
1913       __ movl2ptr(length, LENGTH_arg);    // reload elements count
1914       __ movl2ptr(src_pos, SRC_POS_arg);  // reload src_pos
1915       __ movl2ptr(dst_pos, DST_POS_arg);  // reload dst_pos
1916 
1917       __ movptr(ckval_arg, rbx);          // destination element type
1918       __ movl(rbx, Address(rbx, sco_offset));
1919       __ movl(ckoff_arg, rbx);          // corresponding class check offset
1920 
1921       __ movl(length_arg, length);      // outgoing length argument
1922 
1923       __ lea(from, Address(src, src_pos, Address::times_ptr,
1924                             arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1925       __ movptr(from_arg, from);
1926 
1927       __ lea(to, Address(dst, dst_pos, Address::times_ptr,
1928                           arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1929       __ movptr(to_arg, to);
1930       __ jump(RuntimeAddress(entry_checkcast_arraycopy));
1931     }
1932 
1933     return start;
1934   }
1935 
1936   void generate_arraycopy_stubs() {
1937     address entry;
1938     address entry_jbyte_arraycopy;
1939     address entry_jshort_arraycopy;
1940     address entry_jint_arraycopy;
1941     address entry_oop_arraycopy;
1942     address entry_jlong_arraycopy;
1943     address entry_checkcast_arraycopy;
1944 
1945     StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
1946         generate_disjoint_copy(T_BYTE,  true, Address::times_1, &entry,
1947                                "arrayof_jbyte_disjoint_arraycopy");
1948     StubRoutines::_arrayof_jbyte_arraycopy =
1949         generate_conjoint_copy(T_BYTE,  true, Address::times_1,  entry,
1950                                NULL, "arrayof_jbyte_arraycopy");
1951     StubRoutines::_jbyte_disjoint_arraycopy =
1952         generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
1953                                "jbyte_disjoint_arraycopy");
1954     StubRoutines::_jbyte_arraycopy =
1955         generate_conjoint_copy(T_BYTE, false, Address::times_1,  entry,
1956                                &entry_jbyte_arraycopy, "jbyte_arraycopy");
1957 
1958     StubRoutines::_arrayof_jshort_disjoint_arraycopy =
1959         generate_disjoint_copy(T_SHORT,  true, Address::times_2, &entry,
1960                                "arrayof_jshort_disjoint_arraycopy");
1961     StubRoutines::_arrayof_jshort_arraycopy =
1962         generate_conjoint_copy(T_SHORT,  true, Address::times_2,  entry,
1963                                NULL, "arrayof_jshort_arraycopy");
1964     StubRoutines::_jshort_disjoint_arraycopy =
1965         generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
1966                                "jshort_disjoint_arraycopy");
1967     StubRoutines::_jshort_arraycopy =
1968         generate_conjoint_copy(T_SHORT, false, Address::times_2,  entry,
1969                                &entry_jshort_arraycopy, "jshort_arraycopy");
1970 
1971     // Next arrays are always aligned on 4 bytes at least.
1972     StubRoutines::_jint_disjoint_arraycopy =
1973         generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
1974                                "jint_disjoint_arraycopy");
1975     StubRoutines::_jint_arraycopy =
1976         generate_conjoint_copy(T_INT, true, Address::times_4,  entry,
1977                                &entry_jint_arraycopy, "jint_arraycopy");
1978 
1979     StubRoutines::_oop_disjoint_arraycopy =
1980         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
1981                                "oop_disjoint_arraycopy");
1982     StubRoutines::_oop_arraycopy =
1983         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
1984                                &entry_oop_arraycopy, "oop_arraycopy");
1985 
1986     StubRoutines::_oop_disjoint_arraycopy_uninit =
1987         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
1988                                "oop_disjoint_arraycopy_uninit",
1989                                /*dest_uninitialized*/true);
1990     StubRoutines::_oop_arraycopy_uninit =
1991         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
1992                                NULL, "oop_arraycopy_uninit",
1993                                /*dest_uninitialized*/true);
1994 
1995     StubRoutines::_jlong_disjoint_arraycopy =
1996         generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
1997     StubRoutines::_jlong_arraycopy =
1998         generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
1999                                     "jlong_arraycopy");
2000 
2001     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2002     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2003     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2004     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2005     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2006     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2007 
2008     StubRoutines::_arrayof_jint_disjoint_arraycopy       = StubRoutines::_jint_disjoint_arraycopy;
2009     StubRoutines::_arrayof_oop_disjoint_arraycopy        = StubRoutines::_oop_disjoint_arraycopy;
2010     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;
2011     StubRoutines::_arrayof_jlong_disjoint_arraycopy      = StubRoutines::_jlong_disjoint_arraycopy;
2012 
2013     StubRoutines::_arrayof_jint_arraycopy       = StubRoutines::_jint_arraycopy;
2014     StubRoutines::_arrayof_oop_arraycopy        = StubRoutines::_oop_arraycopy;
2015     StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;
2016     StubRoutines::_arrayof_jlong_arraycopy      = StubRoutines::_jlong_arraycopy;
2017 
2018     StubRoutines::_checkcast_arraycopy =
2019         generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2020     StubRoutines::_checkcast_arraycopy_uninit =
2021         generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, /*dest_uninitialized*/true);
2022 
2023     StubRoutines::_unsafe_arraycopy =
2024         generate_unsafe_copy("unsafe_arraycopy",
2025                                entry_jbyte_arraycopy,
2026                                entry_jshort_arraycopy,
2027                                entry_jint_arraycopy,
2028                                entry_jlong_arraycopy);
2029 
2030     StubRoutines::_generic_arraycopy =
2031         generate_generic_copy("generic_arraycopy",
2032                                entry_jbyte_arraycopy,
2033                                entry_jshort_arraycopy,
2034                                entry_jint_arraycopy,
2035                                entry_oop_arraycopy,
2036                                entry_jlong_arraycopy,
2037                                entry_checkcast_arraycopy);
2038   }
2039 
2040   // AES intrinsic stubs
2041   enum {AESBlockSize = 16};
2042 
2043   address generate_key_shuffle_mask() {
2044     __ align(16);
2045     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
2046     address start = __ pc();
2047     __ emit_data(0x00010203, relocInfo::none, 0 );
2048     __ emit_data(0x04050607, relocInfo::none, 0 );
2049     __ emit_data(0x08090a0b, relocInfo::none, 0 );
2050     __ emit_data(0x0c0d0e0f, relocInfo::none, 0 );
2051     return start;
2052   }
2053 
2054   address generate_counter_shuffle_mask() {
2055     __ align(16);
2056     StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask");
2057     address start = __ pc();
2058     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
2059     __ emit_data(0x08090a0b, relocInfo::none, 0);
2060     __ emit_data(0x04050607, relocInfo::none, 0);
2061     __ emit_data(0x00010203, relocInfo::none, 0);
2062     return start;
2063   }
2064 
2065   // Utility routine for loading a 128-bit key word in little endian format
2066   // can optionally specify that the shuffle mask is already in an xmmregister
2067   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2068     __ movdqu(xmmdst, Address(key, offset));
2069     if (xmm_shuf_mask != NULL) {
2070       __ pshufb(xmmdst, xmm_shuf_mask);
2071     } else {
2072       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2073     }
2074   }
2075 
2076   // aesenc using specified key+offset
2077   // can optionally specify that the shuffle mask is already in an xmmregister
2078   void aes_enc_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2079     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2080     __ aesenc(xmmdst, xmmtmp);
2081   }
2082 
2083   // aesdec using specified key+offset
2084   // can optionally specify that the shuffle mask is already in an xmmregister
2085   void aes_dec_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2086     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2087     __ aesdec(xmmdst, xmmtmp);
2088   }
2089 
2090   // Utility routine for increase 128bit counter (iv in CTR mode)
2091   //  XMM_128bit,  D3, D2, D1, D0
2092   void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) {
2093     __ pextrd(reg, xmmdst, 0x0);
2094     __ addl(reg, inc_delta);
2095     __ pinsrd(xmmdst, reg, 0x0);
2096     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2097 
2098     __ pextrd(reg, xmmdst, 0x01); // Carry-> D1
2099     __ addl(reg, 0x01);
2100     __ pinsrd(xmmdst, reg, 0x01);
2101     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2102 
2103     __ pextrd(reg, xmmdst, 0x02); // Carry-> D2
2104     __ addl(reg, 0x01);
2105     __ pinsrd(xmmdst, reg, 0x02);
2106     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2107 
2108     __ pextrd(reg, xmmdst, 0x03); // Carry -> D3
2109     __ addl(reg, 0x01);
2110     __ pinsrd(xmmdst, reg, 0x03);
2111 
2112     __ BIND(next_block);          // next instruction
2113   }
2114 
2115 
2116   // Arguments:
2117   //
2118   // Inputs:
2119   //   c_rarg0   - source byte array address
2120   //   c_rarg1   - destination byte array address
2121   //   c_rarg2   - K (key) in little endian int array
2122   //
2123   address generate_aescrypt_encryptBlock() {
2124     assert(UseAES, "need AES instructions and misaligned SSE support");
2125     __ align(CodeEntryAlignment);
2126     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2127     Label L_doLast;
2128     address start = __ pc();
2129 
2130     const Register from        = rdx;      // source array address
2131     const Register to          = rdx;      // destination array address
2132     const Register key         = rcx;      // key array address
2133     const Register keylen      = rax;
2134     const Address  from_param(rbp, 8+0);
2135     const Address  to_param  (rbp, 8+4);
2136     const Address  key_param (rbp, 8+8);
2137 
2138     const XMMRegister xmm_result = xmm0;
2139     const XMMRegister xmm_key_shuf_mask = xmm1;
2140     const XMMRegister xmm_temp1  = xmm2;
2141     const XMMRegister xmm_temp2  = xmm3;
2142     const XMMRegister xmm_temp3  = xmm4;
2143     const XMMRegister xmm_temp4  = xmm5;
2144 
2145     __ enter();   // required for proper stackwalking of RuntimeStub frame
2146 
2147     __ movptr(from, from_param);
2148     __ movptr(key, key_param);
2149 
2150     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2151     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2152 
2153     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2154     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
2155     __ movptr(to, to_param);
2156 
2157     // For encryption, the java expanded key ordering is just what we need
2158 
2159     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
2160     __ pxor(xmm_result, xmm_temp1);
2161 
2162     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2163     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2164     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2165     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2166 
2167     __ aesenc(xmm_result, xmm_temp1);
2168     __ aesenc(xmm_result, xmm_temp2);
2169     __ aesenc(xmm_result, xmm_temp3);
2170     __ aesenc(xmm_result, xmm_temp4);
2171 
2172     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2173     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2174     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2175     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2176 
2177     __ aesenc(xmm_result, xmm_temp1);
2178     __ aesenc(xmm_result, xmm_temp2);
2179     __ aesenc(xmm_result, xmm_temp3);
2180     __ aesenc(xmm_result, xmm_temp4);
2181 
2182     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2183     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2184 
2185     __ cmpl(keylen, 44);
2186     __ jccb(Assembler::equal, L_doLast);
2187 
2188     __ aesenc(xmm_result, xmm_temp1);
2189     __ aesenc(xmm_result, xmm_temp2);
2190 
2191     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2192     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2193 
2194     __ cmpl(keylen, 52);
2195     __ jccb(Assembler::equal, L_doLast);
2196 
2197     __ aesenc(xmm_result, xmm_temp1);
2198     __ aesenc(xmm_result, xmm_temp2);
2199 
2200     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2201     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2202 
2203     __ BIND(L_doLast);
2204     __ aesenc(xmm_result, xmm_temp1);
2205     __ aesenclast(xmm_result, xmm_temp2);
2206     __ movdqu(Address(to, 0), xmm_result);        // store the result
2207     __ xorptr(rax, rax); // return 0
2208     __ leave(); // required for proper stackwalking of RuntimeStub frame
2209     __ ret(0);
2210 
2211     return start;
2212   }
2213 
2214 
2215   // Arguments:
2216   //
2217   // Inputs:
2218   //   c_rarg0   - source byte array address
2219   //   c_rarg1   - destination byte array address
2220   //   c_rarg2   - K (key) in little endian int array
2221   //
2222   address generate_aescrypt_decryptBlock() {
2223     assert(UseAES, "need AES instructions and misaligned SSE support");
2224     __ align(CodeEntryAlignment);
2225     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
2226     Label L_doLast;
2227     address start = __ pc();
2228 
2229     const Register from        = rdx;      // source array address
2230     const Register to          = rdx;      // destination array address
2231     const Register key         = rcx;      // key array address
2232     const Register keylen      = rax;
2233     const Address  from_param(rbp, 8+0);
2234     const Address  to_param  (rbp, 8+4);
2235     const Address  key_param (rbp, 8+8);
2236 
2237     const XMMRegister xmm_result = xmm0;
2238     const XMMRegister xmm_key_shuf_mask = xmm1;
2239     const XMMRegister xmm_temp1  = xmm2;
2240     const XMMRegister xmm_temp2  = xmm3;
2241     const XMMRegister xmm_temp3  = xmm4;
2242     const XMMRegister xmm_temp4  = xmm5;
2243 
2244     __ enter(); // required for proper stackwalking of RuntimeStub frame
2245 
2246     __ movptr(from, from_param);
2247     __ movptr(key, key_param);
2248 
2249     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2250     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2251 
2252     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2253     __ movdqu(xmm_result, Address(from, 0));
2254     __ movptr(to, to_param);
2255 
2256     // for decryption java expanded key ordering is rotated one position from what we want
2257     // so we start from 0x10 here and hit 0x00 last
2258     // we don't know if the key is aligned, hence not using load-execute form
2259     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2260     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2261     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2262     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2263 
2264     __ pxor  (xmm_result, xmm_temp1);
2265     __ aesdec(xmm_result, xmm_temp2);
2266     __ aesdec(xmm_result, xmm_temp3);
2267     __ aesdec(xmm_result, xmm_temp4);
2268 
2269     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2270     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2271     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2272     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2273 
2274     __ aesdec(xmm_result, xmm_temp1);
2275     __ aesdec(xmm_result, xmm_temp2);
2276     __ aesdec(xmm_result, xmm_temp3);
2277     __ aesdec(xmm_result, xmm_temp4);
2278 
2279     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2280     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2281     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
2282 
2283     __ cmpl(keylen, 44);
2284     __ jccb(Assembler::equal, L_doLast);
2285 
2286     __ aesdec(xmm_result, xmm_temp1);
2287     __ aesdec(xmm_result, xmm_temp2);
2288 
2289     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2290     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2291 
2292     __ cmpl(keylen, 52);
2293     __ jccb(Assembler::equal, L_doLast);
2294 
2295     __ aesdec(xmm_result, xmm_temp1);
2296     __ aesdec(xmm_result, xmm_temp2);
2297 
2298     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2299     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2300 
2301     __ BIND(L_doLast);
2302     __ aesdec(xmm_result, xmm_temp1);
2303     __ aesdec(xmm_result, xmm_temp2);
2304 
2305     // for decryption the aesdeclast operation is always on key+0x00
2306     __ aesdeclast(xmm_result, xmm_temp3);
2307     __ movdqu(Address(to, 0), xmm_result);  // store the result
2308     __ xorptr(rax, rax); // return 0
2309     __ leave(); // required for proper stackwalking of RuntimeStub frame
2310     __ ret(0);
2311 
2312     return start;
2313   }
2314 
2315   void handleSOERegisters(bool saving) {
2316     const int saveFrameSizeInBytes = 4 * wordSize;
2317     const Address saved_rbx     (rbp, -3 * wordSize);
2318     const Address saved_rsi     (rbp, -2 * wordSize);
2319     const Address saved_rdi     (rbp, -1 * wordSize);
2320 
2321     if (saving) {
2322       __ subptr(rsp, saveFrameSizeInBytes);
2323       __ movptr(saved_rsi, rsi);
2324       __ movptr(saved_rdi, rdi);
2325       __ movptr(saved_rbx, rbx);
2326     } else {
2327       // restoring
2328       __ movptr(rsi, saved_rsi);
2329       __ movptr(rdi, saved_rdi);
2330       __ movptr(rbx, saved_rbx);
2331     }
2332   }
2333 
2334   // Arguments:
2335   //
2336   // Inputs:
2337   //   c_rarg0   - source byte array address
2338   //   c_rarg1   - destination byte array address
2339   //   c_rarg2   - K (key) in little endian int array
2340   //   c_rarg3   - r vector byte array address
2341   //   c_rarg4   - input length
2342   //
2343   // Output:
2344   //   rax       - input length
2345   //
2346   address generate_cipherBlockChaining_encryptAESCrypt() {
2347     assert(UseAES, "need AES instructions and misaligned SSE support");
2348     __ align(CodeEntryAlignment);
2349     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
2350     address start = __ pc();
2351 
2352     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
2353     const Register from        = rsi;      // source array address
2354     const Register to          = rdx;      // destination array address
2355     const Register key         = rcx;      // key array address
2356     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2357                                            // and left with the results of the last encryption block
2358     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2359     const Register pos         = rax;
2360 
2361     // xmm register assignments for the loops below
2362     const XMMRegister xmm_result = xmm0;
2363     const XMMRegister xmm_temp   = xmm1;
2364     // first 6 keys preloaded into xmm2-xmm7
2365     const int XMM_REG_NUM_KEY_FIRST = 2;
2366     const int XMM_REG_NUM_KEY_LAST  = 7;
2367     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
2368 
2369     __ enter(); // required for proper stackwalking of RuntimeStub frame
2370     handleSOERegisters(true /*saving*/);
2371 
2372     // load registers from incoming parameters
2373     const Address  from_param(rbp, 8+0);
2374     const Address  to_param  (rbp, 8+4);
2375     const Address  key_param (rbp, 8+8);
2376     const Address  rvec_param (rbp, 8+12);
2377     const Address  len_param  (rbp, 8+16);
2378     __ movptr(from , from_param);
2379     __ movptr(to   , to_param);
2380     __ movptr(key  , key_param);
2381     __ movptr(rvec , rvec_param);
2382     __ movptr(len_reg , len_param);
2383 
2384     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
2385     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2386     // load up xmm regs 2 thru 7 with keys 0-5
2387     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2388       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
2389       offset += 0x10;
2390     }
2391 
2392     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
2393 
2394     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2395     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2396     __ cmpl(rax, 44);
2397     __ jcc(Assembler::notEqual, L_key_192_256);
2398 
2399     // 128 bit code follows here
2400     __ movl(pos, 0);
2401     __ align(OptoLoopAlignment);
2402     __ BIND(L_loopTop_128);
2403     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2404     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2405 
2406     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2407     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2408       __ aesenc(xmm_result, as_XMMRegister(rnum));
2409     }
2410     for (int key_offset = 0x60; key_offset <= 0x90; key_offset += 0x10) {
2411       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2412     }
2413     load_key(xmm_temp, key, 0xa0);
2414     __ aesenclast(xmm_result, xmm_temp);
2415 
2416     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
2417     // no need to store r to memory until we exit
2418     __ addptr(pos, AESBlockSize);
2419     __ subptr(len_reg, AESBlockSize);
2420     __ jcc(Assembler::notEqual, L_loopTop_128);
2421 
2422     __ BIND(L_exit);
2423     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
2424 
2425     handleSOERegisters(false /*restoring*/);
2426     __ movptr(rax, len_param); // return length
2427     __ leave();                                  // required for proper stackwalking of RuntimeStub frame
2428     __ ret(0);
2429 
2430     __ BIND(L_key_192_256);
2431     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
2432     __ cmpl(rax, 52);
2433     __ jcc(Assembler::notEqual, L_key_256);
2434 
2435     // 192-bit code follows here (could be changed to use more xmm registers)
2436     __ movl(pos, 0);
2437     __ align(OptoLoopAlignment);
2438     __ BIND(L_loopTop_192);
2439     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2440     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2441 
2442     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2443     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2444       __ aesenc(xmm_result, as_XMMRegister(rnum));
2445     }
2446     for (int key_offset = 0x60; key_offset <= 0xb0; key_offset += 0x10) {
2447       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2448     }
2449     load_key(xmm_temp, key, 0xc0);
2450     __ aesenclast(xmm_result, xmm_temp);
2451 
2452     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2453     // no need to store r to memory until we exit
2454     __ addptr(pos, AESBlockSize);
2455     __ subptr(len_reg, AESBlockSize);
2456     __ jcc(Assembler::notEqual, L_loopTop_192);
2457     __ jmp(L_exit);
2458 
2459     __ BIND(L_key_256);
2460     // 256-bit code follows here (could be changed to use more xmm registers)
2461     __ movl(pos, 0);
2462     __ align(OptoLoopAlignment);
2463     __ BIND(L_loopTop_256);
2464     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2465     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2466 
2467     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2468     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2469       __ aesenc(xmm_result, as_XMMRegister(rnum));
2470     }
2471     for (int key_offset = 0x60; key_offset <= 0xd0; key_offset += 0x10) {
2472       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2473     }
2474     load_key(xmm_temp, key, 0xe0);
2475     __ aesenclast(xmm_result, xmm_temp);
2476 
2477     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2478     // no need to store r to memory until we exit
2479     __ addptr(pos, AESBlockSize);
2480     __ subptr(len_reg, AESBlockSize);
2481     __ jcc(Assembler::notEqual, L_loopTop_256);
2482     __ jmp(L_exit);
2483 
2484     return start;
2485   }
2486 
2487 
2488   // CBC AES Decryption.
2489   // In 32-bit stub, because of lack of registers we do not try to parallelize 4 blocks at a time.
2490   //
2491   // Arguments:
2492   //
2493   // Inputs:
2494   //   c_rarg0   - source byte array address
2495   //   c_rarg1   - destination byte array address
2496   //   c_rarg2   - K (key) in little endian int array
2497   //   c_rarg3   - r vector byte array address
2498   //   c_rarg4   - input length
2499   //
2500   // Output:
2501   //   rax       - input length
2502   //
2503 
2504   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
2505     assert(UseAES, "need AES instructions and misaligned SSE support");
2506     __ align(CodeEntryAlignment);
2507     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
2508     address start = __ pc();
2509 
2510     const Register from        = rsi;      // source array address
2511     const Register to          = rdx;      // destination array address
2512     const Register key         = rcx;      // key array address
2513     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2514                                            // and left with the results of the last encryption block
2515     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2516     const Register pos         = rax;
2517 
2518     const int PARALLEL_FACTOR = 4;
2519     const int ROUNDS[3] = { 10, 12, 14 }; //aes rounds for key128, key192, key256
2520 
2521     Label L_exit;
2522     Label L_singleBlock_loopTop[3]; //128, 192, 256
2523     Label L_multiBlock_loopTop[3]; //128, 192, 256
2524 
2525     const XMMRegister xmm_prev_block_cipher = xmm0; // holds cipher of previous block
2526     const XMMRegister xmm_key_shuf_mask = xmm1;
2527 
2528     const XMMRegister xmm_key_tmp0 = xmm2;
2529     const XMMRegister xmm_key_tmp1 = xmm3;
2530 
2531     // registers holding the six results in the parallelized loop
2532     const XMMRegister xmm_result0 = xmm4;
2533     const XMMRegister xmm_result1 = xmm5;
2534     const XMMRegister xmm_result2 = xmm6;
2535     const XMMRegister xmm_result3 = xmm7;
2536 
2537     __ enter(); // required for proper stackwalking of RuntimeStub frame
2538     handleSOERegisters(true /*saving*/);
2539 
2540     // load registers from incoming parameters
2541     const Address  from_param(rbp, 8+0);
2542     const Address  to_param  (rbp, 8+4);
2543     const Address  key_param (rbp, 8+8);
2544     const Address  rvec_param (rbp, 8+12);
2545     const Address  len_param  (rbp, 8+16);
2546 
2547     __ movptr(from , from_param);
2548     __ movptr(to   , to_param);
2549     __ movptr(key  , key_param);
2550     __ movptr(rvec , rvec_param);
2551     __ movptr(len_reg , len_param);
2552 
2553     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2554     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // initialize with initial rvec
2555 
2556     __ xorptr(pos, pos);
2557 
2558     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2559     // rvec is reused
2560     __ movl(rvec, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2561     __ cmpl(rvec, 52);
2562     __ jcc(Assembler::equal, L_multiBlock_loopTop[1]);
2563     __ cmpl(rvec, 60);
2564     __ jcc(Assembler::equal, L_multiBlock_loopTop[2]);
2565 
2566 #define DoFour(opc, src_reg)           \
2567   __ opc(xmm_result0, src_reg);         \
2568   __ opc(xmm_result1, src_reg);         \
2569   __ opc(xmm_result2, src_reg);         \
2570   __ opc(xmm_result3, src_reg);         \
2571 
2572     for (int k = 0; k < 3; ++k) {
2573       __ align(OptoLoopAlignment);
2574       __ BIND(L_multiBlock_loopTop[k]);
2575       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
2576       __ jcc(Assembler::less, L_singleBlock_loopTop[k]);
2577 
2578       __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers
2579       __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2580       __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2581       __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2582 
2583       // the java expanded key ordering is rotated one position from what we want
2584       // so we start from 0x10 here and hit 0x00 last
2585       load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2586       DoFour(pxor, xmm_key_tmp0); //xor with first key
2587       // do the aes dec rounds
2588       for (int rnum = 1; rnum <= ROUNDS[k];) {
2589         //load two keys at a time
2590         //k1->0x20, ..., k9->0xa0, k10->0x00
2591         load_key(xmm_key_tmp1, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2592         load_key(xmm_key_tmp0, key, ((rnum + 2) % (ROUNDS[k] + 1)) * 0x10, xmm_key_shuf_mask); // hit 0x00 last!
2593         DoFour(aesdec, xmm_key_tmp1);
2594         rnum++;
2595         if (rnum != ROUNDS[k]) {
2596           DoFour(aesdec, xmm_key_tmp0);
2597         }
2598         else {
2599           DoFour(aesdeclast, xmm_key_tmp0);
2600         }
2601         rnum++;
2602       }
2603 
2604       // for each result, xor with the r vector of previous cipher block
2605       __ pxor(xmm_result0, xmm_prev_block_cipher);
2606       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2607       __ pxor(xmm_result1, xmm_prev_block_cipher);
2608       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2609       __ pxor(xmm_result2, xmm_prev_block_cipher);
2610       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2611       __ pxor(xmm_result3, xmm_prev_block_cipher);
2612       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize)); // this will carry over to next set of blocks
2613 
2614             // store 4 results into the next 64 bytes of output
2615        __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2616        __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2617        __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2618        __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2619 
2620        __ addptr(pos, 4 * AESBlockSize);
2621        __ subptr(len_reg, 4 * AESBlockSize);
2622        __ jmp(L_multiBlock_loopTop[k]);
2623 
2624        //singleBlock starts here
2625        __ align(OptoLoopAlignment);
2626        __ BIND(L_singleBlock_loopTop[k]);
2627        __ cmpptr(len_reg, 0); // any blocks left?
2628        __ jcc(Assembler::equal, L_exit);
2629        __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
2630        __ movdqa(xmm_result1, xmm_result0);
2631 
2632        load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2633        __ pxor(xmm_result0, xmm_key_tmp0);
2634        // do the aes dec rounds
2635        for (int rnum = 1; rnum < ROUNDS[k]; rnum++) {
2636          // the java expanded key ordering is rotated one position from what we want
2637          load_key(xmm_key_tmp0, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2638          __ aesdec(xmm_result0, xmm_key_tmp0);
2639        }
2640        load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
2641        __ aesdeclast(xmm_result0, xmm_key_tmp0);
2642        __ pxor(xmm_result0, xmm_prev_block_cipher); // xor with the current r vector
2643        __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result0); // store into the next 16 bytes of output
2644        // no need to store r to memory until we exit
2645        __ movdqa(xmm_prev_block_cipher, xmm_result1); // set up next r vector with cipher input from this block
2646 
2647        __ addptr(pos, AESBlockSize);
2648        __ subptr(len_reg, AESBlockSize);
2649        __ jmp(L_singleBlock_loopTop[k]);
2650     }//for 128/192/256
2651 
2652     __ BIND(L_exit);
2653     __ movptr(rvec, rvec_param);                        // restore this since reused earlier
2654     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher); // final value of r stored in rvec of CipherBlockChaining object
2655     handleSOERegisters(false /*restoring*/);
2656     __ movptr(rax, len_param);                          // return length
2657     __ leave();                                         // required for proper stackwalking of RuntimeStub frame
2658     __ ret(0);
2659 
2660     return start;
2661   }
2662 
2663   // CTR AES crypt.
2664   // In 32-bit stub, parallelize 4 blocks at a time
2665   // Arguments:
2666   //
2667   // Inputs:
2668   //   c_rarg0   - source byte array address
2669   //   c_rarg1   - destination byte array address
2670   //   c_rarg2   - K (key) in little endian int array
2671   //   c_rarg3   - counter vector byte array address
2672   //   c_rarg4   - input length
2673   //
2674   // Output:
2675   //   rax       - input length
2676   //
2677   address generate_counterMode_AESCrypt_Parallel() {
2678     assert(UseAES, "need AES instructions and misaligned SSE support");
2679     __ align(CodeEntryAlignment);
2680     StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
2681     address start = __ pc();
2682     const Register from        = rsi;      // source array address
2683     const Register to          = rdx;      // destination array address
2684     const Register key         = rcx;      // key array address
2685     const Register counter     = rdi;      // counter byte array initialized from initvector array address
2686                                            // and updated with the incremented counter in the end
2687     const Register len_reg     = rbx;
2688     const Register pos         = rax;
2689 
2690     __ enter(); // required for proper stackwalking of RuntimeStub frame
2691     handleSOERegisters(true /*saving*/); // save rbx, rsi, rdi
2692 
2693     // load registers from incoming parameters
2694     const Address  from_param(rbp, 8+0);
2695     const Address  to_param  (rbp, 8+4);
2696     const Address  key_param (rbp, 8+8);
2697     const Address  rvec_param (rbp, 8+12);
2698     const Address  len_param  (rbp, 8+16);
2699     const Address  saved_counter_param(rbp, 8 + 20);
2700     const Address  used_addr_param(rbp, 8 + 24);
2701 
2702     __ movptr(from , from_param);
2703     __ movptr(to   , to_param);
2704     __ movptr(len_reg , len_param);
2705 
2706     // Use the partially used encrpyted counter from last invocation
2707     Label L_exit_preLoop, L_preLoop_start;
2708 
2709     // Use the registers 'counter' and 'key' here in this preloop
2710     // to hold of last 2 params 'used' and 'saved_encCounter_start'
2711     Register used = counter;
2712     Register saved_encCounter_start = key;
2713     Register used_addr = saved_encCounter_start;
2714 
2715     __ movptr(used_addr, used_addr_param);
2716     __ movptr(used, Address(used_addr, 0));
2717     __ movptr(saved_encCounter_start, saved_counter_param);
2718 
2719     __ BIND(L_preLoop_start);
2720     __ cmpptr(used, 16);
2721     __ jcc(Assembler::aboveEqual, L_exit_preLoop);
2722     __ cmpptr(len_reg, 0);
2723     __ jcc(Assembler::lessEqual, L_exit_preLoop);
2724     __ movb(rax, Address(saved_encCounter_start, used));
2725     __ xorb(rax, Address(from, 0));
2726     __ movb(Address(to, 0), rax);
2727     __ addptr(from, 1);
2728     __ addptr(to, 1);
2729     __ addptr(used, 1);
2730     __ subptr(len_reg, 1);
2731 
2732     __ jmp(L_preLoop_start);
2733 
2734     __ BIND(L_exit_preLoop);
2735     __ movptr(used_addr, used_addr_param);
2736     __ movptr(used_addr, used_addr_param);
2737     __ movl(Address(used_addr, 0), used);
2738 
2739     // load the parameters 'key' and 'counter'
2740     __ movptr(key, key_param);
2741     __ movptr(counter, rvec_param);
2742 
2743     // xmm register assignments for the loops below
2744     const XMMRegister xmm_curr_counter      = xmm0;
2745     const XMMRegister xmm_counter_shuf_mask = xmm1;  // need to be reloaded
2746     const XMMRegister xmm_key_shuf_mask     = xmm2;  // need to be reloaded
2747     const XMMRegister xmm_key               = xmm3;
2748     const XMMRegister xmm_result0           = xmm4;
2749     const XMMRegister xmm_result1           = xmm5;
2750     const XMMRegister xmm_result2           = xmm6;
2751     const XMMRegister xmm_result3           = xmm7;
2752     const XMMRegister xmm_from0             = xmm1;   //reuse XMM register
2753     const XMMRegister xmm_from1             = xmm2;
2754     const XMMRegister xmm_from2             = xmm3;
2755     const XMMRegister xmm_from3             = xmm4;
2756 
2757     //for key_128, key_192, key_256
2758     const int rounds[3] = {10, 12, 14};
2759     Label L_singleBlockLoopTop[3];
2760     Label L_multiBlock_loopTop[3];
2761     Label L_key192_top, L_key256_top;
2762     Label L_incCounter[3][4]; // 3: different key length,  4: 4 blocks at a time
2763     Label L_incCounter_single[3]; //for single block, key128, key192, key256
2764     Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3];
2765     Label L_processTail_extr[3], L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3];
2766 
2767     Label L_exit;
2768     const int PARALLEL_FACTOR = 4;  //because of the limited register number
2769 
2770     // initialize counter with initial counter
2771     __ movdqu(xmm_curr_counter, Address(counter, 0x00));
2772     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2773     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled for increase
2774 
2775     // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
2776     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2777     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2778     __ cmpl(rax, 52);
2779     __ jcc(Assembler::equal, L_key192_top);
2780     __ cmpl(rax, 60);
2781     __ jcc(Assembler::equal, L_key256_top);
2782 
2783     //key128 begins here
2784     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2785 
2786 #define CTR_DoFour(opc, src_reg)               \
2787     __ opc(xmm_result0, src_reg);              \
2788     __ opc(xmm_result1, src_reg);              \
2789     __ opc(xmm_result2, src_reg);              \
2790     __ opc(xmm_result3, src_reg);
2791 
2792     // k == 0 :  generate code for key_128
2793     // k == 1 :  generate code for key_192
2794     // k == 2 :  generate code for key_256
2795     for (int k = 0; k < 3; ++k) {
2796       //multi blocks starts here
2797       __ align(OptoLoopAlignment);
2798       __ BIND(L_multiBlock_loopTop[k]);
2799       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left
2800       __ jcc(Assembler::less, L_singleBlockLoopTop[k]);
2801 
2802       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2803       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2804 
2805       //load, then increase counters
2806       CTR_DoFour(movdqa, xmm_curr_counter);
2807       __ push(rbx);
2808       inc_counter(rbx, xmm_result1, 0x01, L_incCounter[k][0]);
2809       inc_counter(rbx, xmm_result2, 0x02, L_incCounter[k][1]);
2810       inc_counter(rbx, xmm_result3, 0x03, L_incCounter[k][2]);
2811       inc_counter(rbx, xmm_curr_counter, 0x04, L_incCounter[k][3]);
2812       __ pop (rbx);
2813 
2814       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask); // load Round 0 key. interleaving for better performance
2815 
2816       CTR_DoFour(pshufb, xmm_counter_shuf_mask); // after increased, shuffled counters back for PXOR
2817       CTR_DoFour(pxor, xmm_key);   //PXOR with Round 0 key
2818 
2819       for (int i = 1; i < rounds[k]; ++i) {
2820         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2821         CTR_DoFour(aesenc, xmm_key);
2822       }
2823       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2824       CTR_DoFour(aesenclast, xmm_key);
2825 
2826       // get next PARALLEL_FACTOR blocks into xmm_from registers
2827       __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2828       __ movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2829       __ movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2830 
2831       // PXOR with input text
2832       __ pxor(xmm_result0, xmm_from0); //result0 is xmm4
2833       __ pxor(xmm_result1, xmm_from1);
2834       __ pxor(xmm_result2, xmm_from2);
2835 
2836       // store PARALLEL_FACTOR results into the next 64 bytes of output
2837       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2838       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2839       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2840 
2841       // do it here after xmm_result0 is saved, because xmm_from3 reuse the same register of xmm_result0.
2842       __ movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2843       __ pxor(xmm_result3, xmm_from3);
2844       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2845 
2846       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text
2847       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length
2848       __ jmp(L_multiBlock_loopTop[k]);
2849 
2850       // singleBlock starts here
2851       __ align(OptoLoopAlignment);
2852       __ BIND(L_singleBlockLoopTop[k]);
2853       __ cmpptr(len_reg, 0);
2854       __ jcc(Assembler::equal, L_exit);
2855       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2856       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2857       __ movdqa(xmm_result0, xmm_curr_counter);
2858       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask);
2859       __ push(rbx);//rbx is used for increasing counter
2860       inc_counter(rbx, xmm_curr_counter, 0x01, L_incCounter_single[k]);
2861       __ pop (rbx);
2862       __ pshufb(xmm_result0, xmm_counter_shuf_mask);
2863       __ pxor(xmm_result0, xmm_key);
2864       for (int i = 1; i < rounds[k]; i++) {
2865         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2866         __ aesenc(xmm_result0, xmm_key);
2867       }
2868       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2869       __ aesenclast(xmm_result0, xmm_key);
2870       __ cmpptr(len_reg, AESBlockSize);
2871       __ jcc(Assembler::less, L_processTail_insr[k]);
2872         __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2873         __ pxor(xmm_result0, xmm_from0);
2874         __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2875         __ addptr(pos, AESBlockSize);
2876         __ subptr(len_reg, AESBlockSize);
2877         __ jmp(L_singleBlockLoopTop[k]);
2878 
2879       __ BIND(L_processTail_insr[k]);                                               // Process the tail part of the input array
2880         __ addptr(pos, len_reg);                                                    // 1. Insert bytes from src array into xmm_from0 register
2881         __ testptr(len_reg, 8);
2882         __ jcc(Assembler::zero, L_processTail_4_insr[k]);
2883           __ subptr(pos,8);
2884           __ pinsrd(xmm_from0, Address(from, pos), 0);
2885           __ pinsrd(xmm_from0, Address(from, pos, Address::times_1, 4), 1);
2886         __ BIND(L_processTail_4_insr[k]);
2887         __ testptr(len_reg, 4);
2888         __ jcc(Assembler::zero, L_processTail_2_insr[k]);
2889           __ subptr(pos,4);
2890           __ pslldq(xmm_from0, 4);
2891           __ pinsrd(xmm_from0, Address(from, pos), 0);
2892         __ BIND(L_processTail_2_insr[k]);
2893         __ testptr(len_reg, 2);
2894         __ jcc(Assembler::zero, L_processTail_1_insr[k]);
2895           __ subptr(pos, 2);
2896           __ pslldq(xmm_from0, 2);
2897           __ pinsrw(xmm_from0, Address(from, pos), 0);
2898         __ BIND(L_processTail_1_insr[k]);
2899         __ testptr(len_reg, 1);
2900         __ jcc(Assembler::zero, L_processTail_exit_insr[k]);
2901           __ subptr(pos, 1);
2902           __ pslldq(xmm_from0, 1);
2903           __ pinsrb(xmm_from0, Address(from, pos), 0);
2904         __ BIND(L_processTail_exit_insr[k]);
2905 
2906         __ movptr(saved_encCounter_start, saved_counter_param);
2907         __ movdqu(Address(saved_encCounter_start, 0), xmm_result0);               // 2. Perform pxor of the encrypted counter and plaintext Bytes.
2908         __ pxor(xmm_result0, xmm_from0);                                          //    Also the encrypted counter is saved for next invocation.
2909 
2910         __ testptr(len_reg, 8);
2911         __ jcc(Assembler::zero, L_processTail_4_extr[k]);                        // 3. Extract bytes from xmm_result0 into the dest. array
2912           __ pextrd(Address(to, pos), xmm_result0, 0);
2913           __ pextrd(Address(to, pos, Address::times_1, 4), xmm_result0, 1);
2914           __ psrldq(xmm_result0, 8);
2915           __ addptr(pos, 8);
2916         __ BIND(L_processTail_4_extr[k]);
2917         __ testptr(len_reg, 4);
2918         __ jcc(Assembler::zero, L_processTail_2_extr[k]);
2919           __ pextrd(Address(to, pos), xmm_result0, 0);
2920           __ psrldq(xmm_result0, 4);
2921           __ addptr(pos, 4);
2922         __ BIND(L_processTail_2_extr[k]);
2923         __ testptr(len_reg, 2);
2924         __ jcc(Assembler::zero, L_processTail_1_extr[k]);
2925           __ pextrb(Address(to, pos), xmm_result0, 0);
2926           __ pextrb(Address(to, pos, Address::times_1, 1), xmm_result0, 1);
2927           __ psrldq(xmm_result0, 2);
2928           __ addptr(pos, 2);
2929         __ BIND(L_processTail_1_extr[k]);
2930         __ testptr(len_reg, 1);
2931         __ jcc(Assembler::zero, L_processTail_exit_extr[k]);
2932           __ pextrb(Address(to, pos), xmm_result0, 0);
2933 
2934         __ BIND(L_processTail_exit_extr[k]);
2935         __ movptr(used_addr, used_addr_param);
2936         __ movl(Address(used_addr, 0), len_reg);
2937         __ jmp(L_exit);
2938     }
2939 
2940     __ BIND(L_exit);
2941     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2942     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back.
2943     __ movdqu(Address(counter, 0), xmm_curr_counter); //save counter back
2944     handleSOERegisters(false /*restoring*/);
2945     __ movptr(rax, len_param); // return length
2946     __ leave();                // required for proper stackwalking of RuntimeStub frame
2947     __ ret(0);
2948 
2949     __ BIND (L_key192_top);
2950     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2951     __ jmp(L_multiBlock_loopTop[1]); //key192
2952 
2953     __ BIND (L_key256_top);
2954     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2955     __ jmp(L_multiBlock_loopTop[2]); //key192
2956 
2957     return start;
2958   }
2959 
2960   // ofs and limit are use for multi-block byte array.
2961   // int com.sun.security.provider.MD5.implCompress(byte[] b, int ofs)
2962   address generate_md5_implCompress(bool multi_block, const char *name) {
2963     __ align(CodeEntryAlignment);
2964     StubCodeMark mark(this, "StubRoutines", name);
2965     address start = __ pc();
2966 
2967     const Register buf_param = rbp;
2968     const Address state_param(rsp, 0 * wordSize);
2969     const Address ofs_param  (rsp, 1 * wordSize);
2970     const Address limit_param(rsp, 2 * wordSize);
2971 
2972     __ enter();
2973     __ push(rbx);
2974     __ push(rdi);
2975     __ push(rsi);
2976     __ push(rbp);
2977     __ subptr(rsp, 3 * wordSize);
2978 
2979     __ movptr(rsi, Address(rbp, 8 + 4));
2980     __ movptr(state_param, rsi);
2981     if (multi_block) {
2982       __ movptr(rsi, Address(rbp, 8 + 8));
2983       __ movptr(ofs_param, rsi);
2984       __ movptr(rsi, Address(rbp, 8 + 12));
2985       __ movptr(limit_param, rsi);
2986     }
2987     __ movptr(buf_param, Address(rbp, 8 + 0)); // do it last because it override rbp
2988     __ fast_md5(buf_param, state_param, ofs_param, limit_param, multi_block);
2989 
2990     __ addptr(rsp, 3 * wordSize);
2991     __ pop(rbp);
2992     __ pop(rsi);
2993     __ pop(rdi);
2994     __ pop(rbx);
2995     __ leave();
2996     __ ret(0);
2997     return start;
2998   }
2999 
3000   address generate_upper_word_mask() {
3001     __ align64();
3002     StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
3003     address start = __ pc();
3004     __ emit_data(0x00000000, relocInfo::none, 0);
3005     __ emit_data(0x00000000, relocInfo::none, 0);
3006     __ emit_data(0x00000000, relocInfo::none, 0);
3007     __ emit_data(0xFFFFFFFF, relocInfo::none, 0);
3008     return start;
3009   }
3010 
3011   address generate_shuffle_byte_flip_mask() {
3012     __ align64();
3013     StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask");
3014     address start = __ pc();
3015     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3016     __ emit_data(0x08090a0b, relocInfo::none, 0);
3017     __ emit_data(0x04050607, relocInfo::none, 0);
3018     __ emit_data(0x00010203, relocInfo::none, 0);
3019     return start;
3020   }
3021 
3022   // ofs and limit are use for multi-block byte array.
3023   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3024   address generate_sha1_implCompress(bool multi_block, const char *name) {
3025     __ align(CodeEntryAlignment);
3026     StubCodeMark mark(this, "StubRoutines", name);
3027     address start = __ pc();
3028 
3029     Register buf   = rax;
3030     Register state = rdx;
3031     Register ofs   = rcx;
3032     Register limit = rdi;
3033 
3034     const Address  buf_param(rbp, 8 + 0);
3035     const Address  state_param(rbp, 8 + 4);
3036     const Address  ofs_param(rbp, 8 + 8);
3037     const Address  limit_param(rbp, 8 + 12);
3038 
3039     const XMMRegister abcd = xmm0;
3040     const XMMRegister e0 = xmm1;
3041     const XMMRegister e1 = xmm2;
3042     const XMMRegister msg0 = xmm3;
3043 
3044     const XMMRegister msg1 = xmm4;
3045     const XMMRegister msg2 = xmm5;
3046     const XMMRegister msg3 = xmm6;
3047     const XMMRegister shuf_mask = xmm7;
3048 
3049     __ enter();
3050     __ subptr(rsp, 8 * wordSize);
3051     handleSOERegisters(true /*saving*/);
3052 
3053     __ movptr(buf, buf_param);
3054     __ movptr(state, state_param);
3055     if (multi_block) {
3056       __ movptr(ofs, ofs_param);
3057       __ movptr(limit, limit_param);
3058     }
3059 
3060     __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask,
3061       buf, state, ofs, limit, rsp, multi_block);
3062 
3063     handleSOERegisters(false /*restoring*/);
3064     __ addptr(rsp, 8 * wordSize);
3065     __ leave();
3066     __ ret(0);
3067     return start;
3068   }
3069 
3070   address generate_pshuffle_byte_flip_mask() {
3071     __ align64();
3072     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask");
3073     address start = __ pc();
3074     __ emit_data(0x00010203, relocInfo::none, 0);
3075     __ emit_data(0x04050607, relocInfo::none, 0);
3076     __ emit_data(0x08090a0b, relocInfo::none, 0);
3077     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3078     return start;
3079   }
3080 
3081   // ofs and limit are use for multi-block byte array.
3082   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3083  address generate_sha256_implCompress(bool multi_block, const char *name) {
3084     __ align(CodeEntryAlignment);
3085     StubCodeMark mark(this, "StubRoutines", name);
3086     address start = __ pc();
3087 
3088     Register buf = rbx;
3089     Register state = rsi;
3090     Register ofs = rdx;
3091     Register limit = rcx;
3092 
3093     const Address  buf_param(rbp, 8 + 0);
3094     const Address  state_param(rbp, 8 + 4);
3095     const Address  ofs_param(rbp, 8 + 8);
3096     const Address  limit_param(rbp, 8 + 12);
3097 
3098     const XMMRegister msg = xmm0;
3099     const XMMRegister state0 = xmm1;
3100     const XMMRegister state1 = xmm2;
3101     const XMMRegister msgtmp0 = xmm3;
3102 
3103     const XMMRegister msgtmp1 = xmm4;
3104     const XMMRegister msgtmp2 = xmm5;
3105     const XMMRegister msgtmp3 = xmm6;
3106     const XMMRegister msgtmp4 = xmm7;
3107 
3108     __ enter();
3109     __ subptr(rsp, 8 * wordSize);
3110     handleSOERegisters(true /*saving*/);
3111     __ movptr(buf, buf_param);
3112     __ movptr(state, state_param);
3113     if (multi_block) {
3114      __ movptr(ofs, ofs_param);
3115      __ movptr(limit, limit_param);
3116     }
3117 
3118     __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3119       buf, state, ofs, limit, rsp, multi_block);
3120 
3121     handleSOERegisters(false);
3122     __ addptr(rsp, 8 * wordSize);
3123     __ leave();
3124     __ ret(0);
3125     return start;
3126   }
3127 
3128   // byte swap x86 long
3129   address generate_ghash_long_swap_mask() {
3130     __ align(CodeEntryAlignment);
3131     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
3132     address start = __ pc();
3133     __ emit_data(0x0b0a0908, relocInfo::none, 0);
3134     __ emit_data(0x0f0e0d0c, relocInfo::none, 0);
3135     __ emit_data(0x03020100, relocInfo::none, 0);
3136     __ emit_data(0x07060504, relocInfo::none, 0);
3137 
3138   return start;
3139   }
3140 
3141   // byte swap x86 byte array
3142   address generate_ghash_byte_swap_mask() {
3143     __ align(CodeEntryAlignment);
3144     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
3145     address start = __ pc();
3146     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3147     __ emit_data(0x08090a0b, relocInfo::none, 0);
3148     __ emit_data(0x04050607, relocInfo::none, 0);
3149     __ emit_data(0x00010203, relocInfo::none, 0);
3150   return start;
3151   }
3152 
3153   /* Single and multi-block ghash operations */
3154   address generate_ghash_processBlocks() {
3155     assert(UseGHASHIntrinsics, "need GHASH intrinsics and CLMUL support");
3156     __ align(CodeEntryAlignment);
3157     Label L_ghash_loop, L_exit;
3158     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
3159     address start = __ pc();
3160 
3161     const Register state        = rdi;
3162     const Register subkeyH      = rsi;
3163     const Register data         = rdx;
3164     const Register blocks       = rcx;
3165 
3166     const Address  state_param(rbp, 8+0);
3167     const Address  subkeyH_param(rbp, 8+4);
3168     const Address  data_param(rbp, 8+8);
3169     const Address  blocks_param(rbp, 8+12);
3170 
3171     const XMMRegister xmm_temp0 = xmm0;
3172     const XMMRegister xmm_temp1 = xmm1;
3173     const XMMRegister xmm_temp2 = xmm2;
3174     const XMMRegister xmm_temp3 = xmm3;
3175     const XMMRegister xmm_temp4 = xmm4;
3176     const XMMRegister xmm_temp5 = xmm5;
3177     const XMMRegister xmm_temp6 = xmm6;
3178     const XMMRegister xmm_temp7 = xmm7;
3179 
3180     __ enter();
3181     handleSOERegisters(true);  // Save registers
3182 
3183     __ movptr(state, state_param);
3184     __ movptr(subkeyH, subkeyH_param);
3185     __ movptr(data, data_param);
3186     __ movptr(blocks, blocks_param);
3187 
3188     __ movdqu(xmm_temp0, Address(state, 0));
3189     __ pshufb(xmm_temp0, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3190 
3191     __ movdqu(xmm_temp1, Address(subkeyH, 0));
3192     __ pshufb(xmm_temp1, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3193 
3194     __ BIND(L_ghash_loop);
3195     __ movdqu(xmm_temp2, Address(data, 0));
3196     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
3197 
3198     __ pxor(xmm_temp0, xmm_temp2);
3199 
3200     //
3201     // Multiply with the hash key
3202     //
3203     __ movdqu(xmm_temp3, xmm_temp0);
3204     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
3205     __ movdqu(xmm_temp4, xmm_temp0);
3206     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
3207 
3208     __ movdqu(xmm_temp5, xmm_temp0);
3209     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
3210     __ movdqu(xmm_temp6, xmm_temp0);
3211     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
3212 
3213     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
3214 
3215     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
3216     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
3217     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
3218     __ pxor(xmm_temp3, xmm_temp5);
3219     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
3220                                         // of the carry-less multiplication of
3221                                         // xmm0 by xmm1.
3222 
3223     // We shift the result of the multiplication by one bit position
3224     // to the left to cope for the fact that the bits are reversed.
3225     __ movdqu(xmm_temp7, xmm_temp3);
3226     __ movdqu(xmm_temp4, xmm_temp6);
3227     __ pslld (xmm_temp3, 1);
3228     __ pslld(xmm_temp6, 1);
3229     __ psrld(xmm_temp7, 31);
3230     __ psrld(xmm_temp4, 31);
3231     __ movdqu(xmm_temp5, xmm_temp7);
3232     __ pslldq(xmm_temp4, 4);
3233     __ pslldq(xmm_temp7, 4);
3234     __ psrldq(xmm_temp5, 12);
3235     __ por(xmm_temp3, xmm_temp7);
3236     __ por(xmm_temp6, xmm_temp4);
3237     __ por(xmm_temp6, xmm_temp5);
3238 
3239     //
3240     // First phase of the reduction
3241     //
3242     // Move xmm3 into xmm4, xmm5, xmm7 in order to perform the shifts
3243     // independently.
3244     __ movdqu(xmm_temp7, xmm_temp3);
3245     __ movdqu(xmm_temp4, xmm_temp3);
3246     __ movdqu(xmm_temp5, xmm_temp3);
3247     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
3248     __ pslld(xmm_temp4, 30);    // packed right shift shifting << 30
3249     __ pslld(xmm_temp5, 25);    // packed right shift shifting << 25
3250     __ pxor(xmm_temp7, xmm_temp4);      // xor the shifted versions
3251     __ pxor(xmm_temp7, xmm_temp5);
3252     __ movdqu(xmm_temp4, xmm_temp7);
3253     __ pslldq(xmm_temp7, 12);
3254     __ psrldq(xmm_temp4, 4);
3255     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
3256 
3257     //
3258     // Second phase of the reduction
3259     //
3260     // Make 3 copies of xmm3 in xmm2, xmm5, xmm7 for doing these
3261     // shift operations.
3262     __ movdqu(xmm_temp2, xmm_temp3);
3263     __ movdqu(xmm_temp7, xmm_temp3);
3264     __ movdqu(xmm_temp5, xmm_temp3);
3265     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
3266     __ psrld(xmm_temp7, 2);     // packed left shifting >> 2
3267     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
3268     __ pxor(xmm_temp2, xmm_temp7);      // xor the shifted versions
3269     __ pxor(xmm_temp2, xmm_temp5);
3270     __ pxor(xmm_temp2, xmm_temp4);
3271     __ pxor(xmm_temp3, xmm_temp2);
3272     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
3273 
3274     __ decrement(blocks);
3275     __ jcc(Assembler::zero, L_exit);
3276     __ movdqu(xmm_temp0, xmm_temp6);
3277     __ addptr(data, 16);
3278     __ jmp(L_ghash_loop);
3279 
3280     __ BIND(L_exit);
3281        // Byte swap 16-byte result
3282     __ pshufb(xmm_temp6, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3283     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
3284 
3285     handleSOERegisters(false);  // restore registers
3286     __ leave();
3287     __ ret(0);
3288     return start;
3289   }
3290 
3291   /**
3292    *  Arguments:
3293    *
3294    * Inputs:
3295    *   rsp(4)   - int crc
3296    *   rsp(8)   - byte* buf
3297    *   rsp(12)  - int length
3298    *
3299    * Ouput:
3300    *       rax   - int crc result
3301    */
3302   address generate_updateBytesCRC32() {
3303     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
3304 
3305     __ align(CodeEntryAlignment);
3306     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
3307 
3308     address start = __ pc();
3309 
3310     const Register crc   = rdx;  // crc
3311     const Register buf   = rsi;  // source java byte array address
3312     const Register len   = rcx;  // length
3313     const Register table = rdi;  // crc_table address (reuse register)
3314     const Register tmp   = rbx;
3315     assert_different_registers(crc, buf, len, table, tmp, rax);
3316 
3317     BLOCK_COMMENT("Entry:");
3318     __ enter(); // required for proper stackwalking of RuntimeStub frame
3319     __ push(rsi);
3320     __ push(rdi);
3321     __ push(rbx);
3322 
3323     Address crc_arg(rbp, 8 + 0);
3324     Address buf_arg(rbp, 8 + 4);
3325     Address len_arg(rbp, 8 + 8);
3326 
3327     // Load up:
3328     __ movl(crc,   crc_arg);
3329     __ movptr(buf, buf_arg);
3330     __ movl(len,   len_arg);
3331 
3332     __ kernel_crc32(crc, buf, len, table, tmp);
3333 
3334     __ movl(rax, crc);
3335     __ pop(rbx);
3336     __ pop(rdi);
3337     __ pop(rsi);
3338     __ vzeroupper();
3339     __ leave(); // required for proper stackwalking of RuntimeStub frame
3340     __ ret(0);
3341 
3342     return start;
3343   }
3344 
3345   /**
3346   *  Arguments:
3347   *
3348   * Inputs:
3349   *   rsp(4)   - int crc
3350   *   rsp(8)   - byte* buf
3351   *   rsp(12)  - int length
3352   *   rsp(16)  - table_start - optional (present only when doing a library_calll,
3353   *              not used by x86 algorithm)
3354   *
3355   * Ouput:
3356   *       rax  - int crc result
3357   */
3358   address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
3359     assert(UseCRC32CIntrinsics, "need SSE4_2");
3360     __ align(CodeEntryAlignment);
3361     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
3362     address start = __ pc();
3363     const Register crc = rax;  // crc
3364     const Register buf = rcx;  // source java byte array address
3365     const Register len = rdx;  // length
3366     const Register d = rbx;
3367     const Register g = rsi;
3368     const Register h = rdi;
3369     const Register empty = 0; // will never be used, in order not
3370                               // to change a signature for crc32c_IPL_Alg2_Alt2
3371                               // between 64/32 I'm just keeping it here
3372     assert_different_registers(crc, buf, len, d, g, h);
3373 
3374     BLOCK_COMMENT("Entry:");
3375     __ enter(); // required for proper stackwalking of RuntimeStub frame
3376     Address crc_arg(rsp, 4 + 4 + 0); // ESP+4 +
3377                                      // we need to add additional 4 because __ enter
3378                                      // have just pushed ebp on a stack
3379     Address buf_arg(rsp, 4 + 4 + 4);
3380     Address len_arg(rsp, 4 + 4 + 8);
3381       // Load up:
3382       __ movl(crc, crc_arg);
3383       __ movl(buf, buf_arg);
3384       __ movl(len, len_arg);
3385       __ push(d);
3386       __ push(g);
3387       __ push(h);
3388       __ crc32c_ipl_alg2_alt2(crc, buf, len,
3389                               d, g, h,
3390                               empty, empty, empty,
3391                               xmm0, xmm1, xmm2,
3392                               is_pclmulqdq_supported);
3393       __ pop(h);
3394       __ pop(g);
3395       __ pop(d);
3396     __ vzeroupper();
3397     __ leave(); // required for proper stackwalking of RuntimeStub frame
3398     __ ret(0);
3399 
3400     return start;
3401   }
3402 
3403  address generate_libmExp() {
3404     StubCodeMark mark(this, "StubRoutines", "libmExp");
3405 
3406     address start = __ pc();
3407 
3408     const XMMRegister x0  = xmm0;
3409     const XMMRegister x1  = xmm1;
3410     const XMMRegister x2  = xmm2;
3411     const XMMRegister x3  = xmm3;
3412 
3413     const XMMRegister x4  = xmm4;
3414     const XMMRegister x5  = xmm5;
3415     const XMMRegister x6  = xmm6;
3416     const XMMRegister x7  = xmm7;
3417 
3418     const Register tmp   = rbx;
3419 
3420     BLOCK_COMMENT("Entry:");
3421     __ enter(); // required for proper stackwalking of RuntimeStub frame
3422     __ fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3423     __ leave(); // required for proper stackwalking of RuntimeStub frame
3424     __ ret(0);
3425 
3426     return start;
3427 
3428   }
3429 
3430  address generate_libmLog() {
3431    StubCodeMark mark(this, "StubRoutines", "libmLog");
3432 
3433    address start = __ pc();
3434 
3435    const XMMRegister x0 = xmm0;
3436    const XMMRegister x1 = xmm1;
3437    const XMMRegister x2 = xmm2;
3438    const XMMRegister x3 = xmm3;
3439 
3440    const XMMRegister x4 = xmm4;
3441    const XMMRegister x5 = xmm5;
3442    const XMMRegister x6 = xmm6;
3443    const XMMRegister x7 = xmm7;
3444 
3445    const Register tmp = rbx;
3446 
3447    BLOCK_COMMENT("Entry:");
3448    __ enter(); // required for proper stackwalking of RuntimeStub frame
3449    __ fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3450    __ leave(); // required for proper stackwalking of RuntimeStub frame
3451    __ ret(0);
3452 
3453    return start;
3454 
3455  }
3456 
3457  address generate_libmLog10() {
3458    StubCodeMark mark(this, "StubRoutines", "libmLog10");
3459 
3460    address start = __ pc();
3461 
3462    const XMMRegister x0 = xmm0;
3463    const XMMRegister x1 = xmm1;
3464    const XMMRegister x2 = xmm2;
3465    const XMMRegister x3 = xmm3;
3466 
3467    const XMMRegister x4 = xmm4;
3468    const XMMRegister x5 = xmm5;
3469    const XMMRegister x6 = xmm6;
3470    const XMMRegister x7 = xmm7;
3471 
3472    const Register tmp = rbx;
3473 
3474    BLOCK_COMMENT("Entry:");
3475    __ enter(); // required for proper stackwalking of RuntimeStub frame
3476    __ fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3477    __ leave(); // required for proper stackwalking of RuntimeStub frame
3478    __ ret(0);
3479 
3480    return start;
3481 
3482  }
3483 
3484  address generate_libmPow() {
3485    StubCodeMark mark(this, "StubRoutines", "libmPow");
3486 
3487    address start = __ pc();
3488 
3489    const XMMRegister x0 = xmm0;
3490    const XMMRegister x1 = xmm1;
3491    const XMMRegister x2 = xmm2;
3492    const XMMRegister x3 = xmm3;
3493 
3494    const XMMRegister x4 = xmm4;
3495    const XMMRegister x5 = xmm5;
3496    const XMMRegister x6 = xmm6;
3497    const XMMRegister x7 = xmm7;
3498 
3499    const Register tmp = rbx;
3500 
3501    BLOCK_COMMENT("Entry:");
3502    __ enter(); // required for proper stackwalking of RuntimeStub frame
3503    __ fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3504    __ leave(); // required for proper stackwalking of RuntimeStub frame
3505    __ ret(0);
3506 
3507    return start;
3508 
3509  }
3510 
3511  address generate_libm_reduce_pi04l() {
3512    StubCodeMark mark(this, "StubRoutines", "libm_reduce_pi04l");
3513 
3514    address start = __ pc();
3515 
3516    BLOCK_COMMENT("Entry:");
3517    __ libm_reduce_pi04l(rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3518 
3519    return start;
3520 
3521  }
3522 
3523  address generate_libm_sin_cos_huge() {
3524    StubCodeMark mark(this, "StubRoutines", "libm_sin_cos_huge");
3525 
3526    address start = __ pc();
3527 
3528    const XMMRegister x0 = xmm0;
3529    const XMMRegister x1 = xmm1;
3530 
3531    BLOCK_COMMENT("Entry:");
3532    __ libm_sincos_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3533 
3534    return start;
3535 
3536  }
3537 
3538  address generate_libmSin() {
3539    StubCodeMark mark(this, "StubRoutines", "libmSin");
3540 
3541    address start = __ pc();
3542 
3543    const XMMRegister x0 = xmm0;
3544    const XMMRegister x1 = xmm1;
3545    const XMMRegister x2 = xmm2;
3546    const XMMRegister x3 = xmm3;
3547 
3548    const XMMRegister x4 = xmm4;
3549    const XMMRegister x5 = xmm5;
3550    const XMMRegister x6 = xmm6;
3551    const XMMRegister x7 = xmm7;
3552 
3553    BLOCK_COMMENT("Entry:");
3554    __ enter(); // required for proper stackwalking of RuntimeStub frame
3555    __ fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rdx);
3556    __ leave(); // required for proper stackwalking of RuntimeStub frame
3557    __ ret(0);
3558 
3559    return start;
3560 
3561  }
3562 
3563  address generate_libmCos() {
3564    StubCodeMark mark(this, "StubRoutines", "libmCos");
3565 
3566    address start = __ pc();
3567 
3568    const XMMRegister x0 = xmm0;
3569    const XMMRegister x1 = xmm1;
3570    const XMMRegister x2 = xmm2;
3571    const XMMRegister x3 = xmm3;
3572 
3573    const XMMRegister x4 = xmm4;
3574    const XMMRegister x5 = xmm5;
3575    const XMMRegister x6 = xmm6;
3576    const XMMRegister x7 = xmm7;
3577 
3578    const Register tmp = rbx;
3579 
3580    BLOCK_COMMENT("Entry:");
3581    __ enter(); // required for proper stackwalking of RuntimeStub frame
3582    __ fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3583    __ leave(); // required for proper stackwalking of RuntimeStub frame
3584    __ ret(0);
3585 
3586    return start;
3587 
3588  }
3589 
3590  address generate_libm_tan_cot_huge() {
3591    StubCodeMark mark(this, "StubRoutines", "libm_tan_cot_huge");
3592 
3593    address start = __ pc();
3594 
3595    const XMMRegister x0 = xmm0;
3596    const XMMRegister x1 = xmm1;
3597 
3598    BLOCK_COMMENT("Entry:");
3599    __ libm_tancot_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3600 
3601    return start;
3602 
3603  }
3604 
3605  address generate_libmTan() {
3606    StubCodeMark mark(this, "StubRoutines", "libmTan");
3607 
3608    address start = __ pc();
3609 
3610    const XMMRegister x0 = xmm0;
3611    const XMMRegister x1 = xmm1;
3612    const XMMRegister x2 = xmm2;
3613    const XMMRegister x3 = xmm3;
3614 
3615    const XMMRegister x4 = xmm4;
3616    const XMMRegister x5 = xmm5;
3617    const XMMRegister x6 = xmm6;
3618    const XMMRegister x7 = xmm7;
3619 
3620    const Register tmp = rbx;
3621 
3622    BLOCK_COMMENT("Entry:");
3623    __ enter(); // required for proper stackwalking of RuntimeStub frame
3624    __ fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3625    __ leave(); // required for proper stackwalking of RuntimeStub frame
3626    __ ret(0);
3627 
3628    return start;
3629 
3630  }
3631 
3632   // Safefetch stubs.
3633   void generate_safefetch(const char* name, int size, address* entry,
3634                           address* fault_pc, address* continuation_pc) {
3635     // safefetch signatures:
3636     //   int      SafeFetch32(int*      adr, int      errValue);
3637     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3638 
3639     StubCodeMark mark(this, "StubRoutines", name);
3640 
3641     // Entry point, pc or function descriptor.
3642     *entry = __ pc();
3643 
3644     __ movl(rax, Address(rsp, 0x8));
3645     __ movl(rcx, Address(rsp, 0x4));
3646     // Load *adr into eax, may fault.
3647     *fault_pc = __ pc();
3648     switch (size) {
3649       case 4:
3650         // int32_t
3651         __ movl(rax, Address(rcx, 0));
3652         break;
3653       case 8:
3654         // int64_t
3655         Unimplemented();
3656         break;
3657       default:
3658         ShouldNotReachHere();
3659     }
3660 
3661     // Return errValue or *adr.
3662     *continuation_pc = __ pc();
3663     __ ret(0);
3664   }
3665 
3666   address generate_method_entry_barrier() {
3667     __ align(CodeEntryAlignment);
3668     StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier");
3669 
3670     Label deoptimize_label;
3671 
3672     address start = __ pc();
3673 
3674     __ push(-1); // cookie, this is used for writing the new rsp when deoptimizing
3675 
3676     BLOCK_COMMENT("Entry:");
3677     __ enter(); // save rbp
3678 
3679     // save rbx, because we want to use that value.
3680     // We could do without it but then we depend on the number of slots used by pusha
3681     __ push(rbx);
3682 
3683     __ lea(rbx, Address(rsp, wordSize * 3)); // 1 for cookie, 1 for rbp, 1 for rbx - this should be the return address
3684 
3685     __ pusha();
3686 
3687     // xmm0 and xmm1 may be used for passing float/double arguments
3688 
3689     if (UseSSE >= 2) {
3690       const int xmm_size = wordSize * 4;
3691       __ subptr(rsp, xmm_size * 2);
3692       __ movdbl(Address(rsp, xmm_size * 1), xmm1);
3693       __ movdbl(Address(rsp, xmm_size * 0), xmm0);
3694     } else if (UseSSE >= 1) {
3695       const int xmm_size = wordSize * 2;
3696       __ subptr(rsp, xmm_size * 2);
3697       __ movflt(Address(rsp, xmm_size * 1), xmm1);
3698       __ movflt(Address(rsp, xmm_size * 0), xmm0);
3699     }
3700 
3701     __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(address*)>(BarrierSetNMethod::nmethod_stub_entry_barrier)), rbx);
3702 
3703     if (UseSSE >= 2) {
3704       const int xmm_size = wordSize * 4;
3705       __ movdbl(xmm0, Address(rsp, xmm_size * 0));
3706       __ movdbl(xmm1, Address(rsp, xmm_size * 1));
3707       __ addptr(rsp, xmm_size * 2);
3708     } else if (UseSSE >= 1) {
3709       const int xmm_size = wordSize * 2;
3710       __ movflt(xmm0, Address(rsp, xmm_size * 0));
3711       __ movflt(xmm1, Address(rsp, xmm_size * 1));
3712       __ addptr(rsp, xmm_size * 2);
3713     }
3714 
3715     __ cmpl(rax, 1); // 1 means deoptimize
3716     __ jcc(Assembler::equal, deoptimize_label);
3717 
3718     __ popa();
3719     __ pop(rbx);
3720 
3721     __ leave();
3722 
3723     __ addptr(rsp, 1 * wordSize); // cookie
3724     __ ret(0);
3725 
3726     __ BIND(deoptimize_label);
3727 
3728     __ popa();
3729     __ pop(rbx);
3730 
3731     __ leave();
3732 
3733     // this can be taken out, but is good for verification purposes. getting a SIGSEGV
3734     // here while still having a correct stack is valuable
3735     __ testptr(rsp, Address(rsp, 0));
3736 
3737     __ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier
3738     __ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point
3739 
3740     return start;
3741   }
3742 
3743  public:
3744   // Information about frame layout at time of blocking runtime call.
3745   // Note that we only have to preserve callee-saved registers since
3746   // the compilers are responsible for supplying a continuation point
3747   // if they expect all registers to be preserved.
3748   enum layout {
3749     thread_off,    // last_java_sp
3750     arg1_off,
3751     arg2_off,
3752     rbp_off,       // callee saved register
3753     ret_pc,
3754     framesize
3755   };
3756 
3757  private:
3758 
3759 #undef  __
3760 #define __ masm->
3761 
3762   //------------------------------------------------------------------------------------------------------------------------
3763   // Continuation point for throwing of implicit exceptions that are not handled in
3764   // the current activation. Fabricates an exception oop and initiates normal
3765   // exception dispatching in this frame.
3766   //
3767   // Previously the compiler (c2) allowed for callee save registers on Java calls.
3768   // This is no longer true after adapter frames were removed but could possibly
3769   // be brought back in the future if the interpreter code was reworked and it
3770   // was deemed worthwhile. The comment below was left to describe what must
3771   // happen here if callee saves were resurrected. As it stands now this stub
3772   // could actually be a vanilla BufferBlob and have now oopMap at all.
3773   // Since it doesn't make much difference we've chosen to leave it the
3774   // way it was in the callee save days and keep the comment.
3775 
3776   // If we need to preserve callee-saved values we need a callee-saved oop map and
3777   // therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
3778   // If the compiler needs all registers to be preserved between the fault
3779   // point and the exception handler then it must assume responsibility for that in
3780   // AbstractCompiler::continuation_for_implicit_null_exception or
3781   // continuation_for_implicit_division_by_zero_exception. All other implicit
3782   // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
3783   // either at call sites or otherwise assume that stack unwinding will be initiated,
3784   // so caller saved registers were assumed volatile in the compiler.
3785   address generate_throw_exception(const char* name, address runtime_entry,
3786                                    Register arg1 = noreg, Register arg2 = noreg) {
3787 
3788     int insts_size = 256;
3789     int locs_size  = 32;
3790 
3791     CodeBuffer code(name, insts_size, locs_size);
3792     OopMapSet* oop_maps  = new OopMapSet();
3793     MacroAssembler* masm = new MacroAssembler(&code);
3794 
3795     address start = __ pc();
3796 
3797     // This is an inlined and slightly modified version of call_VM
3798     // which has the ability to fetch the return PC out of
3799     // thread-local storage and also sets up last_Java_sp slightly
3800     // differently than the real call_VM
3801     Register java_thread = rbx;
3802     __ get_thread(java_thread);
3803 
3804     __ enter(); // required for proper stackwalking of RuntimeStub frame
3805 
3806     // pc and rbp, already pushed
3807     __ subptr(rsp, (framesize-2) * wordSize); // prolog
3808 
3809     // Frame is now completed as far as size and linkage.
3810 
3811     int frame_complete = __ pc() - start;
3812 
3813     // push java thread (becomes first argument of C function)
3814     __ movptr(Address(rsp, thread_off * wordSize), java_thread);
3815     if (arg1 != noreg) {
3816       __ movptr(Address(rsp, arg1_off * wordSize), arg1);
3817     }
3818     if (arg2 != noreg) {
3819       assert(arg1 != noreg, "missing reg arg");
3820       __ movptr(Address(rsp, arg2_off * wordSize), arg2);
3821     }
3822 
3823     // Set up last_Java_sp and last_Java_fp
3824     __ set_last_Java_frame(java_thread, rsp, rbp, NULL);
3825 
3826     // Call runtime
3827     BLOCK_COMMENT("call runtime_entry");
3828     __ call(RuntimeAddress(runtime_entry));
3829     // Generate oop map
3830     OopMap* map =  new OopMap(framesize, 0);
3831     oop_maps->add_gc_map(__ pc() - start, map);
3832 
3833     // restore the thread (cannot use the pushed argument since arguments
3834     // may be overwritten by C code generated by an optimizing compiler);
3835     // however can use the register value directly if it is callee saved.
3836     __ get_thread(java_thread);
3837 
3838     __ reset_last_Java_frame(java_thread, true);
3839 
3840     __ leave(); // required for proper stackwalking of RuntimeStub frame
3841 
3842     // check for pending exceptions
3843 #ifdef ASSERT
3844     Label L;
3845     __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3846     __ jcc(Assembler::notEqual, L);
3847     __ should_not_reach_here();
3848     __ bind(L);
3849 #endif /* ASSERT */
3850     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3851 
3852 
3853     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
3854     return stub->entry_point();
3855   }
3856 
3857 
3858   void create_control_words() {
3859     // Round to nearest, 53-bit mode, exceptions masked
3860     StubRoutines::x86::_fpu_cntrl_wrd_std   = 0x027F;
3861     // Round to zero, 53-bit mode, exception mased
3862     StubRoutines::x86::_fpu_cntrl_wrd_trunc = 0x0D7F;
3863     // Round to nearest, 24-bit mode, exceptions masked
3864     StubRoutines::x86::_fpu_cntrl_wrd_24    = 0x007F;
3865     // Round to nearest, 64-bit mode, exceptions masked
3866     StubRoutines::x86::_mxcsr_std           = 0x1F80;
3867     // Note: the following two constants are 80-bit values
3868     //       layout is critical for correct loading by FPU.
3869     // Bias for strict fp multiply/divide
3870     StubRoutines::x86::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
3871     StubRoutines::x86::_fpu_subnormal_bias1[1]= 0x80000000;
3872     StubRoutines::x86::_fpu_subnormal_bias1[2]= 0x03ff;
3873     // Un-Bias for strict fp multiply/divide
3874     StubRoutines::x86::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
3875     StubRoutines::x86::_fpu_subnormal_bias2[1]= 0x80000000;
3876     StubRoutines::x86::_fpu_subnormal_bias2[2]= 0x7bff;
3877   }
3878 
3879   //---------------------------------------------------------------------------
3880   // Initialization
3881 
3882   void generate_initial() {
3883     // Generates all stubs and initializes the entry points
3884 
3885     //------------------------------------------------------------------------------------------------------------------------
3886     // entry points that exist in all platforms
3887     // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
3888     //       the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
3889     StubRoutines::_forward_exception_entry      = generate_forward_exception();
3890 
3891     StubRoutines::_call_stub_entry              =
3892       generate_call_stub(StubRoutines::_call_stub_return_address);
3893     // is referenced by megamorphic call
3894     StubRoutines::_catch_exception_entry        = generate_catch_exception();
3895 
3896     // platform dependent
3897     create_control_words();
3898 
3899     StubRoutines::x86::_verify_mxcsr_entry         = generate_verify_mxcsr();
3900     StubRoutines::x86::_verify_fpu_cntrl_wrd_entry = generate_verify_fpu_cntrl_wrd();
3901     StubRoutines::x86::_d2i_wrapper                = generate_d2i_wrapper(T_INT,  CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
3902     StubRoutines::x86::_d2l_wrapper                = generate_d2i_wrapper(T_LONG, CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
3903 
3904     // Build this early so it's available for the interpreter
3905     StubRoutines::_throw_StackOverflowError_entry          = generate_throw_exception("StackOverflowError throw_exception",
3906                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
3907     StubRoutines::_throw_delayed_StackOverflowError_entry  = generate_throw_exception("delayed StackOverflowError throw_exception",
3908                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_delayed_StackOverflowError));
3909 
3910     if (UseCRC32Intrinsics) {
3911       // set table address before stub generation which use it
3912       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
3913       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
3914     }
3915 
3916     if (UseCRC32CIntrinsics) {
3917       bool supports_clmul = VM_Version::supports_clmul();
3918       StubRoutines::x86::generate_CRC32C_table(supports_clmul);
3919       StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table;
3920       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
3921     }
3922     if (VM_Version::supports_sse2() && UseLibmIntrinsic && InlineIntrinsics) {
3923       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3924           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3925           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3926         StubRoutines::x86::_L_2il0floatpacket_0_adr = (address)StubRoutines::x86::_L_2il0floatpacket_0;
3927         StubRoutines::x86::_Pi4Inv_adr = (address)StubRoutines::x86::_Pi4Inv;
3928         StubRoutines::x86::_Pi4x3_adr = (address)StubRoutines::x86::_Pi4x3;
3929         StubRoutines::x86::_Pi4x4_adr = (address)StubRoutines::x86::_Pi4x4;
3930         StubRoutines::x86::_ones_adr = (address)StubRoutines::x86::_ones;
3931       }
3932       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) {
3933         StubRoutines::_dexp = generate_libmExp();
3934       }
3935       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
3936         StubRoutines::_dlog = generate_libmLog();
3937       }
3938       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) {
3939         StubRoutines::_dlog10 = generate_libmLog10();
3940       }
3941       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) {
3942         StubRoutines::_dpow = generate_libmPow();
3943       }
3944       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3945         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3946         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3947         StubRoutines::_dlibm_reduce_pi04l = generate_libm_reduce_pi04l();
3948       }
3949       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3950         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3951         StubRoutines::_dlibm_sin_cos_huge = generate_libm_sin_cos_huge();
3952       }
3953       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
3954         StubRoutines::_dsin = generate_libmSin();
3955       }
3956       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3957         StubRoutines::_dcos = generate_libmCos();
3958       }
3959       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3960         StubRoutines::_dlibm_tan_cot_huge = generate_libm_tan_cot_huge();
3961         StubRoutines::_dtan = generate_libmTan();
3962       }
3963     }
3964 
3965     // Safefetch stubs.
3966     generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
3967                                                    &StubRoutines::_safefetch32_fault_pc,
3968                                                    &StubRoutines::_safefetch32_continuation_pc);
3969     StubRoutines::_safefetchN_entry           = StubRoutines::_safefetch32_entry;
3970     StubRoutines::_safefetchN_fault_pc        = StubRoutines::_safefetch32_fault_pc;
3971     StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc;
3972   }
3973 
3974   void generate_all() {
3975     // Generates all stubs and initializes the entry points
3976 
3977     // These entry points require SharedInfo::stack0 to be set up in non-core builds
3978     // and need to be relocatable, so they each fabricate a RuntimeStub internally.
3979     StubRoutines::_throw_AbstractMethodError_entry         = generate_throw_exception("AbstractMethodError throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
3980     StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
3981     StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
3982 
3983     //------------------------------------------------------------------------------------------------------------------------
3984     // entry points that are platform specific
3985 
3986     StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF);
3987     StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x80000000);
3988     StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask_long_double("vector_double_sign_mask", 0x7FFFFFFF, 0xFFFFFFFF);
3989     StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask_long_double("vector_double_sign_flip", 0x80000000, 0x00000000);
3990     StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff);
3991     StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask("vector_int_to_byte_mask", 0x000000ff);
3992     StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask("vector_int_to_short_mask", 0x0000ffff);
3993     StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32("vector_32_bit_mask", Assembler::AVX_512bit,
3994                                                                         0xFFFFFFFF, 0, 0, 0);
3995     StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32("vector_64_bit_mask", Assembler::AVX_512bit,
3996                                                                         0xFFFFFFFF, 0xFFFFFFFF, 0, 0);
3997     StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask("vector_int_shuffle_mask", 0x03020100);
3998     StubRoutines::x86::_vector_byte_shuffle_mask = generate_vector_byte_shuffle_mask("vector_byte_shuffle_mask");
3999     StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask("vector_short_shuffle_mask", 0x01000100);
4000     StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask_long_double("vector_long_shuffle_mask", 0x00000001, 0x0);
4001     StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
4002     StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask_long_double("vector_long_sign_mask", 0x80000000, 0x00000000);
4003     StubRoutines::x86::_vector_all_bits_set = generate_vector_mask("vector_all_bits_set", 0xFFFFFFFF);
4004     StubRoutines::x86::_vector_iota_indices = generate_iota_indices("iota_indices");
4005 
4006     // support for verify_oop (must happen after universe_init)
4007     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
4008 
4009     // arraycopy stubs used by compilers
4010     generate_arraycopy_stubs();
4011 
4012     // don't bother generating these AES intrinsic stubs unless global flag is set
4013     if (UseAESIntrinsics) {
4014       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // might be needed by the others
4015 
4016       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
4017       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
4018       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
4019       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
4020     }
4021 
4022     if (UseAESCTRIntrinsics) {
4023       StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
4024       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
4025     }
4026 
4027     if (UseMD5Intrinsics) {
4028       StubRoutines::_md5_implCompress = generate_md5_implCompress(false, "md5_implCompress");
4029       StubRoutines::_md5_implCompressMB = generate_md5_implCompress(true, "md5_implCompressMB");
4030     }
4031     if (UseSHA1Intrinsics) {
4032       StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask();
4033       StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask();
4034       StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
4035       StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
4036     }
4037     if (UseSHA256Intrinsics) {
4038       StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
4039       StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
4040       StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
4041       StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
4042     }
4043 
4044     // Generate GHASH intrinsics code
4045     if (UseGHASHIntrinsics) {
4046       StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
4047       StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
4048       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
4049     }
4050 
4051     BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
4052     if (bs_nm != NULL) {
4053       StubRoutines::x86::_method_entry_barrier = generate_method_entry_barrier();
4054     }
4055   }
4056 
4057 
4058  public:
4059   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
4060     if (all) {
4061       generate_all();
4062     } else {
4063       generate_initial();
4064     }
4065   }
4066 }; // end class declaration
4067 
4068 #define UCM_TABLE_MAX_ENTRIES 8
4069 void StubGenerator_generate(CodeBuffer* code, bool all) {
4070   if (UnsafeCopyMemory::_table == NULL) {
4071     UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
4072   }
4073   StubGenerator g(code, all);
4074 }