1 /*
   2  * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "nativeInst_x86.hpp"
  30 #include "oops/instanceOop.hpp"
  31 #include "oops/method.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "oops/oop.inline.hpp"
  34 #include "prims/methodHandles.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/handles.inline.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubCodeGenerator.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/thread.inline.hpp"
  41 #include "utilities/macros.hpp"
  42 #include "utilities/top.hpp"
  43 #ifdef COMPILER2
  44 #include "opto/runtime.hpp"
  45 #endif
  46 
  47 // Declaration and definition of StubGenerator (no .hpp file).
  48 // For a more detailed description of the stub routine structure
  49 // see the comment in stubRoutines.hpp
  50 
  51 #define __ _masm->
  52 #define a__ ((Assembler*)_masm)->
  53 
  54 #ifdef PRODUCT
  55 #define BLOCK_COMMENT(str) /* nothing */
  56 #else
  57 #define BLOCK_COMMENT(str) __ block_comment(str)
  58 #endif
  59 
  60 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  61 
  62 const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions
  63 const int FPU_CNTRL_WRD_MASK = 0xFFFF;
  64 
  65 // -------------------------------------------------------------------------------------------------------------------------
  66 // Stub Code definitions
  67 
  68 static address handle_unsafe_access() {
  69   JavaThread* thread = JavaThread::current();
  70   address pc  = thread->saved_exception_pc();
  71   // pc is the instruction which we must emulate
  72   // doing a no-op is fine:  return garbage from the load
  73   // therefore, compute npc
  74   address npc = Assembler::locate_next_instruction(pc);
  75 
  76   // request an async exception
  77   thread->set_pending_unsafe_access_error();
  78 
  79   // return address of next instruction to execute
  80   return npc;
  81 }
  82 
  83 class StubGenerator: public StubCodeGenerator {
  84  private:
  85 
  86 #ifdef PRODUCT
  87 #define inc_counter_np(counter) ((void)0)
  88 #else
  89   void inc_counter_np_(int& counter) {
  90     __ incrementl(ExternalAddress((address)&counter));
  91   }
  92 #define inc_counter_np(counter) \
  93   BLOCK_COMMENT("inc_counter " #counter); \
  94   inc_counter_np_(counter);
  95 #endif //PRODUCT
  96 
  97   void inc_copy_counter_np(BasicType t) {
  98 #ifndef PRODUCT
  99     switch (t) {
 100     case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
 101     case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
 102     case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
 103     case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
 104     case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
 105     }
 106     ShouldNotReachHere();
 107 #endif //PRODUCT
 108   }
 109 
 110   //------------------------------------------------------------------------------------------------------------------------
 111   // Call stubs are used to call Java from C
 112   //
 113   //    [ return_from_Java     ] <--- rsp
 114   //    [ argument word n      ]
 115   //      ...
 116   // -N [ argument word 1      ]
 117   // -7 [ Possible padding for stack alignment ]
 118   // -6 [ Possible padding for stack alignment ]
 119   // -5 [ Possible padding for stack alignment ]
 120   // -4 [ mxcsr save           ] <--- rsp_after_call
 121   // -3 [ saved rbx,            ]
 122   // -2 [ saved rsi            ]
 123   // -1 [ saved rdi            ]
 124   //  0 [ saved rbp,            ] <--- rbp,
 125   //  1 [ return address       ]
 126   //  2 [ ptr. to call wrapper ]
 127   //  3 [ result               ]
 128   //  4 [ result_type          ]
 129   //  5 [ method               ]
 130   //  6 [ entry_point          ]
 131   //  7 [ parameters           ]
 132   //  8 [ parameter_size       ]
 133   //  9 [ thread               ]
 134 
 135 
 136   address generate_call_stub(address& return_address) {
 137     StubCodeMark mark(this, "StubRoutines", "call_stub");
 138     address start = __ pc();
 139 
 140     // stub code parameters / addresses
 141     assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
 142     bool  sse_save = false;
 143     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
 144     const int     locals_count_in_bytes  (4*wordSize);
 145     const Address mxcsr_save    (rbp, -4 * wordSize);
 146     const Address saved_rbx     (rbp, -3 * wordSize);
 147     const Address saved_rsi     (rbp, -2 * wordSize);
 148     const Address saved_rdi     (rbp, -1 * wordSize);
 149     const Address result        (rbp,  3 * wordSize);
 150     const Address result_type   (rbp,  4 * wordSize);
 151     const Address method        (rbp,  5 * wordSize);
 152     const Address entry_point   (rbp,  6 * wordSize);
 153     const Address parameters    (rbp,  7 * wordSize);
 154     const Address parameter_size(rbp,  8 * wordSize);
 155     const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!
 156     sse_save =  UseSSE > 0;
 157 
 158     // stub code
 159     __ enter();
 160     __ movptr(rcx, parameter_size);              // parameter counter
 161     __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes
 162     __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves
 163     __ subptr(rsp, rcx);
 164     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
 165 
 166     // save rdi, rsi, & rbx, according to C calling conventions
 167     __ movptr(saved_rdi, rdi);
 168     __ movptr(saved_rsi, rsi);
 169     __ movptr(saved_rbx, rbx);
 170     // save and initialize %mxcsr
 171     if (sse_save) {
 172       Label skip_ldmx;
 173       __ stmxcsr(mxcsr_save);
 174       __ movl(rax, mxcsr_save);
 175       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 176       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 177       __ cmp32(rax, mxcsr_std);
 178       __ jcc(Assembler::equal, skip_ldmx);
 179       __ ldmxcsr(mxcsr_std);
 180       __ bind(skip_ldmx);
 181     }
 182 
 183     // make sure the control word is correct.
 184     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 185 
 186 #ifdef ASSERT
 187     // make sure we have no pending exceptions
 188     { Label L;
 189       __ movptr(rcx, thread);
 190       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 191       __ jcc(Assembler::equal, L);
 192       __ stop("StubRoutines::call_stub: entered with pending exception");
 193       __ bind(L);
 194     }
 195 #endif
 196 
 197     // pass parameters if any
 198     BLOCK_COMMENT("pass parameters if any");
 199     Label parameters_done;
 200     __ movl(rcx, parameter_size);  // parameter counter
 201     __ testl(rcx, rcx);
 202     __ jcc(Assembler::zero, parameters_done);
 203 
 204     // parameter passing loop
 205 
 206     Label loop;
 207     // Copy Java parameters in reverse order (receiver last)
 208     // Note that the argument order is inverted in the process
 209     // source is rdx[rcx: N-1..0]
 210     // dest   is rsp[rbx: 0..N-1]
 211 
 212     __ movptr(rdx, parameters);          // parameter pointer
 213     __ xorptr(rbx, rbx);
 214 
 215     __ BIND(loop);
 216 
 217     // get parameter
 218     __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
 219     __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
 220                     Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter
 221     __ increment(rbx);
 222     __ decrement(rcx);
 223     __ jcc(Assembler::notZero, loop);
 224 
 225     // call Java function
 226     __ BIND(parameters_done);
 227     __ movptr(rbx, method);           // get Method*
 228     __ movptr(rax, entry_point);      // get entry_point
 229     __ mov(rsi, rsp);                 // set sender sp
 230     BLOCK_COMMENT("call Java function");
 231     __ call(rax);
 232 
 233     BLOCK_COMMENT("call_stub_return_address:");
 234     return_address = __ pc();
 235 
 236 #ifdef COMPILER2
 237     {
 238       Label L_skip;
 239       if (UseSSE >= 2) {
 240         __ verify_FPU(0, "call_stub_return");
 241       } else {
 242         for (int i = 1; i < 8; i++) {
 243           __ ffree(i);
 244         }
 245 
 246         // UseSSE <= 1 so double result should be left on TOS
 247         __ movl(rsi, result_type);
 248         __ cmpl(rsi, T_DOUBLE);
 249         __ jcc(Assembler::equal, L_skip);
 250         if (UseSSE == 0) {
 251           // UseSSE == 0 so float result should be left on TOS
 252           __ cmpl(rsi, T_FLOAT);
 253           __ jcc(Assembler::equal, L_skip);
 254         }
 255         __ ffree(0);
 256       }
 257       __ BIND(L_skip);
 258     }
 259 #endif // COMPILER2
 260 
 261     // store result depending on type
 262     // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 263     __ movptr(rdi, result);
 264     Label is_long, is_float, is_double, exit;
 265     __ movl(rsi, result_type);
 266     __ cmpl(rsi, T_LONG);
 267     __ jcc(Assembler::equal, is_long);
 268     __ cmpl(rsi, T_FLOAT);
 269     __ jcc(Assembler::equal, is_float);
 270     __ cmpl(rsi, T_DOUBLE);
 271     __ jcc(Assembler::equal, is_double);
 272 
 273     // handle T_INT case
 274     __ movl(Address(rdi, 0), rax);
 275     __ BIND(exit);
 276 
 277     // check that FPU stack is empty
 278     __ verify_FPU(0, "generate_call_stub");
 279 
 280     // pop parameters
 281     __ lea(rsp, rsp_after_call);
 282 
 283     // restore %mxcsr
 284     if (sse_save) {
 285       __ ldmxcsr(mxcsr_save);
 286     }
 287 
 288     // restore rdi, rsi and rbx,
 289     __ movptr(rbx, saved_rbx);
 290     __ movptr(rsi, saved_rsi);
 291     __ movptr(rdi, saved_rdi);
 292     __ addptr(rsp, 4*wordSize);
 293 
 294     // return
 295     __ pop(rbp);
 296     __ ret(0);
 297 
 298     // handle return types different from T_INT
 299     __ BIND(is_long);
 300     __ movl(Address(rdi, 0 * wordSize), rax);
 301     __ movl(Address(rdi, 1 * wordSize), rdx);
 302     __ jmp(exit);
 303 
 304     __ BIND(is_float);
 305     // interpreter uses xmm0 for return values
 306     if (UseSSE >= 1) {
 307       __ movflt(Address(rdi, 0), xmm0);
 308     } else {
 309       __ fstp_s(Address(rdi, 0));
 310     }
 311     __ jmp(exit);
 312 
 313     __ BIND(is_double);
 314     // interpreter uses xmm0 for return values
 315     if (UseSSE >= 2) {
 316       __ movdbl(Address(rdi, 0), xmm0);
 317     } else {
 318       __ fstp_d(Address(rdi, 0));
 319     }
 320     __ jmp(exit);
 321 
 322     return start;
 323   }
 324 
 325 
 326   //------------------------------------------------------------------------------------------------------------------------
 327   // Return point for a Java call if there's an exception thrown in Java code.
 328   // The exception is caught and transformed into a pending exception stored in
 329   // JavaThread that can be tested from within the VM.
 330   //
 331   // Note: Usually the parameters are removed by the callee. In case of an exception
 332   //       crossing an activation frame boundary, that is not the case if the callee
 333   //       is compiled code => need to setup the rsp.
 334   //
 335   // rax,: exception oop
 336 
 337   address generate_catch_exception() {
 338     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 339     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
 340     const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!
 341     address start = __ pc();
 342 
 343     // get thread directly
 344     __ movptr(rcx, thread);
 345 #ifdef ASSERT
 346     // verify that threads correspond
 347     { Label L;
 348       __ get_thread(rbx);
 349       __ cmpptr(rbx, rcx);
 350       __ jcc(Assembler::equal, L);
 351       __ stop("StubRoutines::catch_exception: threads must correspond");
 352       __ bind(L);
 353     }
 354 #endif
 355     // set pending exception
 356     __ verify_oop(rax);
 357     __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );
 358     __ lea(Address(rcx, Thread::exception_file_offset   ()),
 359            ExternalAddress((address)__FILE__));
 360     __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );
 361     // complete return to VM
 362     assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
 363     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 364 
 365     return start;
 366   }
 367 
 368 
 369   //------------------------------------------------------------------------------------------------------------------------
 370   // Continuation point for runtime calls returning with a pending exception.
 371   // The pending exception check happened in the runtime or native call stub.
 372   // The pending exception in Thread is converted into a Java-level exception.
 373   //
 374   // Contract with Java-level exception handlers:
 375   // rax: exception
 376   // rdx: throwing pc
 377   //
 378   // NOTE: At entry of this stub, exception-pc must be on stack !!
 379 
 380   address generate_forward_exception() {
 381     StubCodeMark mark(this, "StubRoutines", "forward exception");
 382     address start = __ pc();
 383     const Register thread = rcx;
 384 
 385     // other registers used in this stub
 386     const Register exception_oop = rax;
 387     const Register handler_addr  = rbx;
 388     const Register exception_pc  = rdx;
 389 
 390     // Upon entry, the sp points to the return address returning into Java
 391     // (interpreted or compiled) code; i.e., the return address becomes the
 392     // throwing pc.
 393     //
 394     // Arguments pushed before the runtime call are still on the stack but
 395     // the exception handler will reset the stack pointer -> ignore them.
 396     // A potential result in registers can be ignored as well.
 397 
 398 #ifdef ASSERT
 399     // make sure this code is only executed if there is a pending exception
 400     { Label L;
 401       __ get_thread(thread);
 402       __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 403       __ jcc(Assembler::notEqual, L);
 404       __ stop("StubRoutines::forward exception: no pending exception (1)");
 405       __ bind(L);
 406     }
 407 #endif
 408 
 409     // compute exception handler into rbx,
 410     __ get_thread(thread);
 411     __ movptr(exception_pc, Address(rsp, 0));
 412     BLOCK_COMMENT("call exception_handler_for_return_address");
 413     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
 414     __ mov(handler_addr, rax);
 415 
 416     // setup rax & rdx, remove return address & clear pending exception
 417     __ get_thread(thread);
 418     __ pop(exception_pc);
 419     __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
 420     __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
 421 
 422 #ifdef ASSERT
 423     // make sure exception is set
 424     { Label L;
 425       __ testptr(exception_oop, exception_oop);
 426       __ jcc(Assembler::notEqual, L);
 427       __ stop("StubRoutines::forward exception: no pending exception (2)");
 428       __ bind(L);
 429     }
 430 #endif
 431 
 432     // Verify that there is really a valid exception in RAX.
 433     __ verify_oop(exception_oop);
 434 
 435     // continue at exception handler (return address removed)
 436     // rax: exception
 437     // rbx: exception handler
 438     // rdx: throwing pc
 439     __ jmp(handler_addr);
 440 
 441     return start;
 442   }
 443 
 444 
 445   //----------------------------------------------------------------------------------------------------
 446   // Support for jint Atomic::xchg(jint exchange_value, volatile jint* dest)
 447   //
 448   // xchg exists as far back as 8086, lock needed for MP only
 449   // Stack layout immediately after call:
 450   //
 451   // 0 [ret addr ] <--- rsp
 452   // 1 [  ex     ]
 453   // 2 [  dest   ]
 454   //
 455   // Result:   *dest <- ex, return (old *dest)
 456   //
 457   // Note: win32 does not currently use this code
 458 
 459   address generate_atomic_xchg() {
 460     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 461     address start = __ pc();
 462 
 463     __ push(rdx);
 464     Address exchange(rsp, 2 * wordSize);
 465     Address dest_addr(rsp, 3 * wordSize);
 466     __ movl(rax, exchange);
 467     __ movptr(rdx, dest_addr);
 468     __ xchgl(rax, Address(rdx, 0));
 469     __ pop(rdx);
 470     __ ret(0);
 471 
 472     return start;
 473   }
 474 
 475   //----------------------------------------------------------------------------------------------------
 476   // Support for void verify_mxcsr()
 477   //
 478   // This routine is used with -Xcheck:jni to verify that native
 479   // JNI code does not return to Java code without restoring the
 480   // MXCSR register to our expected state.
 481 
 482 
 483   address generate_verify_mxcsr() {
 484     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 485     address start = __ pc();
 486 
 487     const Address mxcsr_save(rsp, 0);
 488 
 489     if (CheckJNICalls && UseSSE > 0 ) {
 490       Label ok_ret;
 491       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 492       __ push(rax);
 493       __ subptr(rsp, wordSize);      // allocate a temp location
 494       __ stmxcsr(mxcsr_save);
 495       __ movl(rax, mxcsr_save);
 496       __ andl(rax, MXCSR_MASK);
 497       __ cmp32(rax, mxcsr_std);
 498       __ jcc(Assembler::equal, ok_ret);
 499 
 500       __ warn("MXCSR changed by native JNI code.");
 501 
 502       __ ldmxcsr(mxcsr_std);
 503 
 504       __ bind(ok_ret);
 505       __ addptr(rsp, wordSize);
 506       __ pop(rax);
 507     }
 508 
 509     __ ret(0);
 510 
 511     return start;
 512   }
 513 
 514 
 515   //---------------------------------------------------------------------------
 516   // Support for void verify_fpu_cntrl_wrd()
 517   //
 518   // This routine is used with -Xcheck:jni to verify that native
 519   // JNI code does not return to Java code without restoring the
 520   // FP control word to our expected state.
 521 
 522   address generate_verify_fpu_cntrl_wrd() {
 523     StubCodeMark mark(this, "StubRoutines", "verify_spcw");
 524     address start = __ pc();
 525 
 526     const Address fpu_cntrl_wrd_save(rsp, 0);
 527 
 528     if (CheckJNICalls) {
 529       Label ok_ret;
 530       __ push(rax);
 531       __ subptr(rsp, wordSize);      // allocate a temp location
 532       __ fnstcw(fpu_cntrl_wrd_save);
 533       __ movl(rax, fpu_cntrl_wrd_save);
 534       __ andl(rax, FPU_CNTRL_WRD_MASK);
 535       ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());
 536       __ cmp32(rax, fpu_std);
 537       __ jcc(Assembler::equal, ok_ret);
 538 
 539       __ warn("Floating point control word changed by native JNI code.");
 540 
 541       __ fldcw(fpu_std);
 542 
 543       __ bind(ok_ret);
 544       __ addptr(rsp, wordSize);
 545       __ pop(rax);
 546     }
 547 
 548     __ ret(0);
 549 
 550     return start;
 551   }
 552 
 553   //---------------------------------------------------------------------------
 554   // Wrapper for slow-case handling of double-to-integer conversion
 555   // d2i or f2i fast case failed either because it is nan or because
 556   // of under/overflow.
 557   // Input:  FPU TOS: float value
 558   // Output: rax, (rdx): integer (long) result
 559 
 560   address generate_d2i_wrapper(BasicType t, address fcn) {
 561     StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
 562     address start = __ pc();
 563 
 564   // Capture info about frame layout
 565   enum layout { FPUState_off         = 0,
 566                 rbp_off              = FPUStateSizeInWords,
 567                 rdi_off,
 568                 rsi_off,
 569                 rcx_off,
 570                 rbx_off,
 571                 saved_argument_off,
 572                 saved_argument_off2, // 2nd half of double
 573                 framesize
 574   };
 575 
 576   assert(FPUStateSizeInWords == 27, "update stack layout");
 577 
 578     // Save outgoing argument to stack across push_FPU_state()
 579     __ subptr(rsp, wordSize * 2);
 580     __ fstp_d(Address(rsp, 0));
 581 
 582     // Save CPU & FPU state
 583     __ push(rbx);
 584     __ push(rcx);
 585     __ push(rsi);
 586     __ push(rdi);
 587     __ push(rbp);
 588     __ push_FPU_state();
 589 
 590     // push_FPU_state() resets the FP top of stack
 591     // Load original double into FP top of stack
 592     __ fld_d(Address(rsp, saved_argument_off * wordSize));
 593     // Store double into stack as outgoing argument
 594     __ subptr(rsp, wordSize*2);
 595     __ fst_d(Address(rsp, 0));
 596 
 597     // Prepare FPU for doing math in C-land
 598     __ empty_FPU_stack();
 599     // Call the C code to massage the double.  Result in EAX
 600     if (t == T_INT)
 601       { BLOCK_COMMENT("SharedRuntime::d2i"); }
 602     else if (t == T_LONG)
 603       { BLOCK_COMMENT("SharedRuntime::d2l"); }
 604     __ call_VM_leaf( fcn, 2 );
 605 
 606     // Restore CPU & FPU state
 607     __ pop_FPU_state();
 608     __ pop(rbp);
 609     __ pop(rdi);
 610     __ pop(rsi);
 611     __ pop(rcx);
 612     __ pop(rbx);
 613     __ addptr(rsp, wordSize * 2);
 614 
 615     __ ret(0);
 616 
 617     return start;
 618   }
 619 
 620 
 621   //---------------------------------------------------------------------------
 622   // The following routine generates a subroutine to throw an asynchronous
 623   // UnknownError when an unsafe access gets a fault that could not be
 624   // reasonably prevented by the programmer.  (Example: SIGBUS/OBJERR.)
 625   address generate_handler_for_unsafe_access() {
 626     StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
 627     address start = __ pc();
 628 
 629     __ push(0);                       // hole for return address-to-be
 630     __ pusha();                       // push registers
 631     Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
 632     BLOCK_COMMENT("call handle_unsafe_access");
 633     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
 634     __ movptr(next_pc, rax);          // stuff next address
 635     __ popa();
 636     __ ret(0);                        // jump to next address
 637 
 638     return start;
 639   }
 640 
 641 
 642   //----------------------------------------------------------------------------------------------------
 643   // Non-destructive plausibility checks for oops
 644 
 645   address generate_verify_oop() {
 646     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 647     address start = __ pc();
 648 
 649     // Incoming arguments on stack after saving rax,:
 650     //
 651     // [tos    ]: saved rdx
 652     // [tos + 1]: saved EFLAGS
 653     // [tos + 2]: return address
 654     // [tos + 3]: char* error message
 655     // [tos + 4]: oop   object to verify
 656     // [tos + 5]: saved rax, - saved by caller and bashed
 657 
 658     Label exit, error;
 659     __ pushf();
 660     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 661     __ push(rdx);                                // save rdx
 662     // make sure object is 'reasonable'
 663     __ movptr(rax, Address(rsp, 4 * wordSize));    // get object
 664     __ testptr(rax, rax);
 665     __ jcc(Assembler::zero, exit);               // if obj is NULL it is ok
 666 
 667     // Check if the oop is in the right area of memory
 668     const int oop_mask = Universe::verify_oop_mask();
 669     const int oop_bits = Universe::verify_oop_bits();
 670     __ mov(rdx, rax);
 671     __ andptr(rdx, oop_mask);
 672     __ cmpptr(rdx, oop_bits);
 673     __ jcc(Assembler::notZero, error);
 674 
 675     // make sure klass is 'reasonable', which is not zero.
 676     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
 677     __ testptr(rax, rax);
 678     __ jcc(Assembler::zero, error);              // if klass is NULL it is broken
 679 
 680     // return if everything seems ok
 681     __ bind(exit);
 682     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 683     __ pop(rdx);                                 // restore rdx
 684     __ popf();                                   // restore EFLAGS
 685     __ ret(3 * wordSize);                        // pop arguments
 686 
 687     // handle errors
 688     __ bind(error);
 689     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 690     __ pop(rdx);                                 // get saved rdx back
 691     __ popf();                                   // get saved EFLAGS off stack -- will be ignored
 692     __ pusha();                                  // push registers (eip = return address & msg are already pushed)
 693     BLOCK_COMMENT("call MacroAssembler::debug");
 694     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 695     __ popa();
 696     __ ret(3 * wordSize);                        // pop arguments
 697     return start;
 698   }
 699 
 700   //
 701   //  Generate pre-barrier for array stores
 702   //
 703   //  Input:
 704   //     start   -  starting address
 705   //     count   -  element count
 706   void  gen_write_ref_array_pre_barrier(Register start, Register count, bool uninitialized_target) {
 707     assert_different_registers(start, count);
 708     BarrierSet* bs = Universe::heap()->barrier_set();
 709     switch (bs->kind()) {
 710       case BarrierSet::G1SATBCT:
 711       case BarrierSet::G1SATBCTLogging:
 712       case BarrierSet::ShenandoahBarrierSet:
 713         // With G1, don't generate the call if we statically know that the target in uninitialized
 714         if (!uninitialized_target) {
 715            __ pusha();                      // push registers
 716            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre),
 717                            start, count);
 718            __ popa();
 719          }
 720         break;
 721       case BarrierSet::CardTableModRef:
 722       case BarrierSet::CardTableExtension:
 723       case BarrierSet::ModRef:
 724         break;
 725       default      :
 726         ShouldNotReachHere();
 727 
 728     }
 729   }
 730 
 731 
 732   //
 733   // Generate a post-barrier for an array store
 734   //
 735   //     start    -  starting address
 736   //     count    -  element count
 737   //
 738   //  The two input registers are overwritten.
 739   //
 740   void  gen_write_ref_array_post_barrier(Register start, Register count) {
 741     BarrierSet* bs = Universe::heap()->barrier_set();
 742     assert_different_registers(start, count);
 743     switch (bs->kind()) {
 744       case BarrierSet::G1SATBCT:
 745       case BarrierSet::G1SATBCTLogging:
 746       case BarrierSet::ShenandoahBarrierSet:
 747         {
 748           __ pusha();                      // push registers
 749           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post),
 750                           start, count);
 751           __ popa();
 752         }
 753         break;
 754 
 755       case BarrierSet::CardTableModRef:
 756       case BarrierSet::CardTableExtension:
 757         {
 758           CardTableModRefBS* ct = (CardTableModRefBS*)bs;
 759           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
 760 
 761           Label L_loop;
 762           const Register end = count;  // elements count; end == start+count-1
 763           assert_different_registers(start, end);
 764 
 765           __ lea(end,  Address(start, count, Address::times_ptr, -wordSize));
 766           __ shrptr(start, CardTableModRefBS::card_shift);
 767           __ shrptr(end,   CardTableModRefBS::card_shift);
 768           __ subptr(end, start); // end --> count
 769         __ BIND(L_loop);
 770           intptr_t disp = (intptr_t) ct->byte_map_base;
 771           Address cardtable(start, count, Address::times_1, disp);
 772           __ movb(cardtable, 0);
 773           __ decrement(count);
 774           __ jcc(Assembler::greaterEqual, L_loop);
 775         }
 776         break;
 777       case BarrierSet::ModRef:
 778         break;
 779       default      :
 780         ShouldNotReachHere();
 781 
 782     }
 783   }
 784 
 785 
 786   // Copy 64 bytes chunks
 787   //
 788   // Inputs:
 789   //   from        - source array address
 790   //   to_from     - destination array address - from
 791   //   qword_count - 8-bytes element count, negative
 792   //
 793   void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
 794     assert( UseSSE >= 2, "supported cpu only" );
 795     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 796     // Copy 64-byte chunks
 797     __ jmpb(L_copy_64_bytes);
 798     __ align(OptoLoopAlignment);
 799   __ BIND(L_copy_64_bytes_loop);
 800 
 801     if (UseUnalignedLoadStores) {
 802       if (UseAVX >= 2) {
 803         __ vmovdqu(xmm0, Address(from,  0));
 804         __ vmovdqu(Address(from, to_from, Address::times_1,  0), xmm0);
 805         __ vmovdqu(xmm1, Address(from, 32));
 806         __ vmovdqu(Address(from, to_from, Address::times_1, 32), xmm1);
 807       } else {
 808         __ movdqu(xmm0, Address(from, 0));
 809         __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
 810         __ movdqu(xmm1, Address(from, 16));
 811         __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
 812         __ movdqu(xmm2, Address(from, 32));
 813         __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
 814         __ movdqu(xmm3, Address(from, 48));
 815         __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
 816       }
 817     } else {
 818       __ movq(xmm0, Address(from, 0));
 819       __ movq(Address(from, to_from, Address::times_1, 0), xmm0);
 820       __ movq(xmm1, Address(from, 8));
 821       __ movq(Address(from, to_from, Address::times_1, 8), xmm1);
 822       __ movq(xmm2, Address(from, 16));
 823       __ movq(Address(from, to_from, Address::times_1, 16), xmm2);
 824       __ movq(xmm3, Address(from, 24));
 825       __ movq(Address(from, to_from, Address::times_1, 24), xmm3);
 826       __ movq(xmm4, Address(from, 32));
 827       __ movq(Address(from, to_from, Address::times_1, 32), xmm4);
 828       __ movq(xmm5, Address(from, 40));
 829       __ movq(Address(from, to_from, Address::times_1, 40), xmm5);
 830       __ movq(xmm6, Address(from, 48));
 831       __ movq(Address(from, to_from, Address::times_1, 48), xmm6);
 832       __ movq(xmm7, Address(from, 56));
 833       __ movq(Address(from, to_from, Address::times_1, 56), xmm7);
 834     }
 835 
 836     __ addl(from, 64);
 837   __ BIND(L_copy_64_bytes);
 838     __ subl(qword_count, 8);
 839     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 840 
 841     if (UseUnalignedLoadStores && (UseAVX >= 2)) {
 842       // clean upper bits of YMM registers
 843       __ vpxor(xmm0, xmm0);
 844       __ vpxor(xmm1, xmm1);
 845     }
 846     __ addl(qword_count, 8);
 847     __ jccb(Assembler::zero, L_exit);
 848     //
 849     // length is too short, just copy qwords
 850     //
 851   __ BIND(L_copy_8_bytes);
 852     __ movq(xmm0, Address(from, 0));
 853     __ movq(Address(from, to_from, Address::times_1), xmm0);
 854     __ addl(from, 8);
 855     __ decrement(qword_count);
 856     __ jcc(Assembler::greater, L_copy_8_bytes);
 857   __ BIND(L_exit);
 858   }
 859 
 860   // Copy 64 bytes chunks
 861   //
 862   // Inputs:
 863   //   from        - source array address
 864   //   to_from     - destination array address - from
 865   //   qword_count - 8-bytes element count, negative
 866   //
 867   void mmx_copy_forward(Register from, Register to_from, Register qword_count) {
 868     assert( VM_Version::supports_mmx(), "supported cpu only" );
 869     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 870     // Copy 64-byte chunks
 871     __ jmpb(L_copy_64_bytes);
 872     __ align(OptoLoopAlignment);
 873   __ BIND(L_copy_64_bytes_loop);
 874     __ movq(mmx0, Address(from, 0));
 875     __ movq(mmx1, Address(from, 8));
 876     __ movq(mmx2, Address(from, 16));
 877     __ movq(Address(from, to_from, Address::times_1, 0), mmx0);
 878     __ movq(mmx3, Address(from, 24));
 879     __ movq(Address(from, to_from, Address::times_1, 8), mmx1);
 880     __ movq(mmx4, Address(from, 32));
 881     __ movq(Address(from, to_from, Address::times_1, 16), mmx2);
 882     __ movq(mmx5, Address(from, 40));
 883     __ movq(Address(from, to_from, Address::times_1, 24), mmx3);
 884     __ movq(mmx6, Address(from, 48));
 885     __ movq(Address(from, to_from, Address::times_1, 32), mmx4);
 886     __ movq(mmx7, Address(from, 56));
 887     __ movq(Address(from, to_from, Address::times_1, 40), mmx5);
 888     __ movq(Address(from, to_from, Address::times_1, 48), mmx6);
 889     __ movq(Address(from, to_from, Address::times_1, 56), mmx7);
 890     __ addptr(from, 64);
 891   __ BIND(L_copy_64_bytes);
 892     __ subl(qword_count, 8);
 893     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 894     __ addl(qword_count, 8);
 895     __ jccb(Assembler::zero, L_exit);
 896     //
 897     // length is too short, just copy qwords
 898     //
 899   __ BIND(L_copy_8_bytes);
 900     __ movq(mmx0, Address(from, 0));
 901     __ movq(Address(from, to_from, Address::times_1), mmx0);
 902     __ addptr(from, 8);
 903     __ decrement(qword_count);
 904     __ jcc(Assembler::greater, L_copy_8_bytes);
 905   __ BIND(L_exit);
 906     __ emms();
 907   }
 908 
 909   address generate_disjoint_copy(BasicType t, bool aligned,
 910                                  Address::ScaleFactor sf,
 911                                  address* entry, const char *name,
 912                                  bool dest_uninitialized = false) {
 913     __ align(CodeEntryAlignment);
 914     StubCodeMark mark(this, "StubRoutines", name);
 915     address start = __ pc();
 916 
 917     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
 918     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
 919 
 920     int shift = Address::times_ptr - sf;
 921 
 922     const Register from     = rsi;  // source array address
 923     const Register to       = rdi;  // destination array address
 924     const Register count    = rcx;  // elements count
 925     const Register to_from  = to;   // (to - from)
 926     const Register saved_to = rdx;  // saved destination array address
 927 
 928     __ enter(); // required for proper stackwalking of RuntimeStub frame
 929     __ push(rsi);
 930     __ push(rdi);
 931     __ movptr(from , Address(rsp, 12+ 4));
 932     __ movptr(to   , Address(rsp, 12+ 8));
 933     __ movl(count, Address(rsp, 12+ 12));
 934 
 935     if (entry != NULL) {
 936       *entry = __ pc(); // Entry point from conjoint arraycopy stub.
 937       BLOCK_COMMENT("Entry:");
 938     }
 939 
 940     if (t == T_OBJECT) {
 941       __ testl(count, count);
 942       __ jcc(Assembler::zero, L_0_count);
 943       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
 944       __ mov(saved_to, to);          // save 'to'
 945     }
 946 
 947     __ subptr(to, from); // to --> to_from
 948     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
 949     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
 950     if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 951       // align source address at 4 bytes address boundary
 952       if (t == T_BYTE) {
 953         // One byte misalignment happens only for byte arrays
 954         __ testl(from, 1);
 955         __ jccb(Assembler::zero, L_skip_align1);
 956         __ movb(rax, Address(from, 0));
 957         __ movb(Address(from, to_from, Address::times_1, 0), rax);
 958         __ increment(from);
 959         __ decrement(count);
 960       __ BIND(L_skip_align1);
 961       }
 962       // Two bytes misalignment happens only for byte and short (char) arrays
 963       __ testl(from, 2);
 964       __ jccb(Assembler::zero, L_skip_align2);
 965       __ movw(rax, Address(from, 0));
 966       __ movw(Address(from, to_from, Address::times_1, 0), rax);
 967       __ addptr(from, 2);
 968       __ subl(count, 1<<(shift-1));
 969     __ BIND(L_skip_align2);
 970     }
 971     if (!VM_Version::supports_mmx()) {
 972       __ mov(rax, count);      // save 'count'
 973       __ shrl(count, shift); // bytes count
 974       __ addptr(to_from, from);// restore 'to'
 975       __ rep_mov();
 976       __ subptr(to_from, from);// restore 'to_from'
 977       __ mov(count, rax);      // restore 'count'
 978       __ jmpb(L_copy_2_bytes); // all dwords were copied
 979     } else {
 980       if (!UseUnalignedLoadStores) {
 981         // align to 8 bytes, we know we are 4 byte aligned to start
 982         __ testptr(from, 4);
 983         __ jccb(Assembler::zero, L_copy_64_bytes);
 984         __ movl(rax, Address(from, 0));
 985         __ movl(Address(from, to_from, Address::times_1, 0), rax);
 986         __ addptr(from, 4);
 987         __ subl(count, 1<<shift);
 988       }
 989     __ BIND(L_copy_64_bytes);
 990       __ mov(rax, count);
 991       __ shrl(rax, shift+1);  // 8 bytes chunk count
 992       //
 993       // Copy 8-byte chunks through MMX registers, 8 per iteration of the loop
 994       //
 995       if (UseXMMForArrayCopy) {
 996         xmm_copy_forward(from, to_from, rax);
 997       } else {
 998         mmx_copy_forward(from, to_from, rax);
 999       }
1000     }
1001     // copy tailing dword
1002   __ BIND(L_copy_4_bytes);
1003     __ testl(count, 1<<shift);
1004     __ jccb(Assembler::zero, L_copy_2_bytes);
1005     __ movl(rax, Address(from, 0));
1006     __ movl(Address(from, to_from, Address::times_1, 0), rax);
1007     if (t == T_BYTE || t == T_SHORT) {
1008       __ addptr(from, 4);
1009     __ BIND(L_copy_2_bytes);
1010       // copy tailing word
1011       __ testl(count, 1<<(shift-1));
1012       __ jccb(Assembler::zero, L_copy_byte);
1013       __ movw(rax, Address(from, 0));
1014       __ movw(Address(from, to_from, Address::times_1, 0), rax);
1015       if (t == T_BYTE) {
1016         __ addptr(from, 2);
1017       __ BIND(L_copy_byte);
1018         // copy tailing byte
1019         __ testl(count, 1);
1020         __ jccb(Assembler::zero, L_exit);
1021         __ movb(rax, Address(from, 0));
1022         __ movb(Address(from, to_from, Address::times_1, 0), rax);
1023       __ BIND(L_exit);
1024       } else {
1025       __ BIND(L_copy_byte);
1026       }
1027     } else {
1028     __ BIND(L_copy_2_bytes);
1029     }
1030 
1031     if (t == T_OBJECT) {
1032       __ movl(count, Address(rsp, 12+12)); // reread 'count'
1033       __ mov(to, saved_to); // restore 'to'
1034       gen_write_ref_array_post_barrier(to, count);
1035     __ BIND(L_0_count);
1036     }
1037     inc_copy_counter_np(t);
1038     __ pop(rdi);
1039     __ pop(rsi);
1040     __ leave(); // required for proper stackwalking of RuntimeStub frame
1041     __ xorptr(rax, rax); // return 0
1042     __ ret(0);
1043     return start;
1044   }
1045 
1046 
1047   address generate_fill(BasicType t, bool aligned, const char *name) {
1048     __ align(CodeEntryAlignment);
1049     StubCodeMark mark(this, "StubRoutines", name);
1050     address start = __ pc();
1051 
1052     BLOCK_COMMENT("Entry:");
1053 
1054     const Register to       = rdi;  // source array address
1055     const Register value    = rdx;  // value
1056     const Register count    = rsi;  // elements count
1057 
1058     __ enter(); // required for proper stackwalking of RuntimeStub frame
1059     __ push(rsi);
1060     __ push(rdi);
1061     __ movptr(to   , Address(rsp, 12+ 4));
1062     __ movl(value, Address(rsp, 12+ 8));
1063     __ movl(count, Address(rsp, 12+ 12));
1064 
1065     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1066 
1067     __ pop(rdi);
1068     __ pop(rsi);
1069     __ leave(); // required for proper stackwalking of RuntimeStub frame
1070     __ ret(0);
1071     return start;
1072   }
1073 
1074   address generate_conjoint_copy(BasicType t, bool aligned,
1075                                  Address::ScaleFactor sf,
1076                                  address nooverlap_target,
1077                                  address* entry, const char *name,
1078                                  bool dest_uninitialized = false) {
1079     __ align(CodeEntryAlignment);
1080     StubCodeMark mark(this, "StubRoutines", name);
1081     address start = __ pc();
1082 
1083     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
1084     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
1085 
1086     int shift = Address::times_ptr - sf;
1087 
1088     const Register src   = rax;  // source array address
1089     const Register dst   = rdx;  // destination array address
1090     const Register from  = rsi;  // source array address
1091     const Register to    = rdi;  // destination array address
1092     const Register count = rcx;  // elements count
1093     const Register end   = rax;  // array end address
1094 
1095     __ enter(); // required for proper stackwalking of RuntimeStub frame
1096     __ push(rsi);
1097     __ push(rdi);
1098     __ movptr(src  , Address(rsp, 12+ 4));   // from
1099     __ movptr(dst  , Address(rsp, 12+ 8));   // to
1100     __ movl2ptr(count, Address(rsp, 12+12)); // count
1101 
1102     if (entry != NULL) {
1103       *entry = __ pc(); // Entry point from generic arraycopy stub.
1104       BLOCK_COMMENT("Entry:");
1105     }
1106 
1107     // nooverlap_target expects arguments in rsi and rdi.
1108     __ mov(from, src);
1109     __ mov(to  , dst);
1110 
1111     // arrays overlap test: dispatch to disjoint stub if necessary.
1112     RuntimeAddress nooverlap(nooverlap_target);
1113     __ cmpptr(dst, src);
1114     __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
1115     __ jump_cc(Assembler::belowEqual, nooverlap);
1116     __ cmpptr(dst, end);
1117     __ jump_cc(Assembler::aboveEqual, nooverlap);
1118 
1119     if (t == T_OBJECT) {
1120       __ testl(count, count);
1121       __ jcc(Assembler::zero, L_0_count);
1122       gen_write_ref_array_pre_barrier(dst, count, dest_uninitialized);
1123     }
1124 
1125     // copy from high to low
1126     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1127     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1128     if (t == T_BYTE || t == T_SHORT) {
1129       // Align the end of destination array at 4 bytes address boundary
1130       __ lea(end, Address(dst, count, sf, 0));
1131       if (t == T_BYTE) {
1132         // One byte misalignment happens only for byte arrays
1133         __ testl(end, 1);
1134         __ jccb(Assembler::zero, L_skip_align1);
1135         __ decrement(count);
1136         __ movb(rdx, Address(from, count, sf, 0));
1137         __ movb(Address(to, count, sf, 0), rdx);
1138       __ BIND(L_skip_align1);
1139       }
1140       // Two bytes misalignment happens only for byte and short (char) arrays
1141       __ testl(end, 2);
1142       __ jccb(Assembler::zero, L_skip_align2);
1143       __ subptr(count, 1<<(shift-1));
1144       __ movw(rdx, Address(from, count, sf, 0));
1145       __ movw(Address(to, count, sf, 0), rdx);
1146     __ BIND(L_skip_align2);
1147       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1148       __ jcc(Assembler::below, L_copy_4_bytes);
1149     }
1150 
1151     if (!VM_Version::supports_mmx()) {
1152       __ std();
1153       __ mov(rax, count); // Save 'count'
1154       __ mov(rdx, to);    // Save 'to'
1155       __ lea(rsi, Address(from, count, sf, -4));
1156       __ lea(rdi, Address(to  , count, sf, -4));
1157       __ shrptr(count, shift); // bytes count
1158       __ rep_mov();
1159       __ cld();
1160       __ mov(count, rax); // restore 'count'
1161       __ andl(count, (1<<shift)-1);      // mask the number of rest elements
1162       __ movptr(from, Address(rsp, 12+4)); // reread 'from'
1163       __ mov(to, rdx);   // restore 'to'
1164       __ jmpb(L_copy_2_bytes); // all dword were copied
1165    } else {
1166       // Align to 8 bytes the end of array. It is aligned to 4 bytes already.
1167       __ testptr(end, 4);
1168       __ jccb(Assembler::zero, L_copy_8_bytes);
1169       __ subl(count, 1<<shift);
1170       __ movl(rdx, Address(from, count, sf, 0));
1171       __ movl(Address(to, count, sf, 0), rdx);
1172       __ jmpb(L_copy_8_bytes);
1173 
1174       __ align(OptoLoopAlignment);
1175       // Move 8 bytes
1176     __ BIND(L_copy_8_bytes_loop);
1177       if (UseXMMForArrayCopy) {
1178         __ movq(xmm0, Address(from, count, sf, 0));
1179         __ movq(Address(to, count, sf, 0), xmm0);
1180       } else {
1181         __ movq(mmx0, Address(from, count, sf, 0));
1182         __ movq(Address(to, count, sf, 0), mmx0);
1183       }
1184     __ BIND(L_copy_8_bytes);
1185       __ subl(count, 2<<shift);
1186       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1187       __ addl(count, 2<<shift);
1188       if (!UseXMMForArrayCopy) {
1189         __ emms();
1190       }
1191     }
1192   __ BIND(L_copy_4_bytes);
1193     // copy prefix qword
1194     __ testl(count, 1<<shift);
1195     __ jccb(Assembler::zero, L_copy_2_bytes);
1196     __ movl(rdx, Address(from, count, sf, -4));
1197     __ movl(Address(to, count, sf, -4), rdx);
1198 
1199     if (t == T_BYTE || t == T_SHORT) {
1200         __ subl(count, (1<<shift));
1201       __ BIND(L_copy_2_bytes);
1202         // copy prefix dword
1203         __ testl(count, 1<<(shift-1));
1204         __ jccb(Assembler::zero, L_copy_byte);
1205         __ movw(rdx, Address(from, count, sf, -2));
1206         __ movw(Address(to, count, sf, -2), rdx);
1207         if (t == T_BYTE) {
1208           __ subl(count, 1<<(shift-1));
1209         __ BIND(L_copy_byte);
1210           // copy prefix byte
1211           __ testl(count, 1);
1212           __ jccb(Assembler::zero, L_exit);
1213           __ movb(rdx, Address(from, 0));
1214           __ movb(Address(to, 0), rdx);
1215         __ BIND(L_exit);
1216         } else {
1217         __ BIND(L_copy_byte);
1218         }
1219     } else {
1220     __ BIND(L_copy_2_bytes);
1221     }
1222     if (t == T_OBJECT) {
1223       __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1224       gen_write_ref_array_post_barrier(to, count);
1225     __ BIND(L_0_count);
1226     }
1227     inc_copy_counter_np(t);
1228     __ pop(rdi);
1229     __ pop(rsi);
1230     __ leave(); // required for proper stackwalking of RuntimeStub frame
1231     __ xorptr(rax, rax); // return 0
1232     __ ret(0);
1233     return start;
1234   }
1235 
1236 
1237   address generate_disjoint_long_copy(address* entry, const char *name) {
1238     __ align(CodeEntryAlignment);
1239     StubCodeMark mark(this, "StubRoutines", name);
1240     address start = __ pc();
1241 
1242     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1243     const Register from       = rax;  // source array address
1244     const Register to         = rdx;  // destination array address
1245     const Register count      = rcx;  // elements count
1246     const Register to_from    = rdx;  // (to - from)
1247 
1248     __ enter(); // required for proper stackwalking of RuntimeStub frame
1249     __ movptr(from , Address(rsp, 8+0));       // from
1250     __ movptr(to   , Address(rsp, 8+4));       // to
1251     __ movl2ptr(count, Address(rsp, 8+8));     // count
1252 
1253     *entry = __ pc(); // Entry point from conjoint arraycopy stub.
1254     BLOCK_COMMENT("Entry:");
1255 
1256     __ subptr(to, from); // to --> to_from
1257     if (VM_Version::supports_mmx()) {
1258       if (UseXMMForArrayCopy) {
1259         xmm_copy_forward(from, to_from, count);
1260       } else {
1261         mmx_copy_forward(from, to_from, count);
1262       }
1263     } else {
1264       __ jmpb(L_copy_8_bytes);
1265       __ align(OptoLoopAlignment);
1266     __ BIND(L_copy_8_bytes_loop);
1267       __ fild_d(Address(from, 0));
1268       __ fistp_d(Address(from, to_from, Address::times_1));
1269       __ addptr(from, 8);
1270     __ BIND(L_copy_8_bytes);
1271       __ decrement(count);
1272       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1273     }
1274     inc_copy_counter_np(T_LONG);
1275     __ leave(); // required for proper stackwalking of RuntimeStub frame
1276     __ xorptr(rax, rax); // return 0
1277     __ ret(0);
1278     return start;
1279   }
1280 
1281   address generate_conjoint_long_copy(address nooverlap_target,
1282                                       address* entry, const char *name) {
1283     __ align(CodeEntryAlignment);
1284     StubCodeMark mark(this, "StubRoutines", name);
1285     address start = __ pc();
1286 
1287     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1288     const Register from       = rax;  // source array address
1289     const Register to         = rdx;  // destination array address
1290     const Register count      = rcx;  // elements count
1291     const Register end_from   = rax;  // source array end address
1292 
1293     __ enter(); // required for proper stackwalking of RuntimeStub frame
1294     __ movptr(from , Address(rsp, 8+0));       // from
1295     __ movptr(to   , Address(rsp, 8+4));       // to
1296     __ movl2ptr(count, Address(rsp, 8+8));     // count
1297 
1298     *entry = __ pc(); // Entry point from generic arraycopy stub.
1299     BLOCK_COMMENT("Entry:");
1300 
1301     // arrays overlap test
1302     __ cmpptr(to, from);
1303     RuntimeAddress nooverlap(nooverlap_target);
1304     __ jump_cc(Assembler::belowEqual, nooverlap);
1305     __ lea(end_from, Address(from, count, Address::times_8, 0));
1306     __ cmpptr(to, end_from);
1307     __ movptr(from, Address(rsp, 8));  // from
1308     __ jump_cc(Assembler::aboveEqual, nooverlap);
1309 
1310     __ jmpb(L_copy_8_bytes);
1311 
1312     __ align(OptoLoopAlignment);
1313   __ BIND(L_copy_8_bytes_loop);
1314     if (VM_Version::supports_mmx()) {
1315       if (UseXMMForArrayCopy) {
1316         __ movq(xmm0, Address(from, count, Address::times_8));
1317         __ movq(Address(to, count, Address::times_8), xmm0);
1318       } else {
1319         __ movq(mmx0, Address(from, count, Address::times_8));
1320         __ movq(Address(to, count, Address::times_8), mmx0);
1321       }
1322     } else {
1323       __ fild_d(Address(from, count, Address::times_8));
1324       __ fistp_d(Address(to, count, Address::times_8));
1325     }
1326   __ BIND(L_copy_8_bytes);
1327     __ decrement(count);
1328     __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1329 
1330     if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
1331       __ emms();
1332     }
1333     inc_copy_counter_np(T_LONG);
1334     __ leave(); // required for proper stackwalking of RuntimeStub frame
1335     __ xorptr(rax, rax); // return 0
1336     __ ret(0);
1337     return start;
1338   }
1339 
1340 
1341   // Helper for generating a dynamic type check.
1342   // The sub_klass must be one of {rbx, rdx, rsi}.
1343   // The temp is killed.
1344   void generate_type_check(Register sub_klass,
1345                            Address& super_check_offset_addr,
1346                            Address& super_klass_addr,
1347                            Register temp,
1348                            Label* L_success, Label* L_failure) {
1349     BLOCK_COMMENT("type_check:");
1350 
1351     Label L_fallthrough;
1352 #define LOCAL_JCC(assembler_con, label_ptr)                             \
1353     if (label_ptr != NULL)  __ jcc(assembler_con, *(label_ptr));        \
1354     else                    __ jcc(assembler_con, L_fallthrough) /*omit semi*/
1355 
1356     // The following is a strange variation of the fast path which requires
1357     // one less register, because needed values are on the argument stack.
1358     // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
1359     //                                  L_success, L_failure, NULL);
1360     assert_different_registers(sub_klass, temp);
1361 
1362     int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1363 
1364     // if the pointers are equal, we are done (e.g., String[] elements)
1365     __ cmpptr(sub_klass, super_klass_addr);
1366     LOCAL_JCC(Assembler::equal, L_success);
1367 
1368     // check the supertype display:
1369     __ movl2ptr(temp, super_check_offset_addr);
1370     Address super_check_addr(sub_klass, temp, Address::times_1, 0);
1371     __ movptr(temp, super_check_addr); // load displayed supertype
1372     __ cmpptr(temp, super_klass_addr); // test the super type
1373     LOCAL_JCC(Assembler::equal, L_success);
1374 
1375     // if it was a primary super, we can just fail immediately
1376     __ cmpl(super_check_offset_addr, sc_offset);
1377     LOCAL_JCC(Assembler::notEqual, L_failure);
1378 
1379     // The repne_scan instruction uses fixed registers, which will get spilled.
1380     // We happen to know this works best when super_klass is in rax.
1381     Register super_klass = temp;
1382     __ movptr(super_klass, super_klass_addr);
1383     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
1384                                      L_success, L_failure);
1385 
1386     __ bind(L_fallthrough);
1387 
1388     if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
1389     if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
1390 
1391 #undef LOCAL_JCC
1392   }
1393 
1394   //
1395   //  Generate checkcasting array copy stub
1396   //
1397   //  Input:
1398   //    4(rsp)   - source array address
1399   //    8(rsp)   - destination array address
1400   //   12(rsp)   - element count, can be zero
1401   //   16(rsp)   - size_t ckoff (super_check_offset)
1402   //   20(rsp)   - oop ckval (super_klass)
1403   //
1404   //  Output:
1405   //    rax, ==  0  -  success
1406   //    rax, == -1^K - failure, where K is partial transfer count
1407   //
1408   address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitialized = false) {
1409     __ align(CodeEntryAlignment);
1410     StubCodeMark mark(this, "StubRoutines", name);
1411     address start = __ pc();
1412 
1413     Label L_load_element, L_store_element, L_do_card_marks, L_done;
1414 
1415     // register use:
1416     //  rax, rdx, rcx -- loop control (end_from, end_to, count)
1417     //  rdi, rsi      -- element access (oop, klass)
1418     //  rbx,           -- temp
1419     const Register from       = rax;    // source array address
1420     const Register to         = rdx;    // destination array address
1421     const Register length     = rcx;    // elements count
1422     const Register elem       = rdi;    // each oop copied
1423     const Register elem_klass = rsi;    // each elem._klass (sub_klass)
1424     const Register temp       = rbx;    // lone remaining temp
1425 
1426     __ enter(); // required for proper stackwalking of RuntimeStub frame
1427 
1428     __ push(rsi);
1429     __ push(rdi);
1430     __ push(rbx);
1431 
1432     Address   from_arg(rsp, 16+ 4);     // from
1433     Address     to_arg(rsp, 16+ 8);     // to
1434     Address length_arg(rsp, 16+12);     // elements count
1435     Address  ckoff_arg(rsp, 16+16);     // super_check_offset
1436     Address  ckval_arg(rsp, 16+20);     // super_klass
1437 
1438     // Load up:
1439     __ movptr(from,     from_arg);
1440     __ movptr(to,         to_arg);
1441     __ movl2ptr(length, length_arg);
1442 
1443     if (entry != NULL) {
1444       *entry = __ pc(); // Entry point from generic arraycopy stub.
1445       BLOCK_COMMENT("Entry:");
1446     }
1447 
1448     //---------------------------------------------------------------
1449     // Assembler stub will be used for this call to arraycopy
1450     // if the two arrays are subtypes of Object[] but the
1451     // destination array type is not equal to or a supertype
1452     // of the source type.  Each element must be separately
1453     // checked.
1454 
1455     // Loop-invariant addresses.  They are exclusive end pointers.
1456     Address end_from_addr(from, length, Address::times_ptr, 0);
1457     Address   end_to_addr(to,   length, Address::times_ptr, 0);
1458 
1459     Register end_from = from;           // re-use
1460     Register end_to   = to;             // re-use
1461     Register count    = length;         // re-use
1462 
1463     // Loop-variant addresses.  They assume post-incremented count < 0.
1464     Address from_element_addr(end_from, count, Address::times_ptr, 0);
1465     Address   to_element_addr(end_to,   count, Address::times_ptr, 0);
1466     Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
1467 
1468     // Copy from low to high addresses, indexed from the end of each array.
1469     gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1470     __ lea(end_from, end_from_addr);
1471     __ lea(end_to,   end_to_addr);
1472     assert(length == count, "");        // else fix next line:
1473     __ negptr(count);                   // negate and test the length
1474     __ jccb(Assembler::notZero, L_load_element);
1475 
1476     // Empty array:  Nothing to do.
1477     __ xorptr(rax, rax);                  // return 0 on (trivial) success
1478     __ jmp(L_done);
1479 
1480     // ======== begin loop ========
1481     // (Loop is rotated; its entry is L_load_element.)
1482     // Loop control:
1483     //   for (count = -count; count != 0; count++)
1484     // Base pointers src, dst are biased by 8*count,to last element.
1485     __ align(OptoLoopAlignment);
1486 
1487     __ BIND(L_store_element);
1488     __ movptr(to_element_addr, elem);     // store the oop
1489     __ increment(count);                // increment the count toward zero
1490 #if INCLUDE_ALL_GCS
1491     if (UseShenandoahGC) {
1492       // Shenandoah barrier is too big for 8-bit offsets to work
1493       __ jcc(Assembler::zero, L_do_card_marks);
1494     } else
1495 #endif
1496     __ jccb(Assembler::zero, L_do_card_marks);
1497 
1498     // ======== loop entry is here ========
1499     __ BIND(L_load_element);
1500 #if INCLUDE_ALL_GCS
1501     if (UseShenandoahGC) {
1502       // Needs GC barriers
1503       __ load_heap_oop(elem, from_element_addr);
1504     } else
1505 #endif
1506     __ movptr(elem, from_element_addr);   // load the oop
1507     __ testptr(elem, elem);
1508 #if INCLUDE_ALL_GCS
1509     if (UseShenandoahGC) {
1510       // Shenandoah barrier is too big for 8-bit offsets to work
1511       __ jcc(Assembler::zero, L_store_element);
1512     } else
1513 #endif
1514     __ jccb(Assembler::zero, L_store_element);
1515 
1516     // (Could do a trick here:  Remember last successful non-null
1517     // element stored and make a quick oop equality check on it.)
1518 
1519     __ movptr(elem_klass, elem_klass_addr); // query the object klass
1520     generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
1521                         &L_store_element, NULL);
1522     // (On fall-through, we have failed the element type check.)
1523     // ======== end loop ========
1524 
1525     // It was a real error; we must depend on the caller to finish the job.
1526     // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
1527     // Emit GC store barriers for the oops we have copied (length_arg + count),
1528     // and report their number to the caller.
1529     assert_different_registers(to, count, rax);
1530     Label L_post_barrier;
1531     __ addl(count, length_arg);         // transfers = (length - remaining)
1532     __ movl2ptr(rax, count);            // save the value
1533     __ notptr(rax);                     // report (-1^K) to caller (does not affect flags)
1534     __ jccb(Assembler::notZero, L_post_barrier);
1535     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
1536 
1537     // Come here on success only.
1538     __ BIND(L_do_card_marks);
1539     __ xorptr(rax, rax);                // return 0 on success
1540     __ movl2ptr(count, length_arg);
1541 
1542     __ BIND(L_post_barrier);
1543     __ movptr(to, to_arg);              // reload
1544     gen_write_ref_array_post_barrier(to, count);
1545 
1546     // Common exit point (success or failure).
1547     __ BIND(L_done);
1548     __ pop(rbx);
1549     __ pop(rdi);
1550     __ pop(rsi);
1551     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1552     __ leave(); // required for proper stackwalking of RuntimeStub frame
1553     __ ret(0);
1554 
1555     return start;
1556   }
1557 
1558   //
1559   //  Generate 'unsafe' array copy stub
1560   //  Though just as safe as the other stubs, it takes an unscaled
1561   //  size_t argument instead of an element count.
1562   //
1563   //  Input:
1564   //    4(rsp)   - source array address
1565   //    8(rsp)   - destination array address
1566   //   12(rsp)   - byte count, can be zero
1567   //
1568   //  Output:
1569   //    rax, ==  0  -  success
1570   //    rax, == -1  -  need to call System.arraycopy
1571   //
1572   // Examines the alignment of the operands and dispatches
1573   // to a long, int, short, or byte copy loop.
1574   //
1575   address generate_unsafe_copy(const char *name,
1576                                address byte_copy_entry,
1577                                address short_copy_entry,
1578                                address int_copy_entry,
1579                                address long_copy_entry) {
1580 
1581     Label L_long_aligned, L_int_aligned, L_short_aligned;
1582 
1583     __ align(CodeEntryAlignment);
1584     StubCodeMark mark(this, "StubRoutines", name);
1585     address start = __ pc();
1586 
1587     const Register from       = rax;  // source array address
1588     const Register to         = rdx;  // destination array address
1589     const Register count      = rcx;  // elements count
1590 
1591     __ enter(); // required for proper stackwalking of RuntimeStub frame
1592     __ push(rsi);
1593     __ push(rdi);
1594     Address  from_arg(rsp, 12+ 4);      // from
1595     Address    to_arg(rsp, 12+ 8);      // to
1596     Address count_arg(rsp, 12+12);      // byte count
1597 
1598     // Load up:
1599     __ movptr(from ,  from_arg);
1600     __ movptr(to   ,    to_arg);
1601     __ movl2ptr(count, count_arg);
1602 
1603     // bump this on entry, not on exit:
1604     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
1605 
1606     const Register bits = rsi;
1607     __ mov(bits, from);
1608     __ orptr(bits, to);
1609     __ orptr(bits, count);
1610 
1611     __ testl(bits, BytesPerLong-1);
1612     __ jccb(Assembler::zero, L_long_aligned);
1613 
1614     __ testl(bits, BytesPerInt-1);
1615     __ jccb(Assembler::zero, L_int_aligned);
1616 
1617     __ testl(bits, BytesPerShort-1);
1618     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
1619 
1620     __ BIND(L_short_aligned);
1621     __ shrptr(count, LogBytesPerShort); // size => short_count
1622     __ movl(count_arg, count);          // update 'count'
1623     __ jump(RuntimeAddress(short_copy_entry));
1624 
1625     __ BIND(L_int_aligned);
1626     __ shrptr(count, LogBytesPerInt); // size => int_count
1627     __ movl(count_arg, count);          // update 'count'
1628     __ jump(RuntimeAddress(int_copy_entry));
1629 
1630     __ BIND(L_long_aligned);
1631     __ shrptr(count, LogBytesPerLong); // size => qword_count
1632     __ movl(count_arg, count);          // update 'count'
1633     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1634     __ pop(rsi);
1635     __ jump(RuntimeAddress(long_copy_entry));
1636 
1637     return start;
1638   }
1639 
1640 
1641   // Perform range checks on the proposed arraycopy.
1642   // Smashes src_pos and dst_pos.  (Uses them up for temps.)
1643   void arraycopy_range_checks(Register src,
1644                               Register src_pos,
1645                               Register dst,
1646                               Register dst_pos,
1647                               Address& length,
1648                               Label& L_failed) {
1649     BLOCK_COMMENT("arraycopy_range_checks:");
1650     const Register src_end = src_pos;   // source array end position
1651     const Register dst_end = dst_pos;   // destination array end position
1652     __ addl(src_end, length); // src_pos + length
1653     __ addl(dst_end, length); // dst_pos + length
1654 
1655     //  if (src_pos + length > arrayOop(src)->length() ) FAIL;
1656     __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
1657     __ jcc(Assembler::above, L_failed);
1658 
1659     //  if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
1660     __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1661     __ jcc(Assembler::above, L_failed);
1662 
1663     BLOCK_COMMENT("arraycopy_range_checks done");
1664   }
1665 
1666 
1667   //
1668   //  Generate generic array copy stubs
1669   //
1670   //  Input:
1671   //     4(rsp)    -  src oop
1672   //     8(rsp)    -  src_pos
1673   //    12(rsp)    -  dst oop
1674   //    16(rsp)    -  dst_pos
1675   //    20(rsp)    -  element count
1676   //
1677   //  Output:
1678   //    rax, ==  0  -  success
1679   //    rax, == -1^K - failure, where K is partial transfer count
1680   //
1681   address generate_generic_copy(const char *name,
1682                                 address entry_jbyte_arraycopy,
1683                                 address entry_jshort_arraycopy,
1684                                 address entry_jint_arraycopy,
1685                                 address entry_oop_arraycopy,
1686                                 address entry_jlong_arraycopy,
1687                                 address entry_checkcast_arraycopy) {
1688     Label L_failed, L_failed_0, L_objArray;
1689 
1690     { int modulus = CodeEntryAlignment;
1691       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
1692       int advance = target - (__ offset() % modulus);
1693       if (advance < 0)  advance += modulus;
1694       if (advance > 0)  __ nop(advance);
1695     }
1696     StubCodeMark mark(this, "StubRoutines", name);
1697 
1698     // Short-hop target to L_failed.  Makes for denser prologue code.
1699     __ BIND(L_failed_0);
1700     __ jmp(L_failed);
1701     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
1702 
1703     __ align(CodeEntryAlignment);
1704     address start = __ pc();
1705 
1706     __ enter(); // required for proper stackwalking of RuntimeStub frame
1707     __ push(rsi);
1708     __ push(rdi);
1709 
1710     // bump this on entry, not on exit:
1711     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
1712 
1713     // Input values
1714     Address SRC     (rsp, 12+ 4);
1715     Address SRC_POS (rsp, 12+ 8);
1716     Address DST     (rsp, 12+12);
1717     Address DST_POS (rsp, 12+16);
1718     Address LENGTH  (rsp, 12+20);
1719 
1720     //-----------------------------------------------------------------------
1721     // Assembler stub will be used for this call to arraycopy
1722     // if the following conditions are met:
1723     //
1724     // (1) src and dst must not be null.
1725     // (2) src_pos must not be negative.
1726     // (3) dst_pos must not be negative.
1727     // (4) length  must not be negative.
1728     // (5) src klass and dst klass should be the same and not NULL.
1729     // (6) src and dst should be arrays.
1730     // (7) src_pos + length must not exceed length of src.
1731     // (8) dst_pos + length must not exceed length of dst.
1732     //
1733 
1734     const Register src     = rax;       // source array oop
1735     const Register src_pos = rsi;
1736     const Register dst     = rdx;       // destination array oop
1737     const Register dst_pos = rdi;
1738     const Register length  = rcx;       // transfer count
1739 
1740     //  if (src == NULL) return -1;
1741     __ movptr(src, SRC);      // src oop
1742     __ testptr(src, src);
1743     __ jccb(Assembler::zero, L_failed_0);
1744 
1745     //  if (src_pos < 0) return -1;
1746     __ movl2ptr(src_pos, SRC_POS);  // src_pos
1747     __ testl(src_pos, src_pos);
1748     __ jccb(Assembler::negative, L_failed_0);
1749 
1750     //  if (dst == NULL) return -1;
1751     __ movptr(dst, DST);      // dst oop
1752     __ testptr(dst, dst);
1753     __ jccb(Assembler::zero, L_failed_0);
1754 
1755     //  if (dst_pos < 0) return -1;
1756     __ movl2ptr(dst_pos, DST_POS);  // dst_pos
1757     __ testl(dst_pos, dst_pos);
1758     __ jccb(Assembler::negative, L_failed_0);
1759 
1760     //  if (length < 0) return -1;
1761     __ movl2ptr(length, LENGTH);   // length
1762     __ testl(length, length);
1763     __ jccb(Assembler::negative, L_failed_0);
1764 
1765     //  if (src->klass() == NULL) return -1;
1766     Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
1767     Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
1768     const Register rcx_src_klass = rcx;    // array klass
1769     __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
1770 
1771 #ifdef ASSERT
1772     //  assert(src->klass() != NULL);
1773     BLOCK_COMMENT("assert klasses not null");
1774     { Label L1, L2;
1775       __ testptr(rcx_src_klass, rcx_src_klass);
1776       __ jccb(Assembler::notZero, L2);   // it is broken if klass is NULL
1777       __ bind(L1);
1778       __ stop("broken null klass");
1779       __ bind(L2);
1780       __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
1781       __ jccb(Assembler::equal, L1);      // this would be broken also
1782       BLOCK_COMMENT("assert done");
1783     }
1784 #endif //ASSERT
1785 
1786     // Load layout helper (32-bits)
1787     //
1788     //  |array_tag|     | header_size | element_type |     |log2_element_size|
1789     // 32        30    24            16              8     2                 0
1790     //
1791     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
1792     //
1793 
1794     int lh_offset = in_bytes(Klass::layout_helper_offset());
1795     Address src_klass_lh_addr(rcx_src_klass, lh_offset);
1796 
1797     // Handle objArrays completely differently...
1798     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
1799     __ cmpl(src_klass_lh_addr, objArray_lh);
1800     __ jcc(Assembler::equal, L_objArray);
1801 
1802     //  if (src->klass() != dst->klass()) return -1;
1803     __ cmpptr(rcx_src_klass, dst_klass_addr);
1804     __ jccb(Assembler::notEqual, L_failed_0);
1805 
1806     const Register rcx_lh = rcx;  // layout helper
1807     assert(rcx_lh == rcx_src_klass, "known alias");
1808     __ movl(rcx_lh, src_klass_lh_addr);
1809 
1810     //  if (!src->is_Array()) return -1;
1811     __ cmpl(rcx_lh, Klass::_lh_neutral_value);
1812     __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
1813 
1814     // At this point, it is known to be a typeArray (array_tag 0x3).
1815 #ifdef ASSERT
1816     { Label L;
1817       __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
1818       __ jcc(Assembler::greaterEqual, L); // signed cmp
1819       __ stop("must be a primitive array");
1820       __ bind(L);
1821     }
1822 #endif
1823 
1824     assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
1825     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1826 
1827     // TypeArrayKlass
1828     //
1829     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
1830     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
1831     //
1832     const Register rsi_offset = rsi; // array offset
1833     const Register src_array  = src; // src array offset
1834     const Register dst_array  = dst; // dst array offset
1835     const Register rdi_elsize = rdi; // log2 element size
1836 
1837     __ mov(rsi_offset, rcx_lh);
1838     __ shrptr(rsi_offset, Klass::_lh_header_size_shift);
1839     __ andptr(rsi_offset, Klass::_lh_header_size_mask);   // array_offset
1840     __ addptr(src_array, rsi_offset);  // src array offset
1841     __ addptr(dst_array, rsi_offset);  // dst array offset
1842     __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
1843 
1844     // next registers should be set before the jump to corresponding stub
1845     const Register from       = src; // source array address
1846     const Register to         = dst; // destination array address
1847     const Register count      = rcx; // elements count
1848     // some of them should be duplicated on stack
1849 #define FROM   Address(rsp, 12+ 4)
1850 #define TO     Address(rsp, 12+ 8)   // Not used now
1851 #define COUNT  Address(rsp, 12+12)   // Only for oop arraycopy
1852 
1853     BLOCK_COMMENT("scale indexes to element size");
1854     __ movl2ptr(rsi, SRC_POS);  // src_pos
1855     __ shlptr(rsi);             // src_pos << rcx (log2 elsize)
1856     assert(src_array == from, "");
1857     __ addptr(from, rsi);       // from = src_array + SRC_POS << log2 elsize
1858     __ movl2ptr(rdi, DST_POS);  // dst_pos
1859     __ shlptr(rdi);             // dst_pos << rcx (log2 elsize)
1860     assert(dst_array == to, "");
1861     __ addptr(to,  rdi);        // to   = dst_array + DST_POS << log2 elsize
1862     __ movptr(FROM, from);      // src_addr
1863     __ mov(rdi_elsize, rcx_lh); // log2 elsize
1864     __ movl2ptr(count, LENGTH); // elements count
1865 
1866     BLOCK_COMMENT("choose copy loop based on element size");
1867     __ cmpl(rdi_elsize, 0);
1868 
1869     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
1870     __ cmpl(rdi_elsize, LogBytesPerShort);
1871     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
1872     __ cmpl(rdi_elsize, LogBytesPerInt);
1873     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
1874 #ifdef ASSERT
1875     __ cmpl(rdi_elsize, LogBytesPerLong);
1876     __ jccb(Assembler::notEqual, L_failed);
1877 #endif
1878     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1879     __ pop(rsi);
1880     __ jump(RuntimeAddress(entry_jlong_arraycopy));
1881 
1882   __ BIND(L_failed);
1883     __ xorptr(rax, rax);
1884     __ notptr(rax); // return -1
1885     __ pop(rdi);
1886     __ pop(rsi);
1887     __ leave(); // required for proper stackwalking of RuntimeStub frame
1888     __ ret(0);
1889 
1890     // ObjArrayKlass
1891   __ BIND(L_objArray);
1892     // live at this point:  rcx_src_klass, src[_pos], dst[_pos]
1893 
1894     Label L_plain_copy, L_checkcast_copy;
1895     //  test array classes for subtyping
1896     __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
1897     __ jccb(Assembler::notEqual, L_checkcast_copy);
1898 
1899     // Identically typed arrays can be copied without element-wise checks.
1900     assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
1901     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1902 
1903   __ BIND(L_plain_copy);
1904     __ movl2ptr(count, LENGTH); // elements count
1905     __ movl2ptr(src_pos, SRC_POS);  // reload src_pos
1906     __ lea(from, Address(src, src_pos, Address::times_ptr,
1907                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
1908     __ movl2ptr(dst_pos, DST_POS);  // reload dst_pos
1909     __ lea(to,   Address(dst, dst_pos, Address::times_ptr,
1910                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
1911     __ movptr(FROM,  from);   // src_addr
1912     __ movptr(TO,    to);     // dst_addr
1913     __ movl(COUNT, count);  // count
1914     __ jump(RuntimeAddress(entry_oop_arraycopy));
1915 
1916   __ BIND(L_checkcast_copy);
1917     // live at this point:  rcx_src_klass, dst[_pos], src[_pos]
1918     {
1919       // Handy offsets:
1920       int  ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
1921       int sco_offset = in_bytes(Klass::super_check_offset_offset());
1922 
1923       Register rsi_dst_klass = rsi;
1924       Register rdi_temp      = rdi;
1925       assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
1926       assert(rdi_temp      == dst_pos, "expected alias w/ dst_pos");
1927       Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
1928 
1929       // Before looking at dst.length, make sure dst is also an objArray.
1930       __ movptr(rsi_dst_klass, dst_klass_addr);
1931       __ cmpl(dst_klass_lh_addr, objArray_lh);
1932       __ jccb(Assembler::notEqual, L_failed);
1933 
1934       // It is safe to examine both src.length and dst.length.
1935       __ movl2ptr(src_pos, SRC_POS);        // reload rsi
1936       arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1937       // (Now src_pos and dst_pos are killed, but not src and dst.)
1938 
1939       // We'll need this temp (don't forget to pop it after the type check).
1940       __ push(rbx);
1941       Register rbx_src_klass = rbx;
1942 
1943       __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
1944       __ movptr(rsi_dst_klass, dst_klass_addr);
1945       Address super_check_offset_addr(rsi_dst_klass, sco_offset);
1946       Label L_fail_array_check;
1947       generate_type_check(rbx_src_klass,
1948                           super_check_offset_addr, dst_klass_addr,
1949                           rdi_temp, NULL, &L_fail_array_check);
1950       // (On fall-through, we have passed the array type check.)
1951       __ pop(rbx);
1952       __ jmp(L_plain_copy);
1953 
1954       __ BIND(L_fail_array_check);
1955       // Reshuffle arguments so we can call checkcast_arraycopy:
1956 
1957       // match initial saves for checkcast_arraycopy
1958       // push(rsi);    // already done; see above
1959       // push(rdi);    // already done; see above
1960       // push(rbx);    // already done; see above
1961 
1962       // Marshal outgoing arguments now, freeing registers.
1963       Address   from_arg(rsp, 16+ 4);   // from
1964       Address     to_arg(rsp, 16+ 8);   // to
1965       Address length_arg(rsp, 16+12);   // elements count
1966       Address  ckoff_arg(rsp, 16+16);   // super_check_offset
1967       Address  ckval_arg(rsp, 16+20);   // super_klass
1968 
1969       Address SRC_POS_arg(rsp, 16+ 8);
1970       Address DST_POS_arg(rsp, 16+16);
1971       Address  LENGTH_arg(rsp, 16+20);
1972       // push rbx, changed the incoming offsets (why not just use rbp,??)
1973       // assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
1974 
1975       __ movptr(rbx, Address(rsi_dst_klass, ek_offset));
1976       __ movl2ptr(length, LENGTH_arg);    // reload elements count
1977       __ movl2ptr(src_pos, SRC_POS_arg);  // reload src_pos
1978       __ movl2ptr(dst_pos, DST_POS_arg);  // reload dst_pos
1979 
1980       __ movptr(ckval_arg, rbx);          // destination element type
1981       __ movl(rbx, Address(rbx, sco_offset));
1982       __ movl(ckoff_arg, rbx);          // corresponding class check offset
1983 
1984       __ movl(length_arg, length);      // outgoing length argument
1985 
1986       __ lea(from, Address(src, src_pos, Address::times_ptr,
1987                             arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1988       __ movptr(from_arg, from);
1989 
1990       __ lea(to, Address(dst, dst_pos, Address::times_ptr,
1991                           arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1992       __ movptr(to_arg, to);
1993       __ jump(RuntimeAddress(entry_checkcast_arraycopy));
1994     }
1995 
1996     return start;
1997   }
1998 
1999   void generate_arraycopy_stubs() {
2000     address entry;
2001     address entry_jbyte_arraycopy;
2002     address entry_jshort_arraycopy;
2003     address entry_jint_arraycopy;
2004     address entry_oop_arraycopy;
2005     address entry_jlong_arraycopy;
2006     address entry_checkcast_arraycopy;
2007 
2008     StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
2009         generate_disjoint_copy(T_BYTE,  true, Address::times_1, &entry,
2010                                "arrayof_jbyte_disjoint_arraycopy");
2011     StubRoutines::_arrayof_jbyte_arraycopy =
2012         generate_conjoint_copy(T_BYTE,  true, Address::times_1,  entry,
2013                                NULL, "arrayof_jbyte_arraycopy");
2014     StubRoutines::_jbyte_disjoint_arraycopy =
2015         generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
2016                                "jbyte_disjoint_arraycopy");
2017     StubRoutines::_jbyte_arraycopy =
2018         generate_conjoint_copy(T_BYTE, false, Address::times_1,  entry,
2019                                &entry_jbyte_arraycopy, "jbyte_arraycopy");
2020 
2021     StubRoutines::_arrayof_jshort_disjoint_arraycopy =
2022         generate_disjoint_copy(T_SHORT,  true, Address::times_2, &entry,
2023                                "arrayof_jshort_disjoint_arraycopy");
2024     StubRoutines::_arrayof_jshort_arraycopy =
2025         generate_conjoint_copy(T_SHORT,  true, Address::times_2,  entry,
2026                                NULL, "arrayof_jshort_arraycopy");
2027     StubRoutines::_jshort_disjoint_arraycopy =
2028         generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
2029                                "jshort_disjoint_arraycopy");
2030     StubRoutines::_jshort_arraycopy =
2031         generate_conjoint_copy(T_SHORT, false, Address::times_2,  entry,
2032                                &entry_jshort_arraycopy, "jshort_arraycopy");
2033 
2034     // Next arrays are always aligned on 4 bytes at least.
2035     StubRoutines::_jint_disjoint_arraycopy =
2036         generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
2037                                "jint_disjoint_arraycopy");
2038     StubRoutines::_jint_arraycopy =
2039         generate_conjoint_copy(T_INT, true, Address::times_4,  entry,
2040                                &entry_jint_arraycopy, "jint_arraycopy");
2041 
2042     StubRoutines::_oop_disjoint_arraycopy =
2043         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
2044                                "oop_disjoint_arraycopy");
2045     StubRoutines::_oop_arraycopy =
2046         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
2047                                &entry_oop_arraycopy, "oop_arraycopy");
2048 
2049     StubRoutines::_oop_disjoint_arraycopy_uninit =
2050         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
2051                                "oop_disjoint_arraycopy_uninit",
2052                                /*dest_uninitialized*/true);
2053     StubRoutines::_oop_arraycopy_uninit =
2054         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
2055                                NULL, "oop_arraycopy_uninit",
2056                                /*dest_uninitialized*/true);
2057 
2058     StubRoutines::_jlong_disjoint_arraycopy =
2059         generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
2060     StubRoutines::_jlong_arraycopy =
2061         generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
2062                                     "jlong_arraycopy");
2063 
2064     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2065     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2066     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2067     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2068     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2069     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2070 
2071     StubRoutines::_arrayof_jint_disjoint_arraycopy       = StubRoutines::_jint_disjoint_arraycopy;
2072     StubRoutines::_arrayof_oop_disjoint_arraycopy        = StubRoutines::_oop_disjoint_arraycopy;
2073     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;
2074     StubRoutines::_arrayof_jlong_disjoint_arraycopy      = StubRoutines::_jlong_disjoint_arraycopy;
2075 
2076     StubRoutines::_arrayof_jint_arraycopy       = StubRoutines::_jint_arraycopy;
2077     StubRoutines::_arrayof_oop_arraycopy        = StubRoutines::_oop_arraycopy;
2078     StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;
2079     StubRoutines::_arrayof_jlong_arraycopy      = StubRoutines::_jlong_arraycopy;
2080 
2081     StubRoutines::_checkcast_arraycopy =
2082         generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2083     StubRoutines::_checkcast_arraycopy_uninit =
2084         generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, /*dest_uninitialized*/true);
2085 
2086     StubRoutines::_unsafe_arraycopy =
2087         generate_unsafe_copy("unsafe_arraycopy",
2088                                entry_jbyte_arraycopy,
2089                                entry_jshort_arraycopy,
2090                                entry_jint_arraycopy,
2091                                entry_jlong_arraycopy);
2092 
2093     StubRoutines::_generic_arraycopy =
2094         generate_generic_copy("generic_arraycopy",
2095                                entry_jbyte_arraycopy,
2096                                entry_jshort_arraycopy,
2097                                entry_jint_arraycopy,
2098                                entry_oop_arraycopy,
2099                                entry_jlong_arraycopy,
2100                                entry_checkcast_arraycopy);
2101   }
2102 
2103   void generate_math_stubs() {
2104     {
2105       StubCodeMark mark(this, "StubRoutines", "log");
2106       StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
2107 
2108       __ fld_d(Address(rsp, 4));
2109       __ flog();
2110       __ ret(0);
2111     }
2112     {
2113       StubCodeMark mark(this, "StubRoutines", "log10");
2114       StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
2115 
2116       __ fld_d(Address(rsp, 4));
2117       __ flog10();
2118       __ ret(0);
2119     }
2120     {
2121       StubCodeMark mark(this, "StubRoutines", "sin");
2122       StubRoutines::_intrinsic_sin = (double (*)(double))  __ pc();
2123 
2124       __ fld_d(Address(rsp, 4));
2125       __ trigfunc('s');
2126       __ ret(0);
2127     }
2128     {
2129       StubCodeMark mark(this, "StubRoutines", "cos");
2130       StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
2131 
2132       __ fld_d(Address(rsp, 4));
2133       __ trigfunc('c');
2134       __ ret(0);
2135     }
2136     {
2137       StubCodeMark mark(this, "StubRoutines", "tan");
2138       StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
2139 
2140       __ fld_d(Address(rsp, 4));
2141       __ trigfunc('t');
2142       __ ret(0);
2143     }
2144     {
2145       StubCodeMark mark(this, "StubRoutines", "exp");
2146       StubRoutines::_intrinsic_exp = (double (*)(double)) __ pc();
2147 
2148       __ fld_d(Address(rsp, 4));
2149       __ exp_with_fallback(0);
2150       __ ret(0);
2151     }
2152     {
2153       StubCodeMark mark(this, "StubRoutines", "pow");
2154       StubRoutines::_intrinsic_pow = (double (*)(double,double)) __ pc();
2155 
2156       __ fld_d(Address(rsp, 12));
2157       __ fld_d(Address(rsp, 4));
2158       __ pow_with_fallback(0);
2159       __ ret(0);
2160     }
2161   }
2162 
2163   // AES intrinsic stubs
2164   enum {AESBlockSize = 16};
2165 
2166   address generate_key_shuffle_mask() {
2167     __ align(16);
2168     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
2169     address start = __ pc();
2170     __ emit_data(0x00010203, relocInfo::none, 0 );
2171     __ emit_data(0x04050607, relocInfo::none, 0 );
2172     __ emit_data(0x08090a0b, relocInfo::none, 0 );
2173     __ emit_data(0x0c0d0e0f, relocInfo::none, 0 );
2174     return start;
2175   }
2176 
2177   // Utility routine for loading a 128-bit key word in little endian format
2178   // can optionally specify that the shuffle mask is already in an xmmregister
2179   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2180     __ movdqu(xmmdst, Address(key, offset));
2181     if (xmm_shuf_mask != NULL) {
2182       __ pshufb(xmmdst, xmm_shuf_mask);
2183     } else {
2184       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2185     }
2186   }
2187 
2188   // aesenc using specified key+offset
2189   // can optionally specify that the shuffle mask is already in an xmmregister
2190   void aes_enc_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2191     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2192     __ aesenc(xmmdst, xmmtmp);
2193   }
2194 
2195   // aesdec using specified key+offset
2196   // can optionally specify that the shuffle mask is already in an xmmregister
2197   void aes_dec_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2198     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2199     __ aesdec(xmmdst, xmmtmp);
2200   }
2201 
2202 
2203   // Arguments:
2204   //
2205   // Inputs:
2206   //   c_rarg0   - source byte array address
2207   //   c_rarg1   - destination byte array address
2208   //   c_rarg2   - K (key) in little endian int array
2209   //
2210   address generate_aescrypt_encryptBlock() {
2211     assert(UseAES, "need AES instructions and misaligned SSE support");
2212     __ align(CodeEntryAlignment);
2213     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2214     Label L_doLast;
2215     address start = __ pc();
2216 
2217     const Register from        = rdx;      // source array address
2218     const Register to          = rdx;      // destination array address
2219     const Register key         = rcx;      // key array address
2220     const Register keylen      = rax;
2221     const Address  from_param(rbp, 8+0);
2222     const Address  to_param  (rbp, 8+4);
2223     const Address  key_param (rbp, 8+8);
2224 
2225     const XMMRegister xmm_result = xmm0;
2226     const XMMRegister xmm_key_shuf_mask = xmm1;
2227     const XMMRegister xmm_temp1  = xmm2;
2228     const XMMRegister xmm_temp2  = xmm3;
2229     const XMMRegister xmm_temp3  = xmm4;
2230     const XMMRegister xmm_temp4  = xmm5;
2231 
2232     __ enter();   // required for proper stackwalking of RuntimeStub frame
2233     __ movptr(from, from_param);
2234     __ movptr(key, key_param);
2235 
2236     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2237     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2238 
2239     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2240     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
2241     __ movptr(to, to_param);
2242 
2243     // For encryption, the java expanded key ordering is just what we need
2244 
2245     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
2246     __ pxor(xmm_result, xmm_temp1);
2247 
2248     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2249     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2250     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2251     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2252 
2253     __ aesenc(xmm_result, xmm_temp1);
2254     __ aesenc(xmm_result, xmm_temp2);
2255     __ aesenc(xmm_result, xmm_temp3);
2256     __ aesenc(xmm_result, xmm_temp4);
2257 
2258     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2259     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2260     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2261     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2262 
2263     __ aesenc(xmm_result, xmm_temp1);
2264     __ aesenc(xmm_result, xmm_temp2);
2265     __ aesenc(xmm_result, xmm_temp3);
2266     __ aesenc(xmm_result, xmm_temp4);
2267 
2268     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2269     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2270 
2271     __ cmpl(keylen, 44);
2272     __ jccb(Assembler::equal, L_doLast);
2273 
2274     __ aesenc(xmm_result, xmm_temp1);
2275     __ aesenc(xmm_result, xmm_temp2);
2276 
2277     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2278     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2279 
2280     __ cmpl(keylen, 52);
2281     __ jccb(Assembler::equal, L_doLast);
2282 
2283     __ aesenc(xmm_result, xmm_temp1);
2284     __ aesenc(xmm_result, xmm_temp2);
2285 
2286     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2287     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2288 
2289     __ BIND(L_doLast);
2290     __ aesenc(xmm_result, xmm_temp1);
2291     __ aesenclast(xmm_result, xmm_temp2);
2292     __ movdqu(Address(to, 0), xmm_result);        // store the result
2293     __ xorptr(rax, rax); // return 0
2294     __ leave(); // required for proper stackwalking of RuntimeStub frame
2295     __ ret(0);
2296 
2297     return start;
2298   }
2299 
2300 
2301   // Arguments:
2302   //
2303   // Inputs:
2304   //   c_rarg0   - source byte array address
2305   //   c_rarg1   - destination byte array address
2306   //   c_rarg2   - K (key) in little endian int array
2307   //
2308   address generate_aescrypt_decryptBlock() {
2309     assert(UseAES, "need AES instructions and misaligned SSE support");
2310     __ align(CodeEntryAlignment);
2311     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
2312     Label L_doLast;
2313     address start = __ pc();
2314 
2315     const Register from        = rdx;      // source array address
2316     const Register to          = rdx;      // destination array address
2317     const Register key         = rcx;      // key array address
2318     const Register keylen      = rax;
2319     const Address  from_param(rbp, 8+0);
2320     const Address  to_param  (rbp, 8+4);
2321     const Address  key_param (rbp, 8+8);
2322 
2323     const XMMRegister xmm_result = xmm0;
2324     const XMMRegister xmm_key_shuf_mask = xmm1;
2325     const XMMRegister xmm_temp1  = xmm2;
2326     const XMMRegister xmm_temp2  = xmm3;
2327     const XMMRegister xmm_temp3  = xmm4;
2328     const XMMRegister xmm_temp4  = xmm5;
2329 
2330     __ enter(); // required for proper stackwalking of RuntimeStub frame
2331     __ movptr(from, from_param);
2332     __ movptr(key, key_param);
2333 
2334     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2335     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2336 
2337     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2338     __ movdqu(xmm_result, Address(from, 0));
2339     __ movptr(to, to_param);
2340 
2341     // for decryption java expanded key ordering is rotated one position from what we want
2342     // so we start from 0x10 here and hit 0x00 last
2343     // we don't know if the key is aligned, hence not using load-execute form
2344     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2345     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2346     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2347     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2348 
2349     __ pxor  (xmm_result, xmm_temp1);
2350     __ aesdec(xmm_result, xmm_temp2);
2351     __ aesdec(xmm_result, xmm_temp3);
2352     __ aesdec(xmm_result, xmm_temp4);
2353 
2354     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2355     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2356     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2357     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2358 
2359     __ aesdec(xmm_result, xmm_temp1);
2360     __ aesdec(xmm_result, xmm_temp2);
2361     __ aesdec(xmm_result, xmm_temp3);
2362     __ aesdec(xmm_result, xmm_temp4);
2363 
2364     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2365     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2366     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
2367 
2368     __ cmpl(keylen, 44);
2369     __ jccb(Assembler::equal, L_doLast);
2370 
2371     __ aesdec(xmm_result, xmm_temp1);
2372     __ aesdec(xmm_result, xmm_temp2);
2373 
2374     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2375     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2376 
2377     __ cmpl(keylen, 52);
2378     __ jccb(Assembler::equal, L_doLast);
2379 
2380     __ aesdec(xmm_result, xmm_temp1);
2381     __ aesdec(xmm_result, xmm_temp2);
2382 
2383     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2384     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2385 
2386     __ BIND(L_doLast);
2387     __ aesdec(xmm_result, xmm_temp1);
2388     __ aesdec(xmm_result, xmm_temp2);
2389 
2390     // for decryption the aesdeclast operation is always on key+0x00
2391     __ aesdeclast(xmm_result, xmm_temp3);
2392     __ movdqu(Address(to, 0), xmm_result);  // store the result
2393     __ xorptr(rax, rax); // return 0
2394     __ leave(); // required for proper stackwalking of RuntimeStub frame
2395     __ ret(0);
2396 
2397     return start;
2398   }
2399 
2400   void handleSOERegisters(bool saving) {
2401     const int saveFrameSizeInBytes = 4 * wordSize;
2402     const Address saved_rbx     (rbp, -3 * wordSize);
2403     const Address saved_rsi     (rbp, -2 * wordSize);
2404     const Address saved_rdi     (rbp, -1 * wordSize);
2405 
2406     if (saving) {
2407       __ subptr(rsp, saveFrameSizeInBytes);
2408       __ movptr(saved_rsi, rsi);
2409       __ movptr(saved_rdi, rdi);
2410       __ movptr(saved_rbx, rbx);
2411     } else {
2412       // restoring
2413       __ movptr(rsi, saved_rsi);
2414       __ movptr(rdi, saved_rdi);
2415       __ movptr(rbx, saved_rbx);
2416     }
2417   }
2418 
2419   // Arguments:
2420   //
2421   // Inputs:
2422   //   c_rarg0   - source byte array address
2423   //   c_rarg1   - destination byte array address
2424   //   c_rarg2   - K (key) in little endian int array
2425   //   c_rarg3   - r vector byte array address
2426   //   c_rarg4   - input length
2427   //
2428   // Output:
2429   //   rax       - input length
2430   //
2431   address generate_cipherBlockChaining_encryptAESCrypt() {
2432     assert(UseAES, "need AES instructions and misaligned SSE support");
2433     __ align(CodeEntryAlignment);
2434     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
2435     address start = __ pc();
2436 
2437     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
2438     const Register from        = rsi;      // source array address
2439     const Register to          = rdx;      // destination array address
2440     const Register key         = rcx;      // key array address
2441     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2442                                            // and left with the results of the last encryption block
2443     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2444     const Register pos         = rax;
2445 
2446     // xmm register assignments for the loops below
2447     const XMMRegister xmm_result = xmm0;
2448     const XMMRegister xmm_temp   = xmm1;
2449     // first 6 keys preloaded into xmm2-xmm7
2450     const int XMM_REG_NUM_KEY_FIRST = 2;
2451     const int XMM_REG_NUM_KEY_LAST  = 7;
2452     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
2453 
2454     __ enter(); // required for proper stackwalking of RuntimeStub frame
2455     handleSOERegisters(true /*saving*/);
2456 
2457     // load registers from incoming parameters
2458     const Address  from_param(rbp, 8+0);
2459     const Address  to_param  (rbp, 8+4);
2460     const Address  key_param (rbp, 8+8);
2461     const Address  rvec_param (rbp, 8+12);
2462     const Address  len_param  (rbp, 8+16);
2463     __ movptr(from , from_param);
2464     __ movptr(to   , to_param);
2465     __ movptr(key  , key_param);
2466     __ movptr(rvec , rvec_param);
2467     __ movptr(len_reg , len_param);
2468 
2469     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
2470     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2471     // load up xmm regs 2 thru 7 with keys 0-5
2472     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2473       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
2474       offset += 0x10;
2475     }
2476 
2477     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
2478 
2479     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2480     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2481     __ cmpl(rax, 44);
2482     __ jcc(Assembler::notEqual, L_key_192_256);
2483 
2484     // 128 bit code follows here
2485     __ movl(pos, 0);
2486     __ align(OptoLoopAlignment);
2487     __ BIND(L_loopTop_128);
2488     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2489     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2490 
2491     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2492     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2493       __ aesenc(xmm_result, as_XMMRegister(rnum));
2494     }
2495     for (int key_offset = 0x60; key_offset <= 0x90; key_offset += 0x10) {
2496       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2497     }
2498     load_key(xmm_temp, key, 0xa0);
2499     __ aesenclast(xmm_result, xmm_temp);
2500 
2501     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
2502     // no need to store r to memory until we exit
2503     __ addptr(pos, AESBlockSize);
2504     __ subptr(len_reg, AESBlockSize);
2505     __ jcc(Assembler::notEqual, L_loopTop_128);
2506 
2507     __ BIND(L_exit);
2508     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
2509 
2510     handleSOERegisters(false /*restoring*/);
2511     __ movptr(rax, len_param); // return length
2512     __ leave();                                  // required for proper stackwalking of RuntimeStub frame
2513     __ ret(0);
2514 
2515     __ BIND(L_key_192_256);
2516     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
2517     __ cmpl(rax, 52);
2518     __ jcc(Assembler::notEqual, L_key_256);
2519 
2520     // 192-bit code follows here (could be changed to use more xmm registers)
2521     __ movl(pos, 0);
2522     __ align(OptoLoopAlignment);
2523     __ BIND(L_loopTop_192);
2524     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2525     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2526 
2527     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2528     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2529       __ aesenc(xmm_result, as_XMMRegister(rnum));
2530     }
2531     for (int key_offset = 0x60; key_offset <= 0xb0; key_offset += 0x10) {
2532       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2533     }
2534     load_key(xmm_temp, key, 0xc0);
2535     __ aesenclast(xmm_result, xmm_temp);
2536 
2537     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2538     // no need to store r to memory until we exit
2539     __ addptr(pos, AESBlockSize);
2540     __ subptr(len_reg, AESBlockSize);
2541     __ jcc(Assembler::notEqual, L_loopTop_192);
2542     __ jmp(L_exit);
2543 
2544     __ BIND(L_key_256);
2545     // 256-bit code follows here (could be changed to use more xmm registers)
2546     __ movl(pos, 0);
2547     __ align(OptoLoopAlignment);
2548     __ BIND(L_loopTop_256);
2549     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2550     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2551 
2552     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2553     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2554       __ aesenc(xmm_result, as_XMMRegister(rnum));
2555     }
2556     for (int key_offset = 0x60; key_offset <= 0xd0; key_offset += 0x10) {
2557       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2558     }
2559     load_key(xmm_temp, key, 0xe0);
2560     __ aesenclast(xmm_result, xmm_temp);
2561 
2562     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2563     // no need to store r to memory until we exit
2564     __ addptr(pos, AESBlockSize);
2565     __ subptr(len_reg, AESBlockSize);
2566     __ jcc(Assembler::notEqual, L_loopTop_256);
2567     __ jmp(L_exit);
2568 
2569     return start;
2570   }
2571 
2572 
2573   // CBC AES Decryption.
2574   // In 32-bit stub, because of lack of registers we do not try to parallelize 4 blocks at a time.
2575   //
2576   // Arguments:
2577   //
2578   // Inputs:
2579   //   c_rarg0   - source byte array address
2580   //   c_rarg1   - destination byte array address
2581   //   c_rarg2   - K (key) in little endian int array
2582   //   c_rarg3   - r vector byte array address
2583   //   c_rarg4   - input length
2584   //
2585   // Output:
2586   //   rax       - input length
2587   //
2588 
2589   address generate_cipherBlockChaining_decryptAESCrypt() {
2590     assert(UseAES, "need AES instructions and misaligned SSE support");
2591     __ align(CodeEntryAlignment);
2592     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
2593     address start = __ pc();
2594 
2595     Label L_exit, L_key_192_256, L_key_256;
2596     Label L_singleBlock_loopTop_128;
2597     Label L_singleBlock_loopTop_192, L_singleBlock_loopTop_256;
2598     const Register from        = rsi;      // source array address
2599     const Register to          = rdx;      // destination array address
2600     const Register key         = rcx;      // key array address
2601     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2602                                            // and left with the results of the last encryption block
2603     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2604     const Register pos         = rax;
2605 
2606     // xmm register assignments for the loops below
2607     const XMMRegister xmm_result = xmm0;
2608     const XMMRegister xmm_temp   = xmm1;
2609     // first 6 keys preloaded into xmm2-xmm7
2610     const int XMM_REG_NUM_KEY_FIRST = 2;
2611     const int XMM_REG_NUM_KEY_LAST  = 7;
2612     const int FIRST_NON_REG_KEY_offset = 0x70;
2613     const XMMRegister xmm_key_first   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
2614 
2615     __ enter(); // required for proper stackwalking of RuntimeStub frame
2616     handleSOERegisters(true /*saving*/);
2617 
2618     // load registers from incoming parameters
2619     const Address  from_param(rbp, 8+0);
2620     const Address  to_param  (rbp, 8+4);
2621     const Address  key_param (rbp, 8+8);
2622     const Address  rvec_param (rbp, 8+12);
2623     const Address  len_param  (rbp, 8+16);
2624     __ movptr(from , from_param);
2625     __ movptr(to   , to_param);
2626     __ movptr(key  , key_param);
2627     __ movptr(rvec , rvec_param);
2628     __ movptr(len_reg , len_param);
2629 
2630     // the java expanded key ordering is rotated one position from what we want
2631     // so we start from 0x10 here and hit 0x00 last
2632     const XMMRegister xmm_key_shuf_mask = xmm1;  // used temporarily to swap key bytes up front
2633     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2634     // load up xmm regs 2 thru 6 with first 5 keys
2635     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x10; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2636       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
2637       offset += 0x10;
2638     }
2639 
2640     // inside here, use the rvec register to point to previous block cipher
2641     // with which we xor at the end of each newly decrypted block
2642     const Register  prev_block_cipher_ptr = rvec;
2643 
2644     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2645     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2646     __ cmpl(rax, 44);
2647     __ jcc(Assembler::notEqual, L_key_192_256);
2648 
2649 
2650     // 128-bit code follows here, parallelized
2651     __ movl(pos, 0);
2652     __ align(OptoLoopAlignment);
2653     __ BIND(L_singleBlock_loopTop_128);
2654     __ cmpptr(len_reg, 0);           // any blocks left??
2655     __ jcc(Assembler::equal, L_exit);
2656     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of cipher input
2657     __ pxor  (xmm_result, xmm_key_first);                             // do the aes dec rounds
2658     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2659       __ aesdec(xmm_result, as_XMMRegister(rnum));
2660     }
2661     for (int key_offset = FIRST_NON_REG_KEY_offset; key_offset <= 0xa0; key_offset += 0x10) {   // 128-bit runs up to key offset a0
2662       aes_dec_key(xmm_result, xmm_temp, key, key_offset);
2663     }
2664     load_key(xmm_temp, key, 0x00);                                     // final key is stored in java expanded array at offset 0
2665     __ aesdeclast(xmm_result, xmm_temp);
2666     __ movdqu(xmm_temp, Address(prev_block_cipher_ptr, 0x00));
2667     __ pxor  (xmm_result, xmm_temp);                                  // xor with the current r vector
2668     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
2669     // no need to store r to memory until we exit
2670     __ lea(prev_block_cipher_ptr, Address(from, pos, Address::times_1, 0));     // set up new ptr
2671     __ addptr(pos, AESBlockSize);
2672     __ subptr(len_reg, AESBlockSize);
2673     __ jmp(L_singleBlock_loopTop_128);
2674 
2675 
2676     __ BIND(L_exit);
2677     __ movdqu(xmm_temp, Address(prev_block_cipher_ptr, 0x00));
2678     __ movptr(rvec , rvec_param);                                     // restore this since used in loop
2679     __ movdqu(Address(rvec, 0), xmm_temp);                            // final value of r stored in rvec of CipherBlockChaining object
2680     handleSOERegisters(false /*restoring*/);
2681     __ movptr(rax, len_param); // return length
2682     __ leave();                                                       // required for proper stackwalking of RuntimeStub frame
2683     __ ret(0);
2684 
2685 
2686     __ BIND(L_key_192_256);
2687     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
2688     __ cmpl(rax, 52);
2689     __ jcc(Assembler::notEqual, L_key_256);
2690 
2691     // 192-bit code follows here (could be optimized to use parallelism)
2692     __ movl(pos, 0);
2693     __ align(OptoLoopAlignment);
2694     __ BIND(L_singleBlock_loopTop_192);
2695     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of cipher input
2696     __ pxor  (xmm_result, xmm_key_first);                             // do the aes dec rounds
2697     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST; rnum++) {
2698       __ aesdec(xmm_result, as_XMMRegister(rnum));
2699     }
2700     for (int key_offset = FIRST_NON_REG_KEY_offset; key_offset <= 0xc0; key_offset += 0x10) {   // 192-bit runs up to key offset c0
2701       aes_dec_key(xmm_result, xmm_temp, key, key_offset);
2702     }
2703     load_key(xmm_temp, key, 0x00);                                     // final key is stored in java expanded array at offset 0
2704     __ aesdeclast(xmm_result, xmm_temp);
2705     __ movdqu(xmm_temp, Address(prev_block_cipher_ptr, 0x00));
2706     __ pxor  (xmm_result, xmm_temp);                                  // xor with the current r vector
2707     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
2708     // no need to store r to memory until we exit
2709     __ lea(prev_block_cipher_ptr, Address(from, pos, Address::times_1, 0));     // set up new ptr
2710     __ addptr(pos, AESBlockSize);
2711     __ subptr(len_reg, AESBlockSize);
2712     __ jcc(Assembler::notEqual,L_singleBlock_loopTop_192);
2713     __ jmp(L_exit);
2714 
2715     __ BIND(L_key_256);
2716     // 256-bit code follows here (could be optimized to use parallelism)
2717     __ movl(pos, 0);
2718     __ align(OptoLoopAlignment);
2719     __ BIND(L_singleBlock_loopTop_256);
2720     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of cipher input
2721     __ pxor  (xmm_result, xmm_key_first);                             // do the aes dec rounds
2722     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST; rnum++) {
2723       __ aesdec(xmm_result, as_XMMRegister(rnum));
2724     }
2725     for (int key_offset = FIRST_NON_REG_KEY_offset; key_offset <= 0xe0; key_offset += 0x10) {   // 256-bit runs up to key offset e0
2726       aes_dec_key(xmm_result, xmm_temp, key, key_offset);
2727     }
2728     load_key(xmm_temp, key, 0x00);                                     // final key is stored in java expanded array at offset 0
2729     __ aesdeclast(xmm_result, xmm_temp);
2730     __ movdqu(xmm_temp, Address(prev_block_cipher_ptr, 0x00));
2731     __ pxor  (xmm_result, xmm_temp);                                  // xor with the current r vector
2732     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
2733     // no need to store r to memory until we exit
2734     __ lea(prev_block_cipher_ptr, Address(from, pos, Address::times_1, 0));     // set up new ptr
2735     __ addptr(pos, AESBlockSize);
2736     __ subptr(len_reg, AESBlockSize);
2737     __ jcc(Assembler::notEqual,L_singleBlock_loopTop_256);
2738     __ jmp(L_exit);
2739 
2740     return start;
2741   }
2742 
2743   // byte swap x86 long
2744   address generate_ghash_long_swap_mask() {
2745     __ align(CodeEntryAlignment);
2746     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
2747     address start = __ pc();
2748     __ emit_data(0x0b0a0908, relocInfo::none, 0);
2749     __ emit_data(0x0f0e0d0c, relocInfo::none, 0);
2750     __ emit_data(0x03020100, relocInfo::none, 0);
2751     __ emit_data(0x07060504, relocInfo::none, 0);
2752 
2753   return start;
2754   }
2755 
2756   // byte swap x86 byte array
2757   address generate_ghash_byte_swap_mask() {
2758     __ align(CodeEntryAlignment);
2759     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
2760     address start = __ pc();
2761     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
2762     __ emit_data(0x08090a0b, relocInfo::none, 0);
2763     __ emit_data(0x04050607, relocInfo::none, 0);
2764     __ emit_data(0x00010203, relocInfo::none, 0);
2765   return start;
2766   }
2767 
2768   /* Single and multi-block ghash operations */
2769   address generate_ghash_processBlocks() {
2770     assert(UseGHASHIntrinsics, "need GHASH intrinsics and CLMUL support");
2771     __ align(CodeEntryAlignment);
2772     Label L_ghash_loop, L_exit;
2773     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
2774     address start = __ pc();
2775 
2776     const Register state        = rdi;
2777     const Register subkeyH      = rsi;
2778     const Register data         = rdx;
2779     const Register blocks       = rcx;
2780 
2781     const Address  state_param(rbp, 8+0);
2782     const Address  subkeyH_param(rbp, 8+4);
2783     const Address  data_param(rbp, 8+8);
2784     const Address  blocks_param(rbp, 8+12);
2785 
2786     const XMMRegister xmm_temp0 = xmm0;
2787     const XMMRegister xmm_temp1 = xmm1;
2788     const XMMRegister xmm_temp2 = xmm2;
2789     const XMMRegister xmm_temp3 = xmm3;
2790     const XMMRegister xmm_temp4 = xmm4;
2791     const XMMRegister xmm_temp5 = xmm5;
2792     const XMMRegister xmm_temp6 = xmm6;
2793     const XMMRegister xmm_temp7 = xmm7;
2794 
2795     __ enter();
2796     handleSOERegisters(true);  // Save registers
2797 
2798     __ movptr(state, state_param);
2799     __ movptr(subkeyH, subkeyH_param);
2800     __ movptr(data, data_param);
2801     __ movptr(blocks, blocks_param);
2802 
2803     __ movdqu(xmm_temp0, Address(state, 0));
2804     __ pshufb(xmm_temp0, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
2805 
2806     __ movdqu(xmm_temp1, Address(subkeyH, 0));
2807     __ pshufb(xmm_temp1, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
2808 
2809     __ BIND(L_ghash_loop);
2810     __ movdqu(xmm_temp2, Address(data, 0));
2811     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
2812 
2813     __ pxor(xmm_temp0, xmm_temp2);
2814 
2815     //
2816     // Multiply with the hash key
2817     //
2818     __ movdqu(xmm_temp3, xmm_temp0);
2819     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
2820     __ movdqu(xmm_temp4, xmm_temp0);
2821     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
2822 
2823     __ movdqu(xmm_temp5, xmm_temp0);
2824     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
2825     __ movdqu(xmm_temp6, xmm_temp0);
2826     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
2827 
2828     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
2829 
2830     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
2831     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
2832     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
2833     __ pxor(xmm_temp3, xmm_temp5);
2834     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
2835                                         // of the carry-less multiplication of
2836                                         // xmm0 by xmm1.
2837 
2838     // We shift the result of the multiplication by one bit position
2839     // to the left to cope for the fact that the bits are reversed.
2840     __ movdqu(xmm_temp7, xmm_temp3);
2841     __ movdqu(xmm_temp4, xmm_temp6);
2842     __ pslld (xmm_temp3, 1);
2843     __ pslld(xmm_temp6, 1);
2844     __ psrld(xmm_temp7, 31);
2845     __ psrld(xmm_temp4, 31);
2846     __ movdqu(xmm_temp5, xmm_temp7);
2847     __ pslldq(xmm_temp4, 4);
2848     __ pslldq(xmm_temp7, 4);
2849     __ psrldq(xmm_temp5, 12);
2850     __ por(xmm_temp3, xmm_temp7);
2851     __ por(xmm_temp6, xmm_temp4);
2852     __ por(xmm_temp6, xmm_temp5);
2853 
2854     //
2855     // First phase of the reduction
2856     //
2857     // Move xmm3 into xmm4, xmm5, xmm7 in order to perform the shifts
2858     // independently.
2859     __ movdqu(xmm_temp7, xmm_temp3);
2860     __ movdqu(xmm_temp4, xmm_temp3);
2861     __ movdqu(xmm_temp5, xmm_temp3);
2862     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
2863     __ pslld(xmm_temp4, 30);    // packed right shift shifting << 30
2864     __ pslld(xmm_temp5, 25);    // packed right shift shifting << 25
2865     __ pxor(xmm_temp7, xmm_temp4);      // xor the shifted versions
2866     __ pxor(xmm_temp7, xmm_temp5);
2867     __ movdqu(xmm_temp4, xmm_temp7);
2868     __ pslldq(xmm_temp7, 12);
2869     __ psrldq(xmm_temp4, 4);
2870     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
2871 
2872     //
2873     // Second phase of the reduction
2874     //
2875     // Make 3 copies of xmm3 in xmm2, xmm5, xmm7 for doing these
2876     // shift operations.
2877     __ movdqu(xmm_temp2, xmm_temp3);
2878     __ movdqu(xmm_temp7, xmm_temp3);
2879     __ movdqu(xmm_temp5, xmm_temp3);
2880     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
2881     __ psrld(xmm_temp7, 2);     // packed left shifting >> 2
2882     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
2883     __ pxor(xmm_temp2, xmm_temp7);      // xor the shifted versions
2884     __ pxor(xmm_temp2, xmm_temp5);
2885     __ pxor(xmm_temp2, xmm_temp4);
2886     __ pxor(xmm_temp3, xmm_temp2);
2887     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
2888 
2889     __ decrement(blocks);
2890     __ jcc(Assembler::zero, L_exit);
2891     __ movdqu(xmm_temp0, xmm_temp6);
2892     __ addptr(data, 16);
2893     __ jmp(L_ghash_loop);
2894 
2895     __ BIND(L_exit);
2896        // Byte swap 16-byte result
2897     __ pshufb(xmm_temp6, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
2898     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
2899 
2900     handleSOERegisters(false);  // restore registers
2901     __ leave();
2902     __ ret(0);
2903     return start;
2904   }
2905 
2906   /**
2907    *  Arguments:
2908    *
2909    * Inputs:
2910    *   rsp(4)   - int crc
2911    *   rsp(8)   - byte* buf
2912    *   rsp(12)  - int length
2913    *
2914    * Ouput:
2915    *       rax   - int crc result
2916    */
2917   address generate_updateBytesCRC32() {
2918     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
2919 
2920     __ align(CodeEntryAlignment);
2921     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
2922 
2923     address start = __ pc();
2924 
2925     const Register crc   = rdx;  // crc
2926     const Register buf   = rsi;  // source java byte array address
2927     const Register len   = rcx;  // length
2928     const Register table = rdi;  // crc_table address (reuse register)
2929     const Register tmp   = rbx;
2930     assert_different_registers(crc, buf, len, table, tmp, rax);
2931 
2932     BLOCK_COMMENT("Entry:");
2933     __ enter(); // required for proper stackwalking of RuntimeStub frame
2934     __ push(rsi);
2935     __ push(rdi);
2936     __ push(rbx);
2937 
2938     Address crc_arg(rbp, 8 + 0);
2939     Address buf_arg(rbp, 8 + 4);
2940     Address len_arg(rbp, 8 + 8);
2941 
2942     // Load up:
2943     __ movl(crc,   crc_arg);
2944     __ movptr(buf, buf_arg);
2945     __ movl(len,   len_arg);
2946 
2947     __ kernel_crc32(crc, buf, len, table, tmp);
2948 
2949     __ movl(rax, crc);
2950     __ pop(rbx);
2951     __ pop(rdi);
2952     __ pop(rsi);
2953     __ leave(); // required for proper stackwalking of RuntimeStub frame
2954     __ ret(0);
2955 
2956     return start;
2957   }
2958 
2959   // Safefetch stubs.
2960   void generate_safefetch(const char* name, int size, address* entry,
2961                           address* fault_pc, address* continuation_pc) {
2962     // safefetch signatures:
2963     //   int      SafeFetch32(int*      adr, int      errValue);
2964     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
2965 
2966     StubCodeMark mark(this, "StubRoutines", name);
2967 
2968     // Entry point, pc or function descriptor.
2969     *entry = __ pc();
2970 
2971     __ movl(rax, Address(rsp, 0x8));
2972     __ movl(rcx, Address(rsp, 0x4));
2973     // Load *adr into eax, may fault.
2974     *fault_pc = __ pc();
2975     switch (size) {
2976       case 4:
2977         // int32_t
2978         __ movl(rax, Address(rcx, 0));
2979         break;
2980       case 8:
2981         // int64_t
2982         Unimplemented();
2983         break;
2984       default:
2985         ShouldNotReachHere();
2986     }
2987 
2988     // Return errValue or *adr.
2989     *continuation_pc = __ pc();
2990     __ ret(0);
2991   }
2992 
2993  public:
2994   // Information about frame layout at time of blocking runtime call.
2995   // Note that we only have to preserve callee-saved registers since
2996   // the compilers are responsible for supplying a continuation point
2997   // if they expect all registers to be preserved.
2998   enum layout {
2999     thread_off,    // last_java_sp
3000     arg1_off,
3001     arg2_off,
3002     rbp_off,       // callee saved register
3003     ret_pc,
3004     framesize
3005   };
3006 
3007  private:
3008 
3009 #undef  __
3010 #define __ masm->
3011 
3012   //------------------------------------------------------------------------------------------------------------------------
3013   // Continuation point for throwing of implicit exceptions that are not handled in
3014   // the current activation. Fabricates an exception oop and initiates normal
3015   // exception dispatching in this frame.
3016   //
3017   // Previously the compiler (c2) allowed for callee save registers on Java calls.
3018   // This is no longer true after adapter frames were removed but could possibly
3019   // be brought back in the future if the interpreter code was reworked and it
3020   // was deemed worthwhile. The comment below was left to describe what must
3021   // happen here if callee saves were resurrected. As it stands now this stub
3022   // could actually be a vanilla BufferBlob and have now oopMap at all.
3023   // Since it doesn't make much difference we've chosen to leave it the
3024   // way it was in the callee save days and keep the comment.
3025 
3026   // If we need to preserve callee-saved values we need a callee-saved oop map and
3027   // therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
3028   // If the compiler needs all registers to be preserved between the fault
3029   // point and the exception handler then it must assume responsibility for that in
3030   // AbstractCompiler::continuation_for_implicit_null_exception or
3031   // continuation_for_implicit_division_by_zero_exception. All other implicit
3032   // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
3033   // either at call sites or otherwise assume that stack unwinding will be initiated,
3034   // so caller saved registers were assumed volatile in the compiler.
3035   address generate_throw_exception(const char* name, address runtime_entry,
3036                                    Register arg1 = noreg, Register arg2 = noreg) {
3037 
3038     int insts_size = 256;
3039     int locs_size  = 32;
3040 
3041     CodeBuffer code(name, insts_size, locs_size);
3042     OopMapSet* oop_maps  = new OopMapSet();
3043     MacroAssembler* masm = new MacroAssembler(&code);
3044 
3045     address start = __ pc();
3046 
3047     // This is an inlined and slightly modified version of call_VM
3048     // which has the ability to fetch the return PC out of
3049     // thread-local storage and also sets up last_Java_sp slightly
3050     // differently than the real call_VM
3051     Register java_thread = rbx;
3052     __ get_thread(java_thread);
3053 
3054     __ enter(); // required for proper stackwalking of RuntimeStub frame
3055 
3056     // pc and rbp, already pushed
3057     __ subptr(rsp, (framesize-2) * wordSize); // prolog
3058 
3059     // Frame is now completed as far as size and linkage.
3060 
3061     int frame_complete = __ pc() - start;
3062 
3063     // push java thread (becomes first argument of C function)
3064     __ movptr(Address(rsp, thread_off * wordSize), java_thread);
3065     if (arg1 != noreg) {
3066       __ movptr(Address(rsp, arg1_off * wordSize), arg1);
3067     }
3068     if (arg2 != noreg) {
3069       assert(arg1 != noreg, "missing reg arg");
3070       __ movptr(Address(rsp, arg2_off * wordSize), arg2);
3071     }
3072 
3073     // Set up last_Java_sp and last_Java_fp
3074     __ set_last_Java_frame(java_thread, rsp, rbp, NULL);
3075 
3076     // Call runtime
3077     BLOCK_COMMENT("call runtime_entry");
3078     __ call(RuntimeAddress(runtime_entry));
3079     // Generate oop map
3080     OopMap* map =  new OopMap(framesize, 0);
3081     oop_maps->add_gc_map(__ pc() - start, map);
3082 
3083     // restore the thread (cannot use the pushed argument since arguments
3084     // may be overwritten by C code generated by an optimizing compiler);
3085     // however can use the register value directly if it is callee saved.
3086     __ get_thread(java_thread);
3087 
3088     __ reset_last_Java_frame(java_thread, true);
3089 
3090     __ leave(); // required for proper stackwalking of RuntimeStub frame
3091 
3092     // check for pending exceptions
3093 #ifdef ASSERT
3094     Label L;
3095     __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3096     __ jcc(Assembler::notEqual, L);
3097     __ should_not_reach_here();
3098     __ bind(L);
3099 #endif /* ASSERT */
3100     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3101 
3102 
3103     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
3104     return stub->entry_point();
3105   }
3106 
3107 
3108   void create_control_words() {
3109     // Round to nearest, 53-bit mode, exceptions masked
3110     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
3111     // Round to zero, 53-bit mode, exception mased
3112     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
3113     // Round to nearest, 24-bit mode, exceptions masked
3114     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
3115     // Round to nearest, 64-bit mode, exceptions masked
3116     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;
3117     // Round to nearest, 64-bit mode, exceptions masked
3118     StubRoutines::_mxcsr_std           = 0x1F80;
3119     // Note: the following two constants are 80-bit values
3120     //       layout is critical for correct loading by FPU.
3121     // Bias for strict fp multiply/divide
3122     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
3123     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
3124     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
3125     // Un-Bias for strict fp multiply/divide
3126     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
3127     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
3128     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
3129   }
3130 
3131   //---------------------------------------------------------------------------
3132   // Initialization
3133 
3134   void generate_initial() {
3135     // Generates all stubs and initializes the entry points
3136 
3137     //------------------------------------------------------------------------------------------------------------------------
3138     // entry points that exist in all platforms
3139     // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
3140     //       the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
3141     StubRoutines::_forward_exception_entry      = generate_forward_exception();
3142 
3143     StubRoutines::_call_stub_entry              =
3144       generate_call_stub(StubRoutines::_call_stub_return_address);
3145     // is referenced by megamorphic call
3146     StubRoutines::_catch_exception_entry        = generate_catch_exception();
3147 
3148     // These are currently used by Solaris/Intel
3149     StubRoutines::_atomic_xchg_entry            = generate_atomic_xchg();
3150 
3151     StubRoutines::_handler_for_unsafe_access_entry =
3152       generate_handler_for_unsafe_access();
3153 
3154     // platform dependent
3155     create_control_words();
3156 
3157     StubRoutines::x86::_verify_mxcsr_entry                 = generate_verify_mxcsr();
3158     StubRoutines::x86::_verify_fpu_cntrl_wrd_entry         = generate_verify_fpu_cntrl_wrd();
3159     StubRoutines::_d2i_wrapper                              = generate_d2i_wrapper(T_INT,
3160                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
3161     StubRoutines::_d2l_wrapper                              = generate_d2i_wrapper(T_LONG,
3162                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
3163 
3164     // Build this early so it's available for the interpreter
3165     StubRoutines::_throw_StackOverflowError_entry          = generate_throw_exception("StackOverflowError throw_exception",           CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
3166 
3167     if (UseCRC32Intrinsics) {
3168       // set table address before stub generation which use it
3169       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
3170       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
3171     }
3172   }
3173 
3174 
3175   void generate_all() {
3176     // Generates all stubs and initializes the entry points
3177 
3178     // These entry points require SharedInfo::stack0 to be set up in non-core builds
3179     // and need to be relocatable, so they each fabricate a RuntimeStub internally.
3180     StubRoutines::_throw_AbstractMethodError_entry         = generate_throw_exception("AbstractMethodError throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
3181     StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
3182     StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
3183 
3184     //------------------------------------------------------------------------------------------------------------------------
3185     // entry points that are platform specific
3186 
3187     // support for verify_oop (must happen after universe_init)
3188     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
3189 
3190     // arraycopy stubs used by compilers
3191     generate_arraycopy_stubs();
3192 
3193     generate_math_stubs();
3194 
3195     // don't bother generating these AES intrinsic stubs unless global flag is set
3196     if (UseAESIntrinsics) {
3197       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // might be needed by the others
3198 
3199       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
3200       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
3201       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
3202       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt();
3203     }
3204 
3205     // Generate GHASH intrinsics code
3206     if (UseGHASHIntrinsics) {
3207       StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
3208       StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
3209       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
3210     }
3211 
3212     // Safefetch stubs.
3213     generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
3214                                                    &StubRoutines::_safefetch32_fault_pc,
3215                                                    &StubRoutines::_safefetch32_continuation_pc);
3216     StubRoutines::_safefetchN_entry           = StubRoutines::_safefetch32_entry;
3217     StubRoutines::_safefetchN_fault_pc        = StubRoutines::_safefetch32_fault_pc;
3218     StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc;
3219   }
3220 
3221 
3222  public:
3223   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
3224     if (all) {
3225       generate_all();
3226     } else {
3227       generate_initial();
3228     }
3229   }
3230 }; // end class declaration
3231 
3232 
3233 void StubGenerator_generate(CodeBuffer* code, bool all) {
3234   StubGenerator g(code, all);
3235 }