1 /*
   2  * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "nativeInst_x86.hpp"
  30 #include "oops/instanceOop.hpp"
  31 #include "oops/method.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "oops/oop.inline.hpp"
  34 #include "prims/methodHandles.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/handles.inline.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubCodeGenerator.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/thread.inline.hpp"
  41 #include "utilities/macros.hpp"
  42 #include "utilities/top.hpp"
  43 #ifdef COMPILER2
  44 #include "opto/runtime.hpp"
  45 #endif
  46 #if INCLUDE_ALL_GCS
  47 #include "shenandoahBarrierSetAssembler_x86.hpp"
  48 #endif
  49 
  50 // Declaration and definition of StubGenerator (no .hpp file).
  51 // For a more detailed description of the stub routine structure
  52 // see the comment in stubRoutines.hpp
  53 
  54 #define __ _masm->
  55 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  56 #define a__ ((Assembler*)_masm)->
  57 
  58 #ifdef PRODUCT
  59 #define BLOCK_COMMENT(str) /* nothing */
  60 #else
  61 #define BLOCK_COMMENT(str) __ block_comment(str)
  62 #endif
  63 
  64 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  65 const int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
  66 
  67 // Stub Code definitions
  68 
  69 static address handle_unsafe_access() {
  70   JavaThread* thread = JavaThread::current();
  71   address pc = thread->saved_exception_pc();
  72   // pc is the instruction which we must emulate
  73   // doing a no-op is fine:  return garbage from the load
  74   // therefore, compute npc
  75   address npc = Assembler::locate_next_instruction(pc);
  76 
  77   // request an async exception
  78   thread->set_pending_unsafe_access_error();
  79 
  80   // return address of next instruction to execute
  81   return npc;
  82 }
  83 
  84 class StubGenerator: public StubCodeGenerator {
  85  private:
  86 
  87 #ifdef PRODUCT
  88 #define inc_counter_np(counter) ((void)0)
  89 #else
  90   void inc_counter_np_(int& counter) {
  91     // This can destroy rscratch1 if counter is far from the code cache
  92     __ incrementl(ExternalAddress((address)&counter));
  93   }
  94 #define inc_counter_np(counter) \
  95   BLOCK_COMMENT("inc_counter " #counter); \
  96   inc_counter_np_(counter);
  97 #endif
  98 
  99   // Call stubs are used to call Java from C
 100   //
 101   // Linux Arguments:
 102   //    c_rarg0:   call wrapper address                   address
 103   //    c_rarg1:   result                                 address
 104   //    c_rarg2:   result type                            BasicType
 105   //    c_rarg3:   method                                 Method*
 106   //    c_rarg4:   (interpreter) entry point              address
 107   //    c_rarg5:   parameters                             intptr_t*
 108   //    16(rbp): parameter size (in words)              int
 109   //    24(rbp): thread                                 Thread*
 110   //
 111   //     [ return_from_Java     ] <--- rsp
 112   //     [ argument word n      ]
 113   //      ...
 114   // -12 [ argument word 1      ]
 115   // -11 [ saved r15            ] <--- rsp_after_call
 116   // -10 [ saved r14            ]
 117   //  -9 [ saved r13            ]
 118   //  -8 [ saved r12            ]
 119   //  -7 [ saved rbx            ]
 120   //  -6 [ call wrapper         ]
 121   //  -5 [ result               ]
 122   //  -4 [ result type          ]
 123   //  -3 [ method               ]
 124   //  -2 [ entry point          ]
 125   //  -1 [ parameters           ]
 126   //   0 [ saved rbp            ] <--- rbp
 127   //   1 [ return address       ]
 128   //   2 [ parameter size       ]
 129   //   3 [ thread               ]
 130   //
 131   // Windows Arguments:
 132   //    c_rarg0:   call wrapper address                   address
 133   //    c_rarg1:   result                                 address
 134   //    c_rarg2:   result type                            BasicType
 135   //    c_rarg3:   method                                 Method*
 136   //    48(rbp): (interpreter) entry point              address
 137   //    56(rbp): parameters                             intptr_t*
 138   //    64(rbp): parameter size (in words)              int
 139   //    72(rbp): thread                                 Thread*
 140   //
 141   //     [ return_from_Java     ] <--- rsp
 142   //     [ argument word n      ]
 143   //      ...
 144   // -28 [ argument word 1      ]
 145   // -27 [ saved xmm15          ] <--- rsp_after_call
 146   //     [ saved xmm7-xmm14     ]
 147   //  -9 [ saved xmm6           ] (each xmm register takes 2 slots)
 148   //  -7 [ saved r15            ]
 149   //  -6 [ saved r14            ]
 150   //  -5 [ saved r13            ]
 151   //  -4 [ saved r12            ]
 152   //  -3 [ saved rdi            ]
 153   //  -2 [ saved rsi            ]
 154   //  -1 [ saved rbx            ]
 155   //   0 [ saved rbp            ] <--- rbp
 156   //   1 [ return address       ]
 157   //   2 [ call wrapper         ]
 158   //   3 [ result               ]
 159   //   4 [ result type          ]
 160   //   5 [ method               ]
 161   //   6 [ entry point          ]
 162   //   7 [ parameters           ]
 163   //   8 [ parameter size       ]
 164   //   9 [ thread               ]
 165   //
 166   //    Windows reserves the callers stack space for arguments 1-4.
 167   //    We spill c_rarg0-c_rarg3 to this space.
 168 
 169   // Call stub stack layout word offsets from rbp
 170   enum call_stub_layout {
 171 #ifdef _WIN64
 172     xmm_save_first     = 6,  // save from xmm6
 173     xmm_save_last      = 15, // to xmm15
 174     xmm_save_base      = -9,
 175     rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27
 176     r15_off            = -7,
 177     r14_off            = -6,
 178     r13_off            = -5,
 179     r12_off            = -4,
 180     rdi_off            = -3,
 181     rsi_off            = -2,
 182     rbx_off            = -1,
 183     rbp_off            =  0,
 184     retaddr_off        =  1,
 185     call_wrapper_off   =  2,
 186     result_off         =  3,
 187     result_type_off    =  4,
 188     method_off         =  5,
 189     entry_point_off    =  6,
 190     parameters_off     =  7,
 191     parameter_size_off =  8,
 192     thread_off         =  9
 193 #else
 194     rsp_after_call_off = -12,
 195     mxcsr_off          = rsp_after_call_off,
 196     r15_off            = -11,
 197     r14_off            = -10,
 198     r13_off            = -9,
 199     r12_off            = -8,
 200     rbx_off            = -7,
 201     call_wrapper_off   = -6,
 202     result_off         = -5,
 203     result_type_off    = -4,
 204     method_off         = -3,
 205     entry_point_off    = -2,
 206     parameters_off     = -1,
 207     rbp_off            =  0,
 208     retaddr_off        =  1,
 209     parameter_size_off =  2,
 210     thread_off         =  3
 211 #endif
 212   };
 213 
 214 #ifdef _WIN64
 215   Address xmm_save(int reg) {
 216     assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range");
 217     return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize);
 218   }
 219 #endif
 220 
 221   address generate_call_stub(address& return_address) {
 222     assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&
 223            (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
 224            "adjust this code");
 225     StubCodeMark mark(this, "StubRoutines", "call_stub");
 226     address start = __ pc();
 227 
 228     // same as in generate_catch_exception()!
 229     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 230 
 231     const Address call_wrapper  (rbp, call_wrapper_off   * wordSize);
 232     const Address result        (rbp, result_off         * wordSize);
 233     const Address result_type   (rbp, result_type_off    * wordSize);
 234     const Address method        (rbp, method_off         * wordSize);
 235     const Address entry_point   (rbp, entry_point_off    * wordSize);
 236     const Address parameters    (rbp, parameters_off     * wordSize);
 237     const Address parameter_size(rbp, parameter_size_off * wordSize);
 238 
 239     // same as in generate_catch_exception()!
 240     const Address thread        (rbp, thread_off         * wordSize);
 241 
 242     const Address r15_save(rbp, r15_off * wordSize);
 243     const Address r14_save(rbp, r14_off * wordSize);
 244     const Address r13_save(rbp, r13_off * wordSize);
 245     const Address r12_save(rbp, r12_off * wordSize);
 246     const Address rbx_save(rbp, rbx_off * wordSize);
 247 
 248     // stub code
 249     __ enter();
 250     __ subptr(rsp, -rsp_after_call_off * wordSize);
 251 
 252     // save register parameters
 253 #ifndef _WIN64
 254     __ movptr(parameters,   c_rarg5); // parameters
 255     __ movptr(entry_point,  c_rarg4); // entry_point
 256 #endif
 257 
 258     __ movptr(method,       c_rarg3); // method
 259     __ movl(result_type,  c_rarg2);   // result type
 260     __ movptr(result,       c_rarg1); // result
 261     __ movptr(call_wrapper, c_rarg0); // call wrapper
 262 
 263     // save regs belonging to calling function
 264     __ movptr(rbx_save, rbx);
 265     __ movptr(r12_save, r12);
 266     __ movptr(r13_save, r13);
 267     __ movptr(r14_save, r14);
 268     __ movptr(r15_save, r15);
 269 #ifdef _WIN64
 270     for (int i = 6; i <= 15; i++) {
 271       __ movdqu(xmm_save(i), as_XMMRegister(i));
 272     }
 273 
 274     const Address rdi_save(rbp, rdi_off * wordSize);
 275     const Address rsi_save(rbp, rsi_off * wordSize);
 276 
 277     __ movptr(rsi_save, rsi);
 278     __ movptr(rdi_save, rdi);
 279 #else
 280     const Address mxcsr_save(rbp, mxcsr_off * wordSize);
 281     {
 282       Label skip_ldmx;
 283       __ stmxcsr(mxcsr_save);
 284       __ movl(rax, mxcsr_save);
 285       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 286       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 287       __ cmp32(rax, mxcsr_std);
 288       __ jcc(Assembler::equal, skip_ldmx);
 289       __ ldmxcsr(mxcsr_std);
 290       __ bind(skip_ldmx);
 291     }
 292 #endif
 293 
 294     // Load up thread register
 295     __ movptr(r15_thread, thread);
 296     __ reinit_heapbase();
 297 
 298 #ifdef ASSERT
 299     // make sure we have no pending exceptions
 300     {
 301       Label L;
 302       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 303       __ jcc(Assembler::equal, L);
 304       __ stop("StubRoutines::call_stub: entered with pending exception");
 305       __ bind(L);
 306     }
 307 #endif
 308 
 309     // pass parameters if any
 310     BLOCK_COMMENT("pass parameters if any");
 311     Label parameters_done;
 312     __ movl(c_rarg3, parameter_size);
 313     __ testl(c_rarg3, c_rarg3);
 314     __ jcc(Assembler::zero, parameters_done);
 315 
 316     Label loop;
 317     __ movptr(c_rarg2, parameters);       // parameter pointer
 318     __ movl(c_rarg1, c_rarg3);            // parameter counter is in c_rarg1
 319     __ BIND(loop);
 320     __ movptr(rax, Address(c_rarg2, 0));// get parameter
 321     __ addptr(c_rarg2, wordSize);       // advance to next parameter
 322     __ decrementl(c_rarg1);             // decrement counter
 323     __ push(rax);                       // pass parameter
 324     __ jcc(Assembler::notZero, loop);
 325 
 326     // call Java function
 327     __ BIND(parameters_done);
 328     __ movptr(rbx, method);             // get Method*
 329     __ movptr(c_rarg1, entry_point);    // get entry_point
 330     __ mov(r13, rsp);                   // set sender sp
 331     BLOCK_COMMENT("call Java function");
 332     __ call(c_rarg1);
 333 
 334     BLOCK_COMMENT("call_stub_return_address:");
 335     return_address = __ pc();
 336 
 337     // store result depending on type (everything that is not
 338     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 339     __ movptr(c_rarg0, result);
 340     Label is_long, is_float, is_double, exit;
 341     __ movl(c_rarg1, result_type);
 342     __ cmpl(c_rarg1, T_OBJECT);
 343     __ jcc(Assembler::equal, is_long);
 344     __ cmpl(c_rarg1, T_LONG);
 345     __ jcc(Assembler::equal, is_long);
 346     __ cmpl(c_rarg1, T_FLOAT);
 347     __ jcc(Assembler::equal, is_float);
 348     __ cmpl(c_rarg1, T_DOUBLE);
 349     __ jcc(Assembler::equal, is_double);
 350 
 351     // handle T_INT case
 352     __ movl(Address(c_rarg0, 0), rax);
 353 
 354     __ BIND(exit);
 355 
 356     // pop parameters
 357     __ lea(rsp, rsp_after_call);
 358 
 359 #ifdef ASSERT
 360     // verify that threads correspond
 361     {
 362       Label L, S;
 363       __ cmpptr(r15_thread, thread);
 364       __ jcc(Assembler::notEqual, S);
 365       __ get_thread(rbx);
 366       __ cmpptr(r15_thread, rbx);
 367       __ jcc(Assembler::equal, L);
 368       __ bind(S);
 369       __ jcc(Assembler::equal, L);
 370       __ stop("StubRoutines::call_stub: threads must correspond");
 371       __ bind(L);
 372     }
 373 #endif
 374 
 375     // restore regs belonging to calling function
 376 #ifdef _WIN64
 377     for (int i = 15; i >= 6; i--) {
 378       __ movdqu(as_XMMRegister(i), xmm_save(i));
 379     }
 380 #endif
 381     __ movptr(r15, r15_save);
 382     __ movptr(r14, r14_save);
 383     __ movptr(r13, r13_save);
 384     __ movptr(r12, r12_save);
 385     __ movptr(rbx, rbx_save);
 386 
 387 #ifdef _WIN64
 388     __ movptr(rdi, rdi_save);
 389     __ movptr(rsi, rsi_save);
 390 #else
 391     __ ldmxcsr(mxcsr_save);
 392 #endif
 393 
 394     // restore rsp
 395     __ addptr(rsp, -rsp_after_call_off * wordSize);
 396 
 397     // return
 398     __ pop(rbp);
 399     __ ret(0);
 400 
 401     // handle return types different from T_INT
 402     __ BIND(is_long);
 403     __ movq(Address(c_rarg0, 0), rax);
 404     __ jmp(exit);
 405 
 406     __ BIND(is_float);
 407     __ movflt(Address(c_rarg0, 0), xmm0);
 408     __ jmp(exit);
 409 
 410     __ BIND(is_double);
 411     __ movdbl(Address(c_rarg0, 0), xmm0);
 412     __ jmp(exit);
 413 
 414     return start;
 415   }
 416 
 417   // Return point for a Java call if there's an exception thrown in
 418   // Java code.  The exception is caught and transformed into a
 419   // pending exception stored in JavaThread that can be tested from
 420   // within the VM.
 421   //
 422   // Note: Usually the parameters are removed by the callee. In case
 423   // of an exception crossing an activation frame boundary, that is
 424   // not the case if the callee is compiled code => need to setup the
 425   // rsp.
 426   //
 427   // rax: exception oop
 428 
 429   address generate_catch_exception() {
 430     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 431     address start = __ pc();
 432 
 433     // same as in generate_call_stub():
 434     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 435     const Address thread        (rbp, thread_off         * wordSize);
 436 
 437 #ifdef ASSERT
 438     // verify that threads correspond
 439     {
 440       Label L, S;
 441       __ cmpptr(r15_thread, thread);
 442       __ jcc(Assembler::notEqual, S);
 443       __ get_thread(rbx);
 444       __ cmpptr(r15_thread, rbx);
 445       __ jcc(Assembler::equal, L);
 446       __ bind(S);
 447       __ stop("StubRoutines::catch_exception: threads must correspond");
 448       __ bind(L);
 449     }
 450 #endif
 451 
 452     // set pending exception
 453     __ verify_oop(rax);
 454 
 455     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
 456     __ lea(rscratch1, ExternalAddress((address)__FILE__));
 457     __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
 458     __ movl(Address(r15_thread, Thread::exception_line_offset()), (int)  __LINE__);
 459 
 460     // complete return to VM
 461     assert(StubRoutines::_call_stub_return_address != NULL,
 462            "_call_stub_return_address must have been generated before");
 463     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 464 
 465     return start;
 466   }
 467 
 468   // Continuation point for runtime calls returning with a pending
 469   // exception.  The pending exception check happened in the runtime
 470   // or native call stub.  The pending exception in Thread is
 471   // converted into a Java-level exception.
 472   //
 473   // Contract with Java-level exception handlers:
 474   // rax: exception
 475   // rdx: throwing pc
 476   //
 477   // NOTE: At entry of this stub, exception-pc must be on stack !!
 478 
 479   address generate_forward_exception() {
 480     StubCodeMark mark(this, "StubRoutines", "forward exception");
 481     address start = __ pc();
 482 
 483     // Upon entry, the sp points to the return address returning into
 484     // Java (interpreted or compiled) code; i.e., the return address
 485     // becomes the throwing pc.
 486     //
 487     // Arguments pushed before the runtime call are still on the stack
 488     // but the exception handler will reset the stack pointer ->
 489     // ignore them.  A potential result in registers can be ignored as
 490     // well.
 491 
 492 #ifdef ASSERT
 493     // make sure this code is only executed if there is a pending exception
 494     {
 495       Label L;
 496       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL);
 497       __ jcc(Assembler::notEqual, L);
 498       __ stop("StubRoutines::forward exception: no pending exception (1)");
 499       __ bind(L);
 500     }
 501 #endif
 502 
 503     // compute exception handler into rbx
 504     __ movptr(c_rarg0, Address(rsp, 0));
 505     BLOCK_COMMENT("call exception_handler_for_return_address");
 506     __ call_VM_leaf(CAST_FROM_FN_PTR(address,
 507                          SharedRuntime::exception_handler_for_return_address),
 508                     r15_thread, c_rarg0);
 509     __ mov(rbx, rax);
 510 
 511     // setup rax & rdx, remove return address & clear pending exception
 512     __ pop(rdx);
 513     __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
 514     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 515 
 516 #ifdef ASSERT
 517     // make sure exception is set
 518     {
 519       Label L;
 520       __ testptr(rax, rax);
 521       __ jcc(Assembler::notEqual, L);
 522       __ stop("StubRoutines::forward exception: no pending exception (2)");
 523       __ bind(L);
 524     }
 525 #endif
 526 
 527     // continue at exception handler (return address removed)
 528     // rax: exception
 529     // rbx: exception handler
 530     // rdx: throwing pc
 531     __ verify_oop(rax);
 532     __ jmp(rbx);
 533 
 534     return start;
 535   }
 536 
 537   // Support for jint atomic::xchg(jint exchange_value, volatile jint* dest)
 538   //
 539   // Arguments :
 540   //    c_rarg0: exchange_value
 541   //    c_rarg0: dest
 542   //
 543   // Result:
 544   //    *dest <- ex, return (orig *dest)
 545   address generate_atomic_xchg() {
 546     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 547     address start = __ pc();
 548 
 549     __ movl(rax, c_rarg0); // Copy to eax we need a return value anyhow
 550     __ xchgl(rax, Address(c_rarg1, 0)); // automatic LOCK
 551     __ ret(0);
 552 
 553     return start;
 554   }
 555 
 556   // Support for intptr_t atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest)
 557   //
 558   // Arguments :
 559   //    c_rarg0: exchange_value
 560   //    c_rarg1: dest
 561   //
 562   // Result:
 563   //    *dest <- ex, return (orig *dest)
 564   address generate_atomic_xchg_ptr() {
 565     StubCodeMark mark(this, "StubRoutines", "atomic_xchg_ptr");
 566     address start = __ pc();
 567 
 568     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 569     __ xchgptr(rax, Address(c_rarg1, 0)); // automatic LOCK
 570     __ ret(0);
 571 
 572     return start;
 573   }
 574 
 575   // Support for jint atomic::atomic_cmpxchg(jint exchange_value, volatile jint* dest,
 576   //                                         jint compare_value)
 577   //
 578   // Arguments :
 579   //    c_rarg0: exchange_value
 580   //    c_rarg1: dest
 581   //    c_rarg2: compare_value
 582   //
 583   // Result:
 584   //    if ( compare_value == *dest ) {
 585   //       *dest = exchange_value
 586   //       return compare_value;
 587   //    else
 588   //       return *dest;
 589   address generate_atomic_cmpxchg() {
 590     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");
 591     address start = __ pc();
 592 
 593     __ movl(rax, c_rarg2);
 594    if ( os::is_MP() ) __ lock();
 595     __ cmpxchgl(c_rarg0, Address(c_rarg1, 0));
 596     __ ret(0);
 597 
 598     return start;
 599   }
 600 
 601   // Support for jint atomic::atomic_cmpxchg_long(jlong exchange_value,
 602   //                                             volatile jlong* dest,
 603   //                                             jlong compare_value)
 604   // Arguments :
 605   //    c_rarg0: exchange_value
 606   //    c_rarg1: dest
 607   //    c_rarg2: compare_value
 608   //
 609   // Result:
 610   //    if ( compare_value == *dest ) {
 611   //       *dest = exchange_value
 612   //       return compare_value;
 613   //    else
 614   //       return *dest;
 615   address generate_atomic_cmpxchg_long() {
 616     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");
 617     address start = __ pc();
 618 
 619     __ movq(rax, c_rarg2);
 620    if ( os::is_MP() ) __ lock();
 621     __ cmpxchgq(c_rarg0, Address(c_rarg1, 0));
 622     __ ret(0);
 623 
 624     return start;
 625   }
 626 
 627   // Support for jint atomic::add(jint add_value, volatile jint* dest)
 628   //
 629   // Arguments :
 630   //    c_rarg0: add_value
 631   //    c_rarg1: dest
 632   //
 633   // Result:
 634   //    *dest += add_value
 635   //    return *dest;
 636   address generate_atomic_add() {
 637     StubCodeMark mark(this, "StubRoutines", "atomic_add");
 638     address start = __ pc();
 639 
 640     __ movl(rax, c_rarg0);
 641    if ( os::is_MP() ) __ lock();
 642     __ xaddl(Address(c_rarg1, 0), c_rarg0);
 643     __ addl(rax, c_rarg0);
 644     __ ret(0);
 645 
 646     return start;
 647   }
 648 
 649   // Support for intptr_t atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest)
 650   //
 651   // Arguments :
 652   //    c_rarg0: add_value
 653   //    c_rarg1: dest
 654   //
 655   // Result:
 656   //    *dest += add_value
 657   //    return *dest;
 658   address generate_atomic_add_ptr() {
 659     StubCodeMark mark(this, "StubRoutines", "atomic_add_ptr");
 660     address start = __ pc();
 661 
 662     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 663    if ( os::is_MP() ) __ lock();
 664     __ xaddptr(Address(c_rarg1, 0), c_rarg0);
 665     __ addptr(rax, c_rarg0);
 666     __ ret(0);
 667 
 668     return start;
 669   }
 670 
 671   // Support for intptr_t OrderAccess::fence()
 672   //
 673   // Arguments :
 674   //
 675   // Result:
 676   address generate_orderaccess_fence() {
 677     StubCodeMark mark(this, "StubRoutines", "orderaccess_fence");
 678     address start = __ pc();
 679     __ membar(Assembler::StoreLoad);
 680     __ ret(0);
 681 
 682     return start;
 683   }
 684 
 685   // Support for intptr_t get_previous_fp()
 686   //
 687   // This routine is used to find the previous frame pointer for the
 688   // caller (current_frame_guess). This is used as part of debugging
 689   // ps() is seemingly lost trying to find frames.
 690   // This code assumes that caller current_frame_guess) has a frame.
 691   address generate_get_previous_fp() {
 692     StubCodeMark mark(this, "StubRoutines", "get_previous_fp");
 693     const Address old_fp(rbp, 0);
 694     const Address older_fp(rax, 0);
 695     address start = __ pc();
 696 
 697     __ enter();
 698     __ movptr(rax, old_fp); // callers fp
 699     __ movptr(rax, older_fp); // the frame for ps()
 700     __ pop(rbp);
 701     __ ret(0);
 702 
 703     return start;
 704   }
 705 
 706   // Support for intptr_t get_previous_sp()
 707   //
 708   // This routine is used to find the previous stack pointer for the
 709   // caller.
 710   address generate_get_previous_sp() {
 711     StubCodeMark mark(this, "StubRoutines", "get_previous_sp");
 712     address start = __ pc();
 713 
 714     __ movptr(rax, rsp);
 715     __ addptr(rax, 8); // return address is at the top of the stack.
 716     __ ret(0);
 717 
 718     return start;
 719   }
 720 
 721   //----------------------------------------------------------------------------------------------------
 722   // Support for void verify_mxcsr()
 723   //
 724   // This routine is used with -Xcheck:jni to verify that native
 725   // JNI code does not return to Java code without restoring the
 726   // MXCSR register to our expected state.
 727 
 728   address generate_verify_mxcsr() {
 729     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 730     address start = __ pc();
 731 
 732     const Address mxcsr_save(rsp, 0);
 733 
 734     if (CheckJNICalls) {
 735       Label ok_ret;
 736       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 737       __ push(rax);
 738       __ subptr(rsp, wordSize);      // allocate a temp location
 739       __ stmxcsr(mxcsr_save);
 740       __ movl(rax, mxcsr_save);
 741       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 742       __ cmp32(rax, mxcsr_std);
 743       __ jcc(Assembler::equal, ok_ret);
 744 
 745       __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
 746 
 747       __ ldmxcsr(mxcsr_std);
 748 
 749       __ bind(ok_ret);
 750       __ addptr(rsp, wordSize);
 751       __ pop(rax);
 752     }
 753 
 754     __ ret(0);
 755 
 756     return start;
 757   }
 758 
 759   address generate_f2i_fixup() {
 760     StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
 761     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 762 
 763     address start = __ pc();
 764 
 765     Label L;
 766 
 767     __ push(rax);
 768     __ push(c_rarg3);
 769     __ push(c_rarg2);
 770     __ push(c_rarg1);
 771 
 772     __ movl(rax, 0x7f800000);
 773     __ xorl(c_rarg3, c_rarg3);
 774     __ movl(c_rarg2, inout);
 775     __ movl(c_rarg1, c_rarg2);
 776     __ andl(c_rarg1, 0x7fffffff);
 777     __ cmpl(rax, c_rarg1); // NaN? -> 0
 778     __ jcc(Assembler::negative, L);
 779     __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint
 780     __ movl(c_rarg3, 0x80000000);
 781     __ movl(rax, 0x7fffffff);
 782     __ cmovl(Assembler::positive, c_rarg3, rax);
 783 
 784     __ bind(L);
 785     __ movptr(inout, c_rarg3);
 786 
 787     __ pop(c_rarg1);
 788     __ pop(c_rarg2);
 789     __ pop(c_rarg3);
 790     __ pop(rax);
 791 
 792     __ ret(0);
 793 
 794     return start;
 795   }
 796 
 797   address generate_f2l_fixup() {
 798     StubCodeMark mark(this, "StubRoutines", "f2l_fixup");
 799     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 800     address start = __ pc();
 801 
 802     Label L;
 803 
 804     __ push(rax);
 805     __ push(c_rarg3);
 806     __ push(c_rarg2);
 807     __ push(c_rarg1);
 808 
 809     __ movl(rax, 0x7f800000);
 810     __ xorl(c_rarg3, c_rarg3);
 811     __ movl(c_rarg2, inout);
 812     __ movl(c_rarg1, c_rarg2);
 813     __ andl(c_rarg1, 0x7fffffff);
 814     __ cmpl(rax, c_rarg1); // NaN? -> 0
 815     __ jcc(Assembler::negative, L);
 816     __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong
 817     __ mov64(c_rarg3, 0x8000000000000000);
 818     __ mov64(rax, 0x7fffffffffffffff);
 819     __ cmov(Assembler::positive, c_rarg3, rax);
 820 
 821     __ bind(L);
 822     __ movptr(inout, c_rarg3);
 823 
 824     __ pop(c_rarg1);
 825     __ pop(c_rarg2);
 826     __ pop(c_rarg3);
 827     __ pop(rax);
 828 
 829     __ ret(0);
 830 
 831     return start;
 832   }
 833 
 834   address generate_d2i_fixup() {
 835     StubCodeMark mark(this, "StubRoutines", "d2i_fixup");
 836     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 837 
 838     address start = __ pc();
 839 
 840     Label L;
 841 
 842     __ push(rax);
 843     __ push(c_rarg3);
 844     __ push(c_rarg2);
 845     __ push(c_rarg1);
 846     __ push(c_rarg0);
 847 
 848     __ movl(rax, 0x7ff00000);
 849     __ movq(c_rarg2, inout);
 850     __ movl(c_rarg3, c_rarg2);
 851     __ mov(c_rarg1, c_rarg2);
 852     __ mov(c_rarg0, c_rarg2);
 853     __ negl(c_rarg3);
 854     __ shrptr(c_rarg1, 0x20);
 855     __ orl(c_rarg3, c_rarg2);
 856     __ andl(c_rarg1, 0x7fffffff);
 857     __ xorl(c_rarg2, c_rarg2);
 858     __ shrl(c_rarg3, 0x1f);
 859     __ orl(c_rarg1, c_rarg3);
 860     __ cmpl(rax, c_rarg1);
 861     __ jcc(Assembler::negative, L); // NaN -> 0
 862     __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint
 863     __ movl(c_rarg2, 0x80000000);
 864     __ movl(rax, 0x7fffffff);
 865     __ cmov(Assembler::positive, c_rarg2, rax);
 866 
 867     __ bind(L);
 868     __ movptr(inout, c_rarg2);
 869 
 870     __ pop(c_rarg0);
 871     __ pop(c_rarg1);
 872     __ pop(c_rarg2);
 873     __ pop(c_rarg3);
 874     __ pop(rax);
 875 
 876     __ ret(0);
 877 
 878     return start;
 879   }
 880 
 881   address generate_d2l_fixup() {
 882     StubCodeMark mark(this, "StubRoutines", "d2l_fixup");
 883     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 884 
 885     address start = __ pc();
 886 
 887     Label L;
 888 
 889     __ push(rax);
 890     __ push(c_rarg3);
 891     __ push(c_rarg2);
 892     __ push(c_rarg1);
 893     __ push(c_rarg0);
 894 
 895     __ movl(rax, 0x7ff00000);
 896     __ movq(c_rarg2, inout);
 897     __ movl(c_rarg3, c_rarg2);
 898     __ mov(c_rarg1, c_rarg2);
 899     __ mov(c_rarg0, c_rarg2);
 900     __ negl(c_rarg3);
 901     __ shrptr(c_rarg1, 0x20);
 902     __ orl(c_rarg3, c_rarg2);
 903     __ andl(c_rarg1, 0x7fffffff);
 904     __ xorl(c_rarg2, c_rarg2);
 905     __ shrl(c_rarg3, 0x1f);
 906     __ orl(c_rarg1, c_rarg3);
 907     __ cmpl(rax, c_rarg1);
 908     __ jcc(Assembler::negative, L); // NaN -> 0
 909     __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
 910     __ mov64(c_rarg2, 0x8000000000000000);
 911     __ mov64(rax, 0x7fffffffffffffff);
 912     __ cmovq(Assembler::positive, c_rarg2, rax);
 913 
 914     __ bind(L);
 915     __ movq(inout, c_rarg2);
 916 
 917     __ pop(c_rarg0);
 918     __ pop(c_rarg1);
 919     __ pop(c_rarg2);
 920     __ pop(c_rarg3);
 921     __ pop(rax);
 922 
 923     __ ret(0);
 924 
 925     return start;
 926   }
 927 
 928   address generate_fp_mask(const char *stub_name, int64_t mask) {
 929     __ align(CodeEntryAlignment);
 930     StubCodeMark mark(this, "StubRoutines", stub_name);
 931     address start = __ pc();
 932 
 933     __ emit_data64( mask, relocInfo::none );
 934     __ emit_data64( mask, relocInfo::none );
 935 
 936     return start;
 937   }
 938 
 939   // The following routine generates a subroutine to throw an
 940   // asynchronous UnknownError when an unsafe access gets a fault that
 941   // could not be reasonably prevented by the programmer.  (Example:
 942   // SIGBUS/OBJERR.)
 943   address generate_handler_for_unsafe_access() {
 944     StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
 945     address start = __ pc();
 946 
 947     __ push(0);                       // hole for return address-to-be
 948     __ pusha();                       // push registers
 949     Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
 950 
 951     // FIXME: this probably needs alignment logic
 952 
 953     __ subptr(rsp, frame::arg_reg_save_area_bytes);
 954     BLOCK_COMMENT("call handle_unsafe_access");
 955     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
 956     __ addptr(rsp, frame::arg_reg_save_area_bytes);
 957 
 958     __ movptr(next_pc, rax);          // stuff next address
 959     __ popa();
 960     __ ret(0);                        // jump to next address
 961 
 962     return start;
 963   }
 964 
 965   // Non-destructive plausibility checks for oops
 966   //
 967   // Arguments:
 968   //    all args on stack!
 969   //
 970   // Stack after saving c_rarg3:
 971   //    [tos + 0]: saved c_rarg3
 972   //    [tos + 1]: saved c_rarg2
 973   //    [tos + 2]: saved r12 (several TemplateTable methods use it)
 974   //    [tos + 3]: saved flags
 975   //    [tos + 4]: return address
 976   //  * [tos + 5]: error message (char*)
 977   //  * [tos + 6]: object to verify (oop)
 978   //  * [tos + 7]: saved rax - saved by caller and bashed
 979   //  * [tos + 8]: saved r10 (rscratch1) - saved by caller
 980   //  * = popped on exit
 981   address generate_verify_oop() {
 982     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 983     address start = __ pc();
 984 
 985     Label exit, error;
 986 
 987     __ pushf();
 988     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 989 
 990     __ push(r12);
 991 
 992     // save c_rarg2 and c_rarg3
 993     __ push(c_rarg2);
 994     __ push(c_rarg3);
 995 
 996     enum {
 997            // After previous pushes.
 998            oop_to_verify = 6 * wordSize,
 999            saved_rax     = 7 * wordSize,
1000            saved_r10     = 8 * wordSize,
1001 
1002            // Before the call to MacroAssembler::debug(), see below.
1003            return_addr   = 16 * wordSize,
1004            error_msg     = 17 * wordSize
1005     };
1006 
1007     // get object
1008     __ movptr(rax, Address(rsp, oop_to_verify));
1009 
1010     // make sure object is 'reasonable'
1011     __ testptr(rax, rax);
1012     __ jcc(Assembler::zero, exit); // if obj is NULL it is OK
1013     // Check if the oop is in the right area of memory
1014     __ movptr(c_rarg2, rax);
1015     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask());
1016     __ andptr(c_rarg2, c_rarg3);
1017     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits());
1018     __ cmpptr(c_rarg2, c_rarg3);
1019     __ jcc(Assembler::notZero, error);
1020 
1021     // set r12 to heapbase for load_klass()
1022     __ reinit_heapbase();
1023 
1024     // make sure klass is 'reasonable', which is not zero.
1025     __ load_klass(rax, rax);  // get klass
1026     __ testptr(rax, rax);
1027     __ jcc(Assembler::zero, error); // if klass is NULL it is broken
1028 
1029     // return if everything seems ok
1030     __ bind(exit);
1031     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
1032     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1033     __ pop(c_rarg3);                             // restore c_rarg3
1034     __ pop(c_rarg2);                             // restore c_rarg2
1035     __ pop(r12);                                 // restore r12
1036     __ popf();                                   // restore flags
1037     __ ret(4 * wordSize);                        // pop caller saved stuff
1038 
1039     // handle errors
1040     __ bind(error);
1041     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
1042     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1043     __ pop(c_rarg3);                             // get saved c_rarg3 back
1044     __ pop(c_rarg2);                             // get saved c_rarg2 back
1045     __ pop(r12);                                 // get saved r12 back
1046     __ popf();                                   // get saved flags off stack --
1047                                                  // will be ignored
1048 
1049     __ pusha();                                  // push registers
1050                                                  // (rip is already
1051                                                  // already pushed)
1052     // debug(char* msg, int64_t pc, int64_t regs[])
1053     // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
1054     // pushed all the registers, so now the stack looks like:
1055     //     [tos +  0] 16 saved registers
1056     //     [tos + 16] return address
1057     //   * [tos + 17] error message (char*)
1058     //   * [tos + 18] object to verify (oop)
1059     //   * [tos + 19] saved rax - saved by caller and bashed
1060     //   * [tos + 20] saved r10 (rscratch1) - saved by caller
1061     //   * = popped on exit
1062 
1063     __ movptr(c_rarg0, Address(rsp, error_msg));    // pass address of error message
1064     __ movptr(c_rarg1, Address(rsp, return_addr));  // pass return address
1065     __ movq(c_rarg2, rsp);                          // pass address of regs on stack
1066     __ mov(r12, rsp);                               // remember rsp
1067     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1068     __ andptr(rsp, -16);                            // align stack as required by ABI
1069     BLOCK_COMMENT("call MacroAssembler::debug");
1070     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
1071     __ mov(rsp, r12);                               // restore rsp
1072     __ popa();                                      // pop registers (includes r12)
1073     __ ret(4 * wordSize);                           // pop caller saved stuff
1074 
1075     return start;
1076   }
1077 
1078   //
1079   // Verify that a register contains clean 32-bits positive value
1080   // (high 32-bits are 0) so it could be used in 64-bits shifts.
1081   //
1082   //  Input:
1083   //    Rint  -  32-bits value
1084   //    Rtmp  -  scratch
1085   //
1086   void assert_clean_int(Register Rint, Register Rtmp) {
1087 #ifdef ASSERT
1088     Label L;
1089     assert_different_registers(Rtmp, Rint);
1090     __ movslq(Rtmp, Rint);
1091     __ cmpq(Rtmp, Rint);
1092     __ jcc(Assembler::equal, L);
1093     __ stop("high 32-bits of int value are not 0");
1094     __ bind(L);
1095 #endif
1096   }
1097 
1098   //  Generate overlap test for array copy stubs
1099   //
1100   //  Input:
1101   //     c_rarg0 - from
1102   //     c_rarg1 - to
1103   //     c_rarg2 - element count
1104   //
1105   //  Output:
1106   //     rax   - &from[element count - 1]
1107   //
1108   void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) {
1109     assert(no_overlap_target != NULL, "must be generated");
1110     array_overlap_test(no_overlap_target, NULL, sf);
1111   }
1112   void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) {
1113     array_overlap_test(NULL, &L_no_overlap, sf);
1114   }
1115   void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) {
1116     const Register from     = c_rarg0;
1117     const Register to       = c_rarg1;
1118     const Register count    = c_rarg2;
1119     const Register end_from = rax;
1120 
1121     __ cmpptr(to, from);
1122     __ lea(end_from, Address(from, count, sf, 0));
1123     if (NOLp == NULL) {
1124       ExternalAddress no_overlap(no_overlap_target);
1125       __ jump_cc(Assembler::belowEqual, no_overlap);
1126       __ cmpptr(to, end_from);
1127       __ jump_cc(Assembler::aboveEqual, no_overlap);
1128     } else {
1129       __ jcc(Assembler::belowEqual, (*NOLp));
1130       __ cmpptr(to, end_from);
1131       __ jcc(Assembler::aboveEqual, (*NOLp));
1132     }
1133   }
1134 
1135   // Shuffle first three arg regs on Windows into Linux/Solaris locations.
1136   //
1137   // Outputs:
1138   //    rdi - rcx
1139   //    rsi - rdx
1140   //    rdx - r8
1141   //    rcx - r9
1142   //
1143   // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter
1144   // are non-volatile.  r9 and r10 should not be used by the caller.
1145   //
1146   void setup_arg_regs(int nargs = 3) {
1147     const Register saved_rdi = r9;
1148     const Register saved_rsi = r10;
1149     assert(nargs == 3 || nargs == 4, "else fix");
1150 #ifdef _WIN64
1151     assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,
1152            "unexpected argument registers");
1153     if (nargs >= 4)
1154       __ mov(rax, r9);  // r9 is also saved_rdi
1155     __ movptr(saved_rdi, rdi);
1156     __ movptr(saved_rsi, rsi);
1157     __ mov(rdi, rcx); // c_rarg0
1158     __ mov(rsi, rdx); // c_rarg1
1159     __ mov(rdx, r8);  // c_rarg2
1160     if (nargs >= 4)
1161       __ mov(rcx, rax); // c_rarg3 (via rax)
1162 #else
1163     assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,
1164            "unexpected argument registers");
1165 #endif
1166   }
1167 
1168   void restore_arg_regs() {
1169     const Register saved_rdi = r9;
1170     const Register saved_rsi = r10;
1171 #ifdef _WIN64
1172     __ movptr(rdi, saved_rdi);
1173     __ movptr(rsi, saved_rsi);
1174 #endif
1175   }
1176 
1177   // Generate code for an array write pre barrier
1178   //
1179   //     addr    -  starting address
1180   //     count   -  element count
1181   //     tmp     - scratch register
1182   //
1183   //     Destroy no registers!
1184   //
1185   void  gen_write_ref_array_pre_barrier(Register src, Register addr, Register count, bool dest_uninitialized) {
1186     BarrierSet* bs = Universe::heap()->barrier_set();
1187     switch (bs->kind()) {
1188       case BarrierSet::G1SATBCT:
1189       case BarrierSet::G1SATBCTLogging:
1190         // With G1, don't generate the call if we statically know that the target in uninitialized
1191         if (!dest_uninitialized) {
1192            __ pusha();                      // push registers
1193            if (count == c_rarg0) {
1194              if (addr == c_rarg1) {
1195                // exactly backwards!!
1196                __ xchgptr(c_rarg1, c_rarg0);
1197              } else {
1198                __ movptr(c_rarg1, count);
1199                __ movptr(c_rarg0, addr);
1200              }
1201            } else {
1202              __ movptr(c_rarg0, addr);
1203              __ movptr(c_rarg1, count);
1204            }
1205            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
1206            __ popa();
1207         }
1208          break;
1209       case BarrierSet::CardTableModRef:
1210       case BarrierSet::CardTableExtension:
1211       case BarrierSet::ModRef:
1212         break;
1213 #if INCLUDE_ALL_GCS
1214       case BarrierSet::ShenandoahBarrierSet:
1215         ShenandoahBarrierSetAssembler::bsasm()->arraycopy_prologue(_masm, dest_uninitialized, src, addr, count);
1216         break;
1217 #endif
1218       default:
1219         ShouldNotReachHere();
1220 
1221     }
1222   }
1223 
1224   //
1225   // Generate code for an array write post barrier
1226   //
1227   //  Input:
1228   //     start    - register containing starting address of destination array
1229   //     count    - elements count
1230   //     scratch  - scratch register
1231   //
1232   //  The input registers are overwritten.
1233   //
1234   void  gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) {
1235     assert_different_registers(start, count, scratch);
1236     BarrierSet* bs = Universe::heap()->barrier_set();
1237     switch (bs->kind()) {
1238       case BarrierSet::G1SATBCT:
1239       case BarrierSet::G1SATBCTLogging:
1240         {
1241           __ pusha();             // push registers (overkill)
1242           if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
1243             assert_different_registers(c_rarg1, start);
1244             __ mov(c_rarg1, count);
1245             __ mov(c_rarg0, start);
1246           } else {
1247             assert_different_registers(c_rarg0, count);
1248             __ mov(c_rarg0, start);
1249             __ mov(c_rarg1, count);
1250           }
1251           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2);
1252           __ popa();
1253         }
1254         break;
1255       case BarrierSet::CardTableModRef:
1256       case BarrierSet::CardTableExtension:
1257         {
1258           CardTableModRefBS* ct = (CardTableModRefBS*)bs;
1259           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1260 
1261           Label L_loop;
1262           const Register end = count;
1263 
1264           __ leaq(end, Address(start, count, TIMES_OOP, 0));  // end == start+count*oop_size
1265           __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
1266           __ shrptr(start, CardTableModRefBS::card_shift);
1267           __ shrptr(end,   CardTableModRefBS::card_shift);
1268           __ subptr(end, start); // end --> cards count
1269 
1270           int64_t disp = (int64_t) ct->byte_map_base;
1271           __ mov64(scratch, disp);
1272           __ addptr(start, scratch);
1273         __ BIND(L_loop);
1274           __ movb(Address(start, count, Address::times_1), 0);
1275           __ decrement(count);
1276           __ jcc(Assembler::greaterEqual, L_loop);
1277         }
1278         break;
1279 #if INCLUDE_ALL_GCS
1280       case BarrierSet::ShenandoahBarrierSet:
1281         break;
1282 #endif
1283       default:
1284         ShouldNotReachHere();
1285 
1286     }
1287   }
1288 
1289 
1290   // Copy big chunks forward
1291   //
1292   // Inputs:
1293   //   end_from     - source arrays end address
1294   //   end_to       - destination array end address
1295   //   qword_count  - 64-bits element count, negative
1296   //   to           - scratch
1297   //   L_copy_bytes - entry label
1298   //   L_copy_8_bytes  - exit  label
1299   //
1300   void copy_bytes_forward(Register end_from, Register end_to,
1301                              Register qword_count, Register to,
1302                              Label& L_copy_bytes, Label& L_copy_8_bytes) {
1303     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1304     Label L_loop;
1305     __ align(OptoLoopAlignment);
1306     if (UseUnalignedLoadStores) {
1307       Label L_end;
1308       // Copy 64-bytes per iteration
1309       __ BIND(L_loop);
1310       if (UseAVX >= 2) {
1311         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1312         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1313         __ vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24));
1314         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1);
1315       } else {
1316         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1317         __ movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1318         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40));
1319         __ movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1);
1320         __ movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24));
1321         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2);
1322         __ movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8));
1323         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3);
1324       }
1325       __ BIND(L_copy_bytes);
1326       __ addptr(qword_count, 8);
1327       __ jcc(Assembler::lessEqual, L_loop);
1328       __ subptr(qword_count, 4);  // sub(8) and add(4)
1329       __ jccb(Assembler::greater, L_end);
1330       // Copy trailing 32 bytes
1331       if (UseAVX >= 2) {
1332         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1333         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1334       } else {
1335         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1336         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1337         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, - 8));
1338         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm1);
1339       }
1340       __ addptr(qword_count, 4);
1341       __ BIND(L_end);
1342       if (UseAVX >= 2) {
1343         // clean upper bits of YMM registers
1344         __ vpxor(xmm0, xmm0);
1345         __ vpxor(xmm1, xmm1);
1346       }
1347     } else {
1348       // Copy 32-bytes per iteration
1349       __ BIND(L_loop);
1350       __ movq(to, Address(end_from, qword_count, Address::times_8, -24));
1351       __ movq(Address(end_to, qword_count, Address::times_8, -24), to);
1352       __ movq(to, Address(end_from, qword_count, Address::times_8, -16));
1353       __ movq(Address(end_to, qword_count, Address::times_8, -16), to);
1354       __ movq(to, Address(end_from, qword_count, Address::times_8, - 8));
1355       __ movq(Address(end_to, qword_count, Address::times_8, - 8), to);
1356       __ movq(to, Address(end_from, qword_count, Address::times_8, - 0));
1357       __ movq(Address(end_to, qword_count, Address::times_8, - 0), to);
1358 
1359       __ BIND(L_copy_bytes);
1360       __ addptr(qword_count, 4);
1361       __ jcc(Assembler::lessEqual, L_loop);
1362     }
1363     __ subptr(qword_count, 4);
1364     __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
1365   }
1366 
1367   // Copy big chunks backward
1368   //
1369   // Inputs:
1370   //   from         - source arrays address
1371   //   dest         - destination array address
1372   //   qword_count  - 64-bits element count
1373   //   to           - scratch
1374   //   L_copy_bytes - entry label
1375   //   L_copy_8_bytes  - exit  label
1376   //
1377   void copy_bytes_backward(Register from, Register dest,
1378                               Register qword_count, Register to,
1379                               Label& L_copy_bytes, Label& L_copy_8_bytes) {
1380     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1381     Label L_loop;
1382     __ align(OptoLoopAlignment);
1383     if (UseUnalignedLoadStores) {
1384       Label L_end;
1385       // Copy 64-bytes per iteration
1386       __ BIND(L_loop);
1387       if (UseAVX >= 2) {
1388         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32));
1389         __ vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0);
1390         __ vmovdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1391         __ vmovdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1392       } else {
1393         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 48));
1394         __ movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0);
1395         __ movdqu(xmm1, Address(from, qword_count, Address::times_8, 32));
1396         __ movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1);
1397         __ movdqu(xmm2, Address(from, qword_count, Address::times_8, 16));
1398         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2);
1399         __ movdqu(xmm3, Address(from, qword_count, Address::times_8,  0));
1400         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm3);
1401       }
1402       __ BIND(L_copy_bytes);
1403       __ subptr(qword_count, 8);
1404       __ jcc(Assembler::greaterEqual, L_loop);
1405 
1406       __ addptr(qword_count, 4);  // add(8) and sub(4)
1407       __ jccb(Assembler::less, L_end);
1408       // Copy trailing 32 bytes
1409       if (UseAVX >= 2) {
1410         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 0));
1411         __ vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm0);
1412       } else {
1413         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 16));
1414         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm0);
1415         __ movdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1416         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1417       }
1418       __ subptr(qword_count, 4);
1419       __ BIND(L_end);
1420       if (UseAVX >= 2) {
1421         // clean upper bits of YMM registers
1422         __ vpxor(xmm0, xmm0);
1423         __ vpxor(xmm1, xmm1);
1424       }
1425     } else {
1426       // Copy 32-bytes per iteration
1427       __ BIND(L_loop);
1428       __ movq(to, Address(from, qword_count, Address::times_8, 24));
1429       __ movq(Address(dest, qword_count, Address::times_8, 24), to);
1430       __ movq(to, Address(from, qword_count, Address::times_8, 16));
1431       __ movq(Address(dest, qword_count, Address::times_8, 16), to);
1432       __ movq(to, Address(from, qword_count, Address::times_8,  8));
1433       __ movq(Address(dest, qword_count, Address::times_8,  8), to);
1434       __ movq(to, Address(from, qword_count, Address::times_8,  0));
1435       __ movq(Address(dest, qword_count, Address::times_8,  0), to);
1436 
1437       __ BIND(L_copy_bytes);
1438       __ subptr(qword_count, 4);
1439       __ jcc(Assembler::greaterEqual, L_loop);
1440     }
1441     __ addptr(qword_count, 4);
1442     __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
1443   }
1444 
1445 
1446   // Arguments:
1447   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1448   //             ignored
1449   //   name    - stub name string
1450   //
1451   // Inputs:
1452   //   c_rarg0   - source array address
1453   //   c_rarg1   - destination array address
1454   //   c_rarg2   - element count, treated as ssize_t, can be zero
1455   //
1456   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1457   // we let the hardware handle it.  The one to eight bytes within words,
1458   // dwords or qwords that span cache line boundaries will still be loaded
1459   // and stored atomically.
1460   //
1461   // Side Effects:
1462   //   disjoint_byte_copy_entry is set to the no-overlap entry point
1463   //   used by generate_conjoint_byte_copy().
1464   //
1465   address generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) {
1466     __ align(CodeEntryAlignment);
1467     StubCodeMark mark(this, "StubRoutines", name);
1468     address start = __ pc();
1469 
1470     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1471     Label L_copy_byte, L_exit;
1472     const Register from        = rdi;  // source array address
1473     const Register to          = rsi;  // destination array address
1474     const Register count       = rdx;  // elements count
1475     const Register byte_count  = rcx;
1476     const Register qword_count = count;
1477     const Register end_from    = from; // source array end address
1478     const Register end_to      = to;   // destination array end address
1479     // End pointers are inclusive, and if count is not zero they point
1480     // to the last unit copied:  end_to[0] := end_from[0]
1481 
1482     __ enter(); // required for proper stackwalking of RuntimeStub frame
1483     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1484 
1485     if (entry != NULL) {
1486       *entry = __ pc();
1487        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1488       BLOCK_COMMENT("Entry:");
1489     }
1490 
1491     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1492                       // r9 and r10 may be used to save non-volatile registers
1493 
1494     // 'from', 'to' and 'count' are now valid
1495     __ movptr(byte_count, count);
1496     __ shrptr(count, 3); // count => qword_count
1497 
1498     // Copy from low to high addresses.  Use 'to' as scratch.
1499     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1500     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1501     __ negptr(qword_count); // make the count negative
1502     __ jmp(L_copy_bytes);
1503 
1504     // Copy trailing qwords
1505   __ BIND(L_copy_8_bytes);
1506     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1507     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1508     __ increment(qword_count);
1509     __ jcc(Assembler::notZero, L_copy_8_bytes);
1510 
1511     // Check for and copy trailing dword
1512   __ BIND(L_copy_4_bytes);
1513     __ testl(byte_count, 4);
1514     __ jccb(Assembler::zero, L_copy_2_bytes);
1515     __ movl(rax, Address(end_from, 8));
1516     __ movl(Address(end_to, 8), rax);
1517 
1518     __ addptr(end_from, 4);
1519     __ addptr(end_to, 4);
1520 
1521     // Check for and copy trailing word
1522   __ BIND(L_copy_2_bytes);
1523     __ testl(byte_count, 2);
1524     __ jccb(Assembler::zero, L_copy_byte);
1525     __ movw(rax, Address(end_from, 8));
1526     __ movw(Address(end_to, 8), rax);
1527 
1528     __ addptr(end_from, 2);
1529     __ addptr(end_to, 2);
1530 
1531     // Check for and copy trailing byte
1532   __ BIND(L_copy_byte);
1533     __ testl(byte_count, 1);
1534     __ jccb(Assembler::zero, L_exit);
1535     __ movb(rax, Address(end_from, 8));
1536     __ movb(Address(end_to, 8), rax);
1537 
1538   __ BIND(L_exit);
1539     restore_arg_regs();
1540     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1541     __ xorptr(rax, rax); // return 0
1542     __ leave(); // required for proper stackwalking of RuntimeStub frame
1543     __ ret(0);
1544 
1545     // Copy in multi-bytes chunks
1546     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1547     __ jmp(L_copy_4_bytes);
1548 
1549     return start;
1550   }
1551 
1552   // Arguments:
1553   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1554   //             ignored
1555   //   name    - stub name string
1556   //
1557   // Inputs:
1558   //   c_rarg0   - source array address
1559   //   c_rarg1   - destination array address
1560   //   c_rarg2   - element count, treated as ssize_t, can be zero
1561   //
1562   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1563   // we let the hardware handle it.  The one to eight bytes within words,
1564   // dwords or qwords that span cache line boundaries will still be loaded
1565   // and stored atomically.
1566   //
1567   address generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
1568                                       address* entry, const char *name) {
1569     __ align(CodeEntryAlignment);
1570     StubCodeMark mark(this, "StubRoutines", name);
1571     address start = __ pc();
1572 
1573     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1574     const Register from        = rdi;  // source array address
1575     const Register to          = rsi;  // destination array address
1576     const Register count       = rdx;  // elements count
1577     const Register byte_count  = rcx;
1578     const Register qword_count = count;
1579 
1580     __ enter(); // required for proper stackwalking of RuntimeStub frame
1581     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1582 
1583     if (entry != NULL) {
1584       *entry = __ pc();
1585       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1586       BLOCK_COMMENT("Entry:");
1587     }
1588 
1589     array_overlap_test(nooverlap_target, Address::times_1);
1590     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1591                       // r9 and r10 may be used to save non-volatile registers
1592 
1593     // 'from', 'to' and 'count' are now valid
1594     __ movptr(byte_count, count);
1595     __ shrptr(count, 3);   // count => qword_count
1596 
1597     // Copy from high to low addresses.
1598 
1599     // Check for and copy trailing byte
1600     __ testl(byte_count, 1);
1601     __ jcc(Assembler::zero, L_copy_2_bytes);
1602     __ movb(rax, Address(from, byte_count, Address::times_1, -1));
1603     __ movb(Address(to, byte_count, Address::times_1, -1), rax);
1604     __ decrement(byte_count); // Adjust for possible trailing word
1605 
1606     // Check for and copy trailing word
1607   __ BIND(L_copy_2_bytes);
1608     __ testl(byte_count, 2);
1609     __ jcc(Assembler::zero, L_copy_4_bytes);
1610     __ movw(rax, Address(from, byte_count, Address::times_1, -2));
1611     __ movw(Address(to, byte_count, Address::times_1, -2), rax);
1612 
1613     // Check for and copy trailing dword
1614   __ BIND(L_copy_4_bytes);
1615     __ testl(byte_count, 4);
1616     __ jcc(Assembler::zero, L_copy_bytes);
1617     __ movl(rax, Address(from, qword_count, Address::times_8));
1618     __ movl(Address(to, qword_count, Address::times_8), rax);
1619     __ jmp(L_copy_bytes);
1620 
1621     // Copy trailing qwords
1622   __ BIND(L_copy_8_bytes);
1623     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1624     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1625     __ decrement(qword_count);
1626     __ jcc(Assembler::notZero, L_copy_8_bytes);
1627 
1628     restore_arg_regs();
1629     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1630     __ xorptr(rax, rax); // return 0
1631     __ leave(); // required for proper stackwalking of RuntimeStub frame
1632     __ ret(0);
1633 
1634     // Copy in multi-bytes chunks
1635     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1636 
1637     restore_arg_regs();
1638     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1639     __ xorptr(rax, rax); // return 0
1640     __ leave(); // required for proper stackwalking of RuntimeStub frame
1641     __ ret(0);
1642 
1643     return start;
1644   }
1645 
1646   // Arguments:
1647   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1648   //             ignored
1649   //   name    - stub name string
1650   //
1651   // Inputs:
1652   //   c_rarg0   - source array address
1653   //   c_rarg1   - destination array address
1654   //   c_rarg2   - element count, treated as ssize_t, can be zero
1655   //
1656   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1657   // let the hardware handle it.  The two or four words within dwords
1658   // or qwords that span cache line boundaries will still be loaded
1659   // and stored atomically.
1660   //
1661   // Side Effects:
1662   //   disjoint_short_copy_entry is set to the no-overlap entry point
1663   //   used by generate_conjoint_short_copy().
1664   //
1665   address generate_disjoint_short_copy(bool aligned, address *entry, const char *name) {
1666     __ align(CodeEntryAlignment);
1667     StubCodeMark mark(this, "StubRoutines", name);
1668     address start = __ pc();
1669 
1670     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
1671     const Register from        = rdi;  // source array address
1672     const Register to          = rsi;  // destination array address
1673     const Register count       = rdx;  // elements count
1674     const Register word_count  = rcx;
1675     const Register qword_count = count;
1676     const Register end_from    = from; // source array end address
1677     const Register end_to      = to;   // destination array end address
1678     // End pointers are inclusive, and if count is not zero they point
1679     // to the last unit copied:  end_to[0] := end_from[0]
1680 
1681     __ enter(); // required for proper stackwalking of RuntimeStub frame
1682     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1683 
1684     if (entry != NULL) {
1685       *entry = __ pc();
1686       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1687       BLOCK_COMMENT("Entry:");
1688     }
1689 
1690     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1691                       // r9 and r10 may be used to save non-volatile registers
1692 
1693     // 'from', 'to' and 'count' are now valid
1694     __ movptr(word_count, count);
1695     __ shrptr(count, 2); // count => qword_count
1696 
1697     // Copy from low to high addresses.  Use 'to' as scratch.
1698     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1699     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1700     __ negptr(qword_count);
1701     __ jmp(L_copy_bytes);
1702 
1703     // Copy trailing qwords
1704   __ BIND(L_copy_8_bytes);
1705     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1706     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1707     __ increment(qword_count);
1708     __ jcc(Assembler::notZero, L_copy_8_bytes);
1709 
1710     // Original 'dest' is trashed, so we can't use it as a
1711     // base register for a possible trailing word copy
1712 
1713     // Check for and copy trailing dword
1714   __ BIND(L_copy_4_bytes);
1715     __ testl(word_count, 2);
1716     __ jccb(Assembler::zero, L_copy_2_bytes);
1717     __ movl(rax, Address(end_from, 8));
1718     __ movl(Address(end_to, 8), rax);
1719 
1720     __ addptr(end_from, 4);
1721     __ addptr(end_to, 4);
1722 
1723     // Check for and copy trailing word
1724   __ BIND(L_copy_2_bytes);
1725     __ testl(word_count, 1);
1726     __ jccb(Assembler::zero, L_exit);
1727     __ movw(rax, Address(end_from, 8));
1728     __ movw(Address(end_to, 8), rax);
1729 
1730   __ BIND(L_exit);
1731     restore_arg_regs();
1732     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1733     __ xorptr(rax, rax); // return 0
1734     __ leave(); // required for proper stackwalking of RuntimeStub frame
1735     __ ret(0);
1736 
1737     // Copy in multi-bytes chunks
1738     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1739     __ jmp(L_copy_4_bytes);
1740 
1741     return start;
1742   }
1743 
1744   address generate_fill(BasicType t, bool aligned, const char *name) {
1745     __ align(CodeEntryAlignment);
1746     StubCodeMark mark(this, "StubRoutines", name);
1747     address start = __ pc();
1748 
1749     BLOCK_COMMENT("Entry:");
1750 
1751     const Register to       = c_rarg0;  // source array address
1752     const Register value    = c_rarg1;  // value
1753     const Register count    = c_rarg2;  // elements count
1754 
1755     __ enter(); // required for proper stackwalking of RuntimeStub frame
1756 
1757     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1758 
1759     __ leave(); // required for proper stackwalking of RuntimeStub frame
1760     __ ret(0);
1761     return start;
1762   }
1763 
1764   // Arguments:
1765   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1766   //             ignored
1767   //   name    - stub name string
1768   //
1769   // Inputs:
1770   //   c_rarg0   - source array address
1771   //   c_rarg1   - destination array address
1772   //   c_rarg2   - element count, treated as ssize_t, can be zero
1773   //
1774   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1775   // let the hardware handle it.  The two or four words within dwords
1776   // or qwords that span cache line boundaries will still be loaded
1777   // and stored atomically.
1778   //
1779   address generate_conjoint_short_copy(bool aligned, address nooverlap_target,
1780                                        address *entry, const char *name) {
1781     __ align(CodeEntryAlignment);
1782     StubCodeMark mark(this, "StubRoutines", name);
1783     address start = __ pc();
1784 
1785     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes;
1786     const Register from        = rdi;  // source array address
1787     const Register to          = rsi;  // destination array address
1788     const Register count       = rdx;  // elements count
1789     const Register word_count  = rcx;
1790     const Register qword_count = count;
1791 
1792     __ enter(); // required for proper stackwalking of RuntimeStub frame
1793     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1794 
1795     if (entry != NULL) {
1796       *entry = __ pc();
1797       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1798       BLOCK_COMMENT("Entry:");
1799     }
1800 
1801     array_overlap_test(nooverlap_target, Address::times_2);
1802     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1803                       // r9 and r10 may be used to save non-volatile registers
1804 
1805     // 'from', 'to' and 'count' are now valid
1806     __ movptr(word_count, count);
1807     __ shrptr(count, 2); // count => qword_count
1808 
1809     // Copy from high to low addresses.  Use 'to' as scratch.
1810 
1811     // Check for and copy trailing word
1812     __ testl(word_count, 1);
1813     __ jccb(Assembler::zero, L_copy_4_bytes);
1814     __ movw(rax, Address(from, word_count, Address::times_2, -2));
1815     __ movw(Address(to, word_count, Address::times_2, -2), rax);
1816 
1817     // Check for and copy trailing dword
1818   __ BIND(L_copy_4_bytes);
1819     __ testl(word_count, 2);
1820     __ jcc(Assembler::zero, L_copy_bytes);
1821     __ movl(rax, Address(from, qword_count, Address::times_8));
1822     __ movl(Address(to, qword_count, Address::times_8), rax);
1823     __ jmp(L_copy_bytes);
1824 
1825     // Copy trailing qwords
1826   __ BIND(L_copy_8_bytes);
1827     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1828     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1829     __ decrement(qword_count);
1830     __ jcc(Assembler::notZero, L_copy_8_bytes);
1831 
1832     restore_arg_regs();
1833     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1834     __ xorptr(rax, rax); // return 0
1835     __ leave(); // required for proper stackwalking of RuntimeStub frame
1836     __ ret(0);
1837 
1838     // Copy in multi-bytes chunks
1839     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1840 
1841     restore_arg_regs();
1842     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1843     __ xorptr(rax, rax); // return 0
1844     __ leave(); // required for proper stackwalking of RuntimeStub frame
1845     __ ret(0);
1846 
1847     return start;
1848   }
1849 
1850   // Arguments:
1851   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1852   //             ignored
1853   //   is_oop  - true => oop array, so generate store check code
1854   //   name    - stub name string
1855   //
1856   // Inputs:
1857   //   c_rarg0   - source array address
1858   //   c_rarg1   - destination array address
1859   //   c_rarg2   - element count, treated as ssize_t, can be zero
1860   //
1861   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1862   // the hardware handle it.  The two dwords within qwords that span
1863   // cache line boundaries will still be loaded and stored atomicly.
1864   //
1865   // Side Effects:
1866   //   disjoint_int_copy_entry is set to the no-overlap entry point
1867   //   used by generate_conjoint_int_oop_copy().
1868   //
1869   address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry,
1870                                          const char *name, bool dest_uninitialized = false) {
1871     __ align(CodeEntryAlignment);
1872     StubCodeMark mark(this, "StubRoutines", name);
1873     address start = __ pc();
1874 
1875     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
1876     const Register from        = rdi;  // source array address
1877     const Register to          = rsi;  // destination array address
1878     const Register count       = rdx;  // elements count
1879     const Register dword_count = rcx;
1880     const Register qword_count = count;
1881     const Register end_from    = from; // source array end address
1882     const Register end_to      = to;   // destination array end address
1883     const Register saved_to    = r11;  // saved destination array address
1884     // End pointers are inclusive, and if count is not zero they point
1885     // to the last unit copied:  end_to[0] := end_from[0]
1886 
1887     __ enter(); // required for proper stackwalking of RuntimeStub frame
1888     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1889 
1890     if (entry != NULL) {
1891       *entry = __ pc();
1892       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1893       BLOCK_COMMENT("Entry:");
1894     }
1895 
1896     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1897                       // r9 and r10 may be used to save non-volatile registers
1898     if (is_oop) {
1899       __ movq(saved_to, to);
1900       gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized);
1901     }
1902 
1903     // 'from', 'to' and 'count' are now valid
1904     __ movptr(dword_count, count);
1905     __ shrptr(count, 1); // count => qword_count
1906 
1907     // Copy from low to high addresses.  Use 'to' as scratch.
1908     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1909     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1910     __ negptr(qword_count);
1911     __ jmp(L_copy_bytes);
1912 
1913     // Copy trailing qwords
1914   __ BIND(L_copy_8_bytes);
1915     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1916     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1917     __ increment(qword_count);
1918     __ jcc(Assembler::notZero, L_copy_8_bytes);
1919 
1920     // Check for and copy trailing dword
1921   __ BIND(L_copy_4_bytes);
1922     __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
1923     __ jccb(Assembler::zero, L_exit);
1924     __ movl(rax, Address(end_from, 8));
1925     __ movl(Address(end_to, 8), rax);
1926 
1927   __ BIND(L_exit);
1928     if (is_oop) {
1929       gen_write_ref_array_post_barrier(saved_to, dword_count, rax);
1930     }
1931     restore_arg_regs();
1932     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
1933     __ xorptr(rax, rax); // return 0
1934     __ leave(); // required for proper stackwalking of RuntimeStub frame
1935     __ ret(0);
1936 
1937     // Copy in multi-bytes chunks
1938     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1939     __ jmp(L_copy_4_bytes);
1940 
1941     return start;
1942   }
1943 
1944   // Arguments:
1945   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1946   //             ignored
1947   //   is_oop  - true => oop array, so generate store check code
1948   //   name    - stub name string
1949   //
1950   // Inputs:
1951   //   c_rarg0   - source array address
1952   //   c_rarg1   - destination array address
1953   //   c_rarg2   - element count, treated as ssize_t, can be zero
1954   //
1955   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1956   // the hardware handle it.  The two dwords within qwords that span
1957   // cache line boundaries will still be loaded and stored atomicly.
1958   //
1959   address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target,
1960                                          address *entry, const char *name,
1961                                          bool dest_uninitialized = false) {
1962     __ align(CodeEntryAlignment);
1963     StubCodeMark mark(this, "StubRoutines", name);
1964     address start = __ pc();
1965 
1966     Label L_copy_bytes, L_copy_8_bytes, L_copy_2_bytes, L_exit;
1967     const Register from        = rdi;  // source array address
1968     const Register to          = rsi;  // destination array address
1969     const Register count       = rdx;  // elements count
1970     const Register dword_count = rcx;
1971     const Register qword_count = count;
1972 
1973     __ enter(); // required for proper stackwalking of RuntimeStub frame
1974     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1975 
1976     if (entry != NULL) {
1977       *entry = __ pc();
1978        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1979       BLOCK_COMMENT("Entry:");
1980     }
1981 
1982     array_overlap_test(nooverlap_target, Address::times_4);
1983     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1984                       // r9 and r10 may be used to save non-volatile registers
1985 
1986     if (is_oop) {
1987       // no registers are destroyed by this call
1988       gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized);
1989     }
1990 
1991     assert_clean_int(count, rax); // Make sure 'count' is clean int.
1992     // 'from', 'to' and 'count' are now valid
1993     __ movptr(dword_count, count);
1994     __ shrptr(count, 1); // count => qword_count
1995 
1996     // Copy from high to low addresses.  Use 'to' as scratch.
1997 
1998     // Check for and copy trailing dword
1999     __ testl(dword_count, 1);
2000     __ jcc(Assembler::zero, L_copy_bytes);
2001     __ movl(rax, Address(from, dword_count, Address::times_4, -4));
2002     __ movl(Address(to, dword_count, Address::times_4, -4), rax);
2003     __ jmp(L_copy_bytes);
2004 
2005     // Copy trailing qwords
2006   __ BIND(L_copy_8_bytes);
2007     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2008     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2009     __ decrement(qword_count);
2010     __ jcc(Assembler::notZero, L_copy_8_bytes);
2011 
2012     if (is_oop) {
2013       __ jmp(L_exit);
2014     }
2015     restore_arg_regs();
2016     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
2017     __ xorptr(rax, rax); // return 0
2018     __ leave(); // required for proper stackwalking of RuntimeStub frame
2019     __ ret(0);
2020 
2021     // Copy in multi-bytes chunks
2022     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2023 
2024   __ BIND(L_exit);
2025     if (is_oop) {
2026       gen_write_ref_array_post_barrier(to, dword_count, rax);
2027     }
2028     restore_arg_regs();
2029     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
2030     __ xorptr(rax, rax); // return 0
2031     __ leave(); // required for proper stackwalking of RuntimeStub frame
2032     __ ret(0);
2033 
2034     return start;
2035   }
2036 
2037   // Arguments:
2038   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2039   //             ignored
2040   //   is_oop  - true => oop array, so generate store check code
2041   //   name    - stub name string
2042   //
2043   // Inputs:
2044   //   c_rarg0   - source array address
2045   //   c_rarg1   - destination array address
2046   //   c_rarg2   - element count, treated as ssize_t, can be zero
2047   //
2048  // Side Effects:
2049   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
2050   //   no-overlap entry point used by generate_conjoint_long_oop_copy().
2051   //
2052   address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry,
2053                                           const char *name, bool dest_uninitialized = false) {
2054     __ align(CodeEntryAlignment);
2055     StubCodeMark mark(this, "StubRoutines", name);
2056     address start = __ pc();
2057 
2058     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2059     const Register from        = rdi;  // source array address
2060     const Register to          = rsi;  // destination array address
2061     const Register qword_count = rdx;  // elements count
2062     const Register end_from    = from; // source array end address
2063     const Register end_to      = rcx;  // destination array end address
2064     const Register saved_to    = to;
2065     const Register saved_count = r11;
2066     // End pointers are inclusive, and if count is not zero they point
2067     // to the last unit copied:  end_to[0] := end_from[0]
2068 
2069     __ enter(); // required for proper stackwalking of RuntimeStub frame
2070     // Save no-overlap entry point for generate_conjoint_long_oop_copy()
2071     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2072 
2073     if (entry != NULL) {
2074       *entry = __ pc();
2075       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2076       BLOCK_COMMENT("Entry:");
2077     }
2078 
2079     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2080                       // r9 and r10 may be used to save non-volatile registers
2081     // 'from', 'to' and 'qword_count' are now valid
2082     if (is_oop) {
2083       // Save to and count for store barrier
2084       __ movptr(saved_count, qword_count);
2085       // no registers are destroyed by this call
2086       gen_write_ref_array_pre_barrier(from, to, qword_count, dest_uninitialized);
2087     }
2088 
2089     // Copy from low to high addresses.  Use 'to' as scratch.
2090     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
2091     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
2092     __ negptr(qword_count);
2093     __ jmp(L_copy_bytes);
2094 
2095     // Copy trailing qwords
2096   __ BIND(L_copy_8_bytes);
2097     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2098     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2099     __ increment(qword_count);
2100     __ jcc(Assembler::notZero, L_copy_8_bytes);
2101 
2102     if (is_oop) {
2103       __ jmp(L_exit);
2104     } else {
2105       restore_arg_regs();
2106       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2107       __ xorptr(rax, rax); // return 0
2108       __ leave(); // required for proper stackwalking of RuntimeStub frame
2109       __ ret(0);
2110     }
2111 
2112     // Copy in multi-bytes chunks
2113     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2114 
2115     if (is_oop) {
2116     __ BIND(L_exit);
2117       gen_write_ref_array_post_barrier(saved_to, saved_count, rax);
2118     }
2119     restore_arg_regs();
2120     if (is_oop) {
2121       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2122     } else {
2123       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2124     }
2125     __ xorptr(rax, rax); // return 0
2126     __ leave(); // required for proper stackwalking of RuntimeStub frame
2127     __ ret(0);
2128 
2129     return start;
2130   }
2131 
2132   // Arguments:
2133   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2134   //             ignored
2135   //   is_oop  - true => oop array, so generate store check code
2136   //   name    - stub name string
2137   //
2138   // Inputs:
2139   //   c_rarg0   - source array address
2140   //   c_rarg1   - destination array address
2141   //   c_rarg2   - element count, treated as ssize_t, can be zero
2142   //
2143   address generate_conjoint_long_oop_copy(bool aligned, bool is_oop,
2144                                           address nooverlap_target, address *entry,
2145                                           const char *name, bool dest_uninitialized = false) {
2146     __ align(CodeEntryAlignment);
2147     StubCodeMark mark(this, "StubRoutines", name);
2148     address start = __ pc();
2149 
2150     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2151     const Register from        = rdi;  // source array address
2152     const Register to          = rsi;  // destination array address
2153     const Register qword_count = rdx;  // elements count
2154     const Register saved_count = rcx;
2155 
2156     __ enter(); // required for proper stackwalking of RuntimeStub frame
2157     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2158 
2159     if (entry != NULL) {
2160       *entry = __ pc();
2161       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2162       BLOCK_COMMENT("Entry:");
2163     }
2164 
2165     array_overlap_test(nooverlap_target, Address::times_8);
2166     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2167                       // r9 and r10 may be used to save non-volatile registers
2168     // 'from', 'to' and 'qword_count' are now valid
2169     if (is_oop) {
2170       // Save to and count for store barrier
2171       __ movptr(saved_count, qword_count);
2172       // No registers are destroyed by this call
2173       gen_write_ref_array_pre_barrier(from, to, saved_count, dest_uninitialized);
2174     }
2175 
2176     __ jmp(L_copy_bytes);
2177 
2178     // Copy trailing qwords
2179   __ BIND(L_copy_8_bytes);
2180     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2181     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2182     __ decrement(qword_count);
2183     __ jcc(Assembler::notZero, L_copy_8_bytes);
2184 
2185     if (is_oop) {
2186       __ jmp(L_exit);
2187     } else {
2188       restore_arg_regs();
2189       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2190       __ xorptr(rax, rax); // return 0
2191       __ leave(); // required for proper stackwalking of RuntimeStub frame
2192       __ ret(0);
2193     }
2194 
2195     // Copy in multi-bytes chunks
2196     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2197 
2198     if (is_oop) {
2199     __ BIND(L_exit);
2200       gen_write_ref_array_post_barrier(to, saved_count, rax);
2201     }
2202     restore_arg_regs();
2203     if (is_oop) {
2204       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2205     } else {
2206       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2207     }
2208     __ xorptr(rax, rax); // return 0
2209     __ leave(); // required for proper stackwalking of RuntimeStub frame
2210     __ ret(0);
2211 
2212     return start;
2213   }
2214 
2215 
2216   // Helper for generating a dynamic type check.
2217   // Smashes no registers.
2218   void generate_type_check(Register sub_klass,
2219                            Register super_check_offset,
2220                            Register super_klass,
2221                            Label& L_success) {
2222     assert_different_registers(sub_klass, super_check_offset, super_klass);
2223 
2224     BLOCK_COMMENT("type_check:");
2225 
2226     Label L_miss;
2227 
2228     __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg,        &L_success, &L_miss, NULL,
2229                                      super_check_offset);
2230     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL);
2231 
2232     // Fall through on failure!
2233     __ BIND(L_miss);
2234   }
2235 
2236   //
2237   //  Generate checkcasting array copy stub
2238   //
2239   //  Input:
2240   //    c_rarg0   - source array address
2241   //    c_rarg1   - destination array address
2242   //    c_rarg2   - element count, treated as ssize_t, can be zero
2243   //    c_rarg3   - size_t ckoff (super_check_offset)
2244   // not Win64
2245   //    c_rarg4   - oop ckval (super_klass)
2246   // Win64
2247   //    rsp+40    - oop ckval (super_klass)
2248   //
2249   //  Output:
2250   //    rax ==  0  -  success
2251   //    rax == -1^K - failure, where K is partial transfer count
2252   //
2253   address generate_checkcast_copy(const char *name, address *entry,
2254                                   bool dest_uninitialized = false) {
2255 
2256     Label L_load_element, L_store_element, L_do_card_marks, L_done;
2257 
2258     // Input registers (after setup_arg_regs)
2259     const Register from        = rdi;   // source array address
2260     const Register to          = rsi;   // destination array address
2261     const Register length      = rdx;   // elements count
2262     const Register ckoff       = rcx;   // super_check_offset
2263     const Register ckval       = r8;    // super_klass
2264 
2265     // Registers used as temps (r13, r14 are save-on-entry)
2266     const Register end_from    = from;  // source array end address
2267     const Register end_to      = r13;   // destination array end address
2268     const Register count       = rdx;   // -(count_remaining)
2269     const Register r14_length  = r14;   // saved copy of length
2270     // End pointers are inclusive, and if length is not zero they point
2271     // to the last unit copied:  end_to[0] := end_from[0]
2272 
2273     const Register rax_oop    = rax;    // actual oop copied
2274     const Register r11_klass  = r11;    // oop._klass
2275 
2276     //---------------------------------------------------------------
2277     // Assembler stub will be used for this call to arraycopy
2278     // if the two arrays are subtypes of Object[] but the
2279     // destination array type is not equal to or a supertype
2280     // of the source type.  Each element must be separately
2281     // checked.
2282 
2283     __ align(CodeEntryAlignment);
2284     StubCodeMark mark(this, "StubRoutines", name);
2285     address start = __ pc();
2286 
2287     __ enter(); // required for proper stackwalking of RuntimeStub frame
2288 
2289 #ifdef ASSERT
2290     // caller guarantees that the arrays really are different
2291     // otherwise, we would have to make conjoint checks
2292     { Label L;
2293       array_overlap_test(L, TIMES_OOP);
2294       __ stop("checkcast_copy within a single array");
2295       __ bind(L);
2296     }
2297 #endif //ASSERT
2298 
2299     setup_arg_regs(4); // from => rdi, to => rsi, length => rdx
2300                        // ckoff => rcx, ckval => r8
2301                        // r9 and r10 may be used to save non-volatile registers
2302 #ifdef _WIN64
2303     // last argument (#4) is on stack on Win64
2304     __ movptr(ckval, Address(rsp, 6 * wordSize));
2305 #endif
2306 
2307     // Caller of this entry point must set up the argument registers.
2308     if (entry != NULL) {
2309       *entry = __ pc();
2310       BLOCK_COMMENT("Entry:");
2311     }
2312 
2313     // allocate spill slots for r13, r14
2314     enum {
2315       saved_r13_offset,
2316       saved_r14_offset,
2317       saved_rbp_offset
2318     };
2319     __ subptr(rsp, saved_rbp_offset * wordSize);
2320     __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
2321     __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
2322 
2323     // check that int operands are properly extended to size_t
2324     assert_clean_int(length, rax);
2325     assert_clean_int(ckoff, rax);
2326 
2327 #ifdef ASSERT
2328     BLOCK_COMMENT("assert consistent ckoff/ckval");
2329     // The ckoff and ckval must be mutually consistent,
2330     // even though caller generates both.
2331     { Label L;
2332       int sco_offset = in_bytes(Klass::super_check_offset_offset());
2333       __ cmpl(ckoff, Address(ckval, sco_offset));
2334       __ jcc(Assembler::equal, L);
2335       __ stop("super_check_offset inconsistent");
2336       __ bind(L);
2337     }
2338 #endif //ASSERT
2339 
2340     // Loop-invariant addresses.  They are exclusive end pointers.
2341     Address end_from_addr(from, length, TIMES_OOP, 0);
2342     Address   end_to_addr(to,   length, TIMES_OOP, 0);
2343     // Loop-variant addresses.  They assume post-incremented count < 0.
2344     Address from_element_addr(end_from, count, TIMES_OOP, 0);
2345     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
2346 
2347     gen_write_ref_array_pre_barrier(from, to, count, dest_uninitialized);
2348 
2349     // Copy from low to high addresses, indexed from the end of each array.
2350     __ lea(end_from, end_from_addr);
2351     __ lea(end_to,   end_to_addr);
2352     __ movptr(r14_length, length);        // save a copy of the length
2353     assert(length == count, "");          // else fix next line:
2354     __ negptr(count);                     // negate and test the length
2355     __ jcc(Assembler::notZero, L_load_element);
2356 
2357     // Empty array:  Nothing to do.
2358     __ xorptr(rax, rax);                  // return 0 on (trivial) success
2359     __ jmp(L_done);
2360 
2361     // ======== begin loop ========
2362     // (Loop is rotated; its entry is L_load_element.)
2363     // Loop control:
2364     //   for (count = -count; count != 0; count++)
2365     // Base pointers src, dst are biased by 8*(count-1),to last element.
2366     __ align(OptoLoopAlignment);
2367 
2368     __ BIND(L_store_element);
2369     __ store_heap_oop(to_element_addr, rax_oop);  // store the oop
2370     __ increment(count);               // increment the count toward zero
2371     __ jcc(Assembler::zero, L_do_card_marks);
2372 
2373     // ======== loop entry is here ========
2374     __ BIND(L_load_element);
2375     __ load_heap_oop(rax_oop, from_element_addr); // load the oop
2376     __ testptr(rax_oop, rax_oop);
2377     __ jcc(Assembler::zero, L_store_element);
2378 
2379     __ load_klass(r11_klass, rax_oop);// query the object klass
2380     generate_type_check(r11_klass, ckoff, ckval, L_store_element);
2381     // ======== end loop ========
2382 
2383     // It was a real error; we must depend on the caller to finish the job.
2384     // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
2385     // Emit GC store barriers for the oops we have copied (r14 + rdx),
2386     // and report their number to the caller.
2387     assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1);
2388     Label L_post_barrier;
2389     __ addptr(r14_length, count);     // K = (original - remaining) oops
2390     __ movptr(rax, r14_length);       // save the value
2391     __ notptr(rax);                   // report (-1^K) to caller (does not affect flags)
2392     __ jccb(Assembler::notZero, L_post_barrier);
2393     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
2394 
2395     // Come here on success only.
2396     __ BIND(L_do_card_marks);
2397     __ xorptr(rax, rax);              // return 0 on success
2398 
2399     __ BIND(L_post_barrier);
2400     gen_write_ref_array_post_barrier(to, r14_length, rscratch1);
2401 
2402     // Common exit point (success or failure).
2403     __ BIND(L_done);
2404     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
2405     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
2406     restore_arg_regs();
2407     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); // Update counter after rscratch1 is free
2408     __ leave(); // required for proper stackwalking of RuntimeStub frame
2409     __ ret(0);
2410 
2411     return start;
2412   }
2413 
2414   //
2415   //  Generate 'unsafe' array copy stub
2416   //  Though just as safe as the other stubs, it takes an unscaled
2417   //  size_t argument instead of an element count.
2418   //
2419   //  Input:
2420   //    c_rarg0   - source array address
2421   //    c_rarg1   - destination array address
2422   //    c_rarg2   - byte count, treated as ssize_t, can be zero
2423   //
2424   // Examines the alignment of the operands and dispatches
2425   // to a long, int, short, or byte copy loop.
2426   //
2427   address generate_unsafe_copy(const char *name,
2428                                address byte_copy_entry, address short_copy_entry,
2429                                address int_copy_entry, address long_copy_entry) {
2430 
2431     Label L_long_aligned, L_int_aligned, L_short_aligned;
2432 
2433     // Input registers (before setup_arg_regs)
2434     const Register from        = c_rarg0;  // source array address
2435     const Register to          = c_rarg1;  // destination array address
2436     const Register size        = c_rarg2;  // byte count (size_t)
2437 
2438     // Register used as a temp
2439     const Register bits        = rax;      // test copy of low bits
2440 
2441     __ align(CodeEntryAlignment);
2442     StubCodeMark mark(this, "StubRoutines", name);
2443     address start = __ pc();
2444 
2445     __ enter(); // required for proper stackwalking of RuntimeStub frame
2446 
2447     // bump this on entry, not on exit:
2448     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2449 
2450     __ mov(bits, from);
2451     __ orptr(bits, to);
2452     __ orptr(bits, size);
2453 
2454     __ testb(bits, BytesPerLong-1);
2455     __ jccb(Assembler::zero, L_long_aligned);
2456 
2457     __ testb(bits, BytesPerInt-1);
2458     __ jccb(Assembler::zero, L_int_aligned);
2459 
2460     __ testb(bits, BytesPerShort-1);
2461     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2462 
2463     __ BIND(L_short_aligned);
2464     __ shrptr(size, LogBytesPerShort); // size => short_count
2465     __ jump(RuntimeAddress(short_copy_entry));
2466 
2467     __ BIND(L_int_aligned);
2468     __ shrptr(size, LogBytesPerInt); // size => int_count
2469     __ jump(RuntimeAddress(int_copy_entry));
2470 
2471     __ BIND(L_long_aligned);
2472     __ shrptr(size, LogBytesPerLong); // size => qword_count
2473     __ jump(RuntimeAddress(long_copy_entry));
2474 
2475     return start;
2476   }
2477 
2478   // Perform range checks on the proposed arraycopy.
2479   // Kills temp, but nothing else.
2480   // Also, clean the sign bits of src_pos and dst_pos.
2481   void arraycopy_range_checks(Register src,     // source array oop (c_rarg0)
2482                               Register src_pos, // source position (c_rarg1)
2483                               Register dst,     // destination array oo (c_rarg2)
2484                               Register dst_pos, // destination position (c_rarg3)
2485                               Register length,
2486                               Register temp,
2487                               Label& L_failed) {
2488     BLOCK_COMMENT("arraycopy_range_checks:");
2489 
2490     //  if (src_pos + length > arrayOop(src)->length())  FAIL;
2491     __ movl(temp, length);
2492     __ addl(temp, src_pos);             // src_pos + length
2493     __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
2494     __ jcc(Assembler::above, L_failed);
2495 
2496     //  if (dst_pos + length > arrayOop(dst)->length())  FAIL;
2497     __ movl(temp, length);
2498     __ addl(temp, dst_pos);             // dst_pos + length
2499     __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2500     __ jcc(Assembler::above, L_failed);
2501 
2502     // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
2503     // Move with sign extension can be used since they are positive.
2504     __ movslq(src_pos, src_pos);
2505     __ movslq(dst_pos, dst_pos);
2506 
2507     BLOCK_COMMENT("arraycopy_range_checks done");
2508   }
2509 
2510   //
2511   //  Generate generic array copy stubs
2512   //
2513   //  Input:
2514   //    c_rarg0    -  src oop
2515   //    c_rarg1    -  src_pos (32-bits)
2516   //    c_rarg2    -  dst oop
2517   //    c_rarg3    -  dst_pos (32-bits)
2518   // not Win64
2519   //    c_rarg4    -  element count (32-bits)
2520   // Win64
2521   //    rsp+40     -  element count (32-bits)
2522   //
2523   //  Output:
2524   //    rax ==  0  -  success
2525   //    rax == -1^K - failure, where K is partial transfer count
2526   //
2527   address generate_generic_copy(const char *name,
2528                                 address byte_copy_entry, address short_copy_entry,
2529                                 address int_copy_entry, address oop_copy_entry,
2530                                 address long_copy_entry, address checkcast_copy_entry) {
2531 
2532     Label L_failed, L_failed_0, L_objArray;
2533     Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
2534 
2535     // Input registers
2536     const Register src        = c_rarg0;  // source array oop
2537     const Register src_pos    = c_rarg1;  // source position
2538     const Register dst        = c_rarg2;  // destination array oop
2539     const Register dst_pos    = c_rarg3;  // destination position
2540 #ifndef _WIN64
2541     const Register length     = c_rarg4;
2542 #else
2543     const Address  length(rsp, 6 * wordSize);  // elements count is on stack on Win64
2544 #endif
2545 
2546     { int modulus = CodeEntryAlignment;
2547       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
2548       int advance = target - (__ offset() % modulus);
2549       if (advance < 0)  advance += modulus;
2550       if (advance > 0)  __ nop(advance);
2551     }
2552     StubCodeMark mark(this, "StubRoutines", name);
2553 
2554     // Short-hop target to L_failed.  Makes for denser prologue code.
2555     __ BIND(L_failed_0);
2556     __ jmp(L_failed);
2557     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
2558 
2559     __ align(CodeEntryAlignment);
2560     address start = __ pc();
2561 
2562     __ enter(); // required for proper stackwalking of RuntimeStub frame
2563 
2564     // bump this on entry, not on exit:
2565     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
2566 
2567     //-----------------------------------------------------------------------
2568     // Assembler stub will be used for this call to arraycopy
2569     // if the following conditions are met:
2570     //
2571     // (1) src and dst must not be null.
2572     // (2) src_pos must not be negative.
2573     // (3) dst_pos must not be negative.
2574     // (4) length  must not be negative.
2575     // (5) src klass and dst klass should be the same and not NULL.
2576     // (6) src and dst should be arrays.
2577     // (7) src_pos + length must not exceed length of src.
2578     // (8) dst_pos + length must not exceed length of dst.
2579     //
2580 
2581     //  if (src == NULL) return -1;
2582     __ testptr(src, src);         // src oop
2583     size_t j1off = __ offset();
2584     __ jccb(Assembler::zero, L_failed_0);
2585 
2586     //  if (src_pos < 0) return -1;
2587     __ testl(src_pos, src_pos); // src_pos (32-bits)
2588     __ jccb(Assembler::negative, L_failed_0);
2589 
2590     //  if (dst == NULL) return -1;
2591     __ testptr(dst, dst);         // dst oop
2592     __ jccb(Assembler::zero, L_failed_0);
2593 
2594     //  if (dst_pos < 0) return -1;
2595     __ testl(dst_pos, dst_pos); // dst_pos (32-bits)
2596     size_t j4off = __ offset();
2597     __ jccb(Assembler::negative, L_failed_0);
2598 
2599     // The first four tests are very dense code,
2600     // but not quite dense enough to put four
2601     // jumps in a 16-byte instruction fetch buffer.
2602     // That's good, because some branch predicters
2603     // do not like jumps so close together.
2604     // Make sure of this.
2605     guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
2606 
2607     // registers used as temp
2608     const Register r11_length    = r11; // elements count to copy
2609     const Register r10_src_klass = r10; // array klass
2610 
2611     //  if (length < 0) return -1;
2612     __ movl(r11_length, length);        // length (elements count, 32-bits value)
2613     __ testl(r11_length, r11_length);
2614     __ jccb(Assembler::negative, L_failed_0);
2615 
2616     __ load_klass(r10_src_klass, src);
2617 #ifdef ASSERT
2618     //  assert(src->klass() != NULL);
2619     {
2620       BLOCK_COMMENT("assert klasses not null {");
2621       Label L1, L2;
2622       __ testptr(r10_src_klass, r10_src_klass);
2623       __ jcc(Assembler::notZero, L2);   // it is broken if klass is NULL
2624       __ bind(L1);
2625       __ stop("broken null klass");
2626       __ bind(L2);
2627       __ load_klass(rax, dst);
2628       __ cmpq(rax, 0);
2629       __ jcc(Assembler::equal, L1);     // this would be broken also
2630       BLOCK_COMMENT("} assert klasses not null done");
2631     }
2632 #endif
2633 
2634     // Load layout helper (32-bits)
2635     //
2636     //  |array_tag|     | header_size | element_type |     |log2_element_size|
2637     // 32        30    24            16              8     2                 0
2638     //
2639     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2640     //
2641 
2642     const int lh_offset = in_bytes(Klass::layout_helper_offset());
2643 
2644     // Handle objArrays completely differently...
2645     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2646     __ cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
2647     __ jcc(Assembler::equal, L_objArray);
2648 
2649     //  if (src->klass() != dst->klass()) return -1;
2650     __ load_klass(rax, dst);
2651     __ cmpq(r10_src_klass, rax);
2652     __ jcc(Assembler::notEqual, L_failed);
2653 
2654     const Register rax_lh = rax;  // layout helper
2655     __ movl(rax_lh, Address(r10_src_klass, lh_offset));
2656 
2657     //  if (!src->is_Array()) return -1;
2658     __ cmpl(rax_lh, Klass::_lh_neutral_value);
2659     __ jcc(Assembler::greaterEqual, L_failed);
2660 
2661     // At this point, it is known to be a typeArray (array_tag 0x3).
2662 #ifdef ASSERT
2663     {
2664       BLOCK_COMMENT("assert primitive array {");
2665       Label L;
2666       __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
2667       __ jcc(Assembler::greaterEqual, L);
2668       __ stop("must be a primitive array");
2669       __ bind(L);
2670       BLOCK_COMMENT("} assert primitive array done");
2671     }
2672 #endif
2673 
2674     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2675                            r10, L_failed);
2676 
2677     // TypeArrayKlass
2678     //
2679     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2680     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2681     //
2682 
2683     const Register r10_offset = r10;    // array offset
2684     const Register rax_elsize = rax_lh; // element size
2685 
2686     __ movl(r10_offset, rax_lh);
2687     __ shrl(r10_offset, Klass::_lh_header_size_shift);
2688     __ andptr(r10_offset, Klass::_lh_header_size_mask);   // array_offset
2689     __ addptr(src, r10_offset);           // src array offset
2690     __ addptr(dst, r10_offset);           // dst array offset
2691     BLOCK_COMMENT("choose copy loop based on element size");
2692     __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
2693 
2694     // next registers should be set before the jump to corresponding stub
2695     const Register from     = c_rarg0;  // source array address
2696     const Register to       = c_rarg1;  // destination array address
2697     const Register count    = c_rarg2;  // elements count
2698 
2699     // 'from', 'to', 'count' registers should be set in such order
2700     // since they are the same as 'src', 'src_pos', 'dst'.
2701 
2702   __ BIND(L_copy_bytes);
2703     __ cmpl(rax_elsize, 0);
2704     __ jccb(Assembler::notEqual, L_copy_shorts);
2705     __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
2706     __ lea(to,   Address(dst, dst_pos, Address::times_1, 0));// dst_addr
2707     __ movl2ptr(count, r11_length); // length
2708     __ jump(RuntimeAddress(byte_copy_entry));
2709 
2710   __ BIND(L_copy_shorts);
2711     __ cmpl(rax_elsize, LogBytesPerShort);
2712     __ jccb(Assembler::notEqual, L_copy_ints);
2713     __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
2714     __ lea(to,   Address(dst, dst_pos, Address::times_2, 0));// dst_addr
2715     __ movl2ptr(count, r11_length); // length
2716     __ jump(RuntimeAddress(short_copy_entry));
2717 
2718   __ BIND(L_copy_ints);
2719     __ cmpl(rax_elsize, LogBytesPerInt);
2720     __ jccb(Assembler::notEqual, L_copy_longs);
2721     __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
2722     __ lea(to,   Address(dst, dst_pos, Address::times_4, 0));// dst_addr
2723     __ movl2ptr(count, r11_length); // length
2724     __ jump(RuntimeAddress(int_copy_entry));
2725 
2726   __ BIND(L_copy_longs);
2727 #ifdef ASSERT
2728     {
2729       BLOCK_COMMENT("assert long copy {");
2730       Label L;
2731       __ cmpl(rax_elsize, LogBytesPerLong);
2732       __ jcc(Assembler::equal, L);
2733       __ stop("must be long copy, but elsize is wrong");
2734       __ bind(L);
2735       BLOCK_COMMENT("} assert long copy done");
2736     }
2737 #endif
2738     __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
2739     __ lea(to,   Address(dst, dst_pos, Address::times_8, 0));// dst_addr
2740     __ movl2ptr(count, r11_length); // length
2741     __ jump(RuntimeAddress(long_copy_entry));
2742 
2743     // ObjArrayKlass
2744   __ BIND(L_objArray);
2745     // live at this point:  r10_src_klass, r11_length, src[_pos], dst[_pos]
2746 
2747     Label L_plain_copy, L_checkcast_copy;
2748     //  test array classes for subtyping
2749     __ load_klass(rax, dst);
2750     __ cmpq(r10_src_klass, rax); // usual case is exact equality
2751     __ jcc(Assembler::notEqual, L_checkcast_copy);
2752 
2753     // Identically typed arrays can be copied without element-wise checks.
2754     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2755                            r10, L_failed);
2756 
2757     __ lea(from, Address(src, src_pos, TIMES_OOP,
2758                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
2759     __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2760                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
2761     __ movl2ptr(count, r11_length); // length
2762   __ BIND(L_plain_copy);
2763     __ jump(RuntimeAddress(oop_copy_entry));
2764 
2765   __ BIND(L_checkcast_copy);
2766     // live at this point:  r10_src_klass, r11_length, rax (dst_klass)
2767     {
2768       // Before looking at dst.length, make sure dst is also an objArray.
2769       __ cmpl(Address(rax, lh_offset), objArray_lh);
2770       __ jcc(Assembler::notEqual, L_failed);
2771 
2772       // It is safe to examine both src.length and dst.length.
2773       arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2774                              rax, L_failed);
2775 
2776       const Register r11_dst_klass = r11;
2777       __ load_klass(r11_dst_klass, dst); // reload
2778 
2779       // Marshal the base address arguments now, freeing registers.
2780       __ lea(from, Address(src, src_pos, TIMES_OOP,
2781                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2782       __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2783                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2784       __ movl(count, length);           // length (reloaded)
2785       Register sco_temp = c_rarg3;      // this register is free now
2786       assert_different_registers(from, to, count, sco_temp,
2787                                  r11_dst_klass, r10_src_klass);
2788       assert_clean_int(count, sco_temp);
2789 
2790       // Generate the type check.
2791       const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2792       __ movl(sco_temp, Address(r11_dst_klass, sco_offset));
2793       assert_clean_int(sco_temp, rax);
2794       generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
2795 
2796       // Fetch destination element klass from the ObjArrayKlass header.
2797       int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2798       __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
2799       __ movl(  sco_temp,      Address(r11_dst_klass, sco_offset));
2800       assert_clean_int(sco_temp, rax);
2801 
2802       // the checkcast_copy loop needs two extra arguments:
2803       assert(c_rarg3 == sco_temp, "#3 already in place");
2804       // Set up arguments for checkcast_copy_entry.
2805       setup_arg_regs(4);
2806       __ movptr(r8, r11_dst_klass);  // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
2807       __ jump(RuntimeAddress(checkcast_copy_entry));
2808     }
2809 
2810   __ BIND(L_failed);
2811     __ xorptr(rax, rax);
2812     __ notptr(rax); // return -1
2813     __ leave();   // required for proper stackwalking of RuntimeStub frame
2814     __ ret(0);
2815 
2816     return start;
2817   }
2818 
2819   void generate_arraycopy_stubs() {
2820     address entry;
2821     address entry_jbyte_arraycopy;
2822     address entry_jshort_arraycopy;
2823     address entry_jint_arraycopy;
2824     address entry_oop_arraycopy;
2825     address entry_jlong_arraycopy;
2826     address entry_checkcast_arraycopy;
2827 
2828     StubRoutines::_jbyte_disjoint_arraycopy  = generate_disjoint_byte_copy(false, &entry,
2829                                                                            "jbyte_disjoint_arraycopy");
2830     StubRoutines::_jbyte_arraycopy           = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy,
2831                                                                            "jbyte_arraycopy");
2832 
2833     StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
2834                                                                             "jshort_disjoint_arraycopy");
2835     StubRoutines::_jshort_arraycopy          = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy,
2836                                                                             "jshort_arraycopy");
2837 
2838     StubRoutines::_jint_disjoint_arraycopy   = generate_disjoint_int_oop_copy(false, false, &entry,
2839                                                                               "jint_disjoint_arraycopy");
2840     StubRoutines::_jint_arraycopy            = generate_conjoint_int_oop_copy(false, false, entry,
2841                                                                               &entry_jint_arraycopy, "jint_arraycopy");
2842 
2843     StubRoutines::_jlong_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, false, &entry,
2844                                                                                "jlong_disjoint_arraycopy");
2845     StubRoutines::_jlong_arraycopy           = generate_conjoint_long_oop_copy(false, false, entry,
2846                                                                                &entry_jlong_arraycopy, "jlong_arraycopy");
2847 
2848 
2849     if (UseCompressedOops) {
2850       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_int_oop_copy(false, true, &entry,
2851                                                                               "oop_disjoint_arraycopy");
2852       StubRoutines::_oop_arraycopy           = generate_conjoint_int_oop_copy(false, true, entry,
2853                                                                               &entry_oop_arraycopy, "oop_arraycopy");
2854       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_int_oop_copy(false, true, &entry,
2855                                                                                      "oop_disjoint_arraycopy_uninit",
2856                                                                                      /*dest_uninitialized*/true);
2857       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_int_oop_copy(false, true, entry,
2858                                                                                      NULL, "oop_arraycopy_uninit",
2859                                                                                      /*dest_uninitialized*/true);
2860     } else {
2861       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, true, &entry,
2862                                                                                "oop_disjoint_arraycopy");
2863       StubRoutines::_oop_arraycopy           = generate_conjoint_long_oop_copy(false, true, entry,
2864                                                                                &entry_oop_arraycopy, "oop_arraycopy");
2865       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_long_oop_copy(false, true, &entry,
2866                                                                                       "oop_disjoint_arraycopy_uninit",
2867                                                                                       /*dest_uninitialized*/true);
2868       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_long_oop_copy(false, true, entry,
2869                                                                                       NULL, "oop_arraycopy_uninit",
2870                                                                                       /*dest_uninitialized*/true);
2871     }
2872 
2873     StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2874     StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL,
2875                                                                         /*dest_uninitialized*/true);
2876 
2877     StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy("unsafe_arraycopy",
2878                                                               entry_jbyte_arraycopy,
2879                                                               entry_jshort_arraycopy,
2880                                                               entry_jint_arraycopy,
2881                                                               entry_jlong_arraycopy);
2882     StubRoutines::_generic_arraycopy   = generate_generic_copy("generic_arraycopy",
2883                                                                entry_jbyte_arraycopy,
2884                                                                entry_jshort_arraycopy,
2885                                                                entry_jint_arraycopy,
2886                                                                entry_oop_arraycopy,
2887                                                                entry_jlong_arraycopy,
2888                                                                entry_checkcast_arraycopy);
2889 
2890     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2891     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2892     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2893     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2894     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2895     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2896 
2897     // We don't generate specialized code for HeapWord-aligned source
2898     // arrays, so just use the code we've already generated
2899     StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = StubRoutines::_jbyte_disjoint_arraycopy;
2900     StubRoutines::_arrayof_jbyte_arraycopy           = StubRoutines::_jbyte_arraycopy;
2901 
2902     StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
2903     StubRoutines::_arrayof_jshort_arraycopy          = StubRoutines::_jshort_arraycopy;
2904 
2905     StubRoutines::_arrayof_jint_disjoint_arraycopy   = StubRoutines::_jint_disjoint_arraycopy;
2906     StubRoutines::_arrayof_jint_arraycopy            = StubRoutines::_jint_arraycopy;
2907 
2908     StubRoutines::_arrayof_jlong_disjoint_arraycopy  = StubRoutines::_jlong_disjoint_arraycopy;
2909     StubRoutines::_arrayof_jlong_arraycopy           = StubRoutines::_jlong_arraycopy;
2910 
2911     StubRoutines::_arrayof_oop_disjoint_arraycopy    = StubRoutines::_oop_disjoint_arraycopy;
2912     StubRoutines::_arrayof_oop_arraycopy             = StubRoutines::_oop_arraycopy;
2913 
2914     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit    = StubRoutines::_oop_disjoint_arraycopy_uninit;
2915     StubRoutines::_arrayof_oop_arraycopy_uninit             = StubRoutines::_oop_arraycopy_uninit;
2916   }
2917 
2918   void generate_math_stubs() {
2919     {
2920       StubCodeMark mark(this, "StubRoutines", "log");
2921       StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
2922 
2923       __ subq(rsp, 8);
2924       __ movdbl(Address(rsp, 0), xmm0);
2925       __ fld_d(Address(rsp, 0));
2926       __ flog();
2927       __ fstp_d(Address(rsp, 0));
2928       __ movdbl(xmm0, Address(rsp, 0));
2929       __ addq(rsp, 8);
2930       __ ret(0);
2931     }
2932     {
2933       StubCodeMark mark(this, "StubRoutines", "log10");
2934       StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
2935 
2936       __ subq(rsp, 8);
2937       __ movdbl(Address(rsp, 0), xmm0);
2938       __ fld_d(Address(rsp, 0));
2939       __ flog10();
2940       __ fstp_d(Address(rsp, 0));
2941       __ movdbl(xmm0, Address(rsp, 0));
2942       __ addq(rsp, 8);
2943       __ ret(0);
2944     }
2945     {
2946       StubCodeMark mark(this, "StubRoutines", "sin");
2947       StubRoutines::_intrinsic_sin = (double (*)(double)) __ pc();
2948 
2949       __ subq(rsp, 8);
2950       __ movdbl(Address(rsp, 0), xmm0);
2951       __ fld_d(Address(rsp, 0));
2952       __ trigfunc('s');
2953       __ fstp_d(Address(rsp, 0));
2954       __ movdbl(xmm0, Address(rsp, 0));
2955       __ addq(rsp, 8);
2956       __ ret(0);
2957     }
2958     {
2959       StubCodeMark mark(this, "StubRoutines", "cos");
2960       StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
2961 
2962       __ subq(rsp, 8);
2963       __ movdbl(Address(rsp, 0), xmm0);
2964       __ fld_d(Address(rsp, 0));
2965       __ trigfunc('c');
2966       __ fstp_d(Address(rsp, 0));
2967       __ movdbl(xmm0, Address(rsp, 0));
2968       __ addq(rsp, 8);
2969       __ ret(0);
2970     }
2971     {
2972       StubCodeMark mark(this, "StubRoutines", "tan");
2973       StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
2974 
2975       __ subq(rsp, 8);
2976       __ movdbl(Address(rsp, 0), xmm0);
2977       __ fld_d(Address(rsp, 0));
2978       __ trigfunc('t');
2979       __ fstp_d(Address(rsp, 0));
2980       __ movdbl(xmm0, Address(rsp, 0));
2981       __ addq(rsp, 8);
2982       __ ret(0);
2983     }
2984     {
2985       StubCodeMark mark(this, "StubRoutines", "exp");
2986       StubRoutines::_intrinsic_exp = (double (*)(double)) __ pc();
2987 
2988       __ subq(rsp, 8);
2989       __ movdbl(Address(rsp, 0), xmm0);
2990       __ fld_d(Address(rsp, 0));
2991       __ exp_with_fallback(0);
2992       __ fstp_d(Address(rsp, 0));
2993       __ movdbl(xmm0, Address(rsp, 0));
2994       __ addq(rsp, 8);
2995       __ ret(0);
2996     }
2997     {
2998       StubCodeMark mark(this, "StubRoutines", "pow");
2999       StubRoutines::_intrinsic_pow = (double (*)(double,double)) __ pc();
3000 
3001       __ subq(rsp, 8);
3002       __ movdbl(Address(rsp, 0), xmm1);
3003       __ fld_d(Address(rsp, 0));
3004       __ movdbl(Address(rsp, 0), xmm0);
3005       __ fld_d(Address(rsp, 0));
3006       __ pow_with_fallback(0);
3007       __ fstp_d(Address(rsp, 0));
3008       __ movdbl(xmm0, Address(rsp, 0));
3009       __ addq(rsp, 8);
3010       __ ret(0);
3011     }
3012   }
3013 
3014   // AES intrinsic stubs
3015   enum {AESBlockSize = 16};
3016 
3017   address generate_key_shuffle_mask() {
3018     __ align(16);
3019     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
3020     address start = __ pc();
3021     __ emit_data64( 0x0405060700010203, relocInfo::none );
3022     __ emit_data64( 0x0c0d0e0f08090a0b, relocInfo::none );
3023     return start;
3024   }
3025 
3026   // Utility routine for loading a 128-bit key word in little endian format
3027   // can optionally specify that the shuffle mask is already in an xmmregister
3028   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
3029     __ movdqu(xmmdst, Address(key, offset));
3030     if (xmm_shuf_mask != NULL) {
3031       __ pshufb(xmmdst, xmm_shuf_mask);
3032     } else {
3033       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3034     }
3035   }
3036 
3037   // Arguments:
3038   //
3039   // Inputs:
3040   //   c_rarg0   - source byte array address
3041   //   c_rarg1   - destination byte array address
3042   //   c_rarg2   - K (key) in little endian int array
3043   //
3044   address generate_aescrypt_encryptBlock() {
3045     assert(UseAES, "need AES instructions and misaligned SSE support");
3046     __ align(CodeEntryAlignment);
3047     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
3048     Label L_doLast;
3049     address start = __ pc();
3050 
3051     const Register from        = c_rarg0;  // source array address
3052     const Register to          = c_rarg1;  // destination array address
3053     const Register key         = c_rarg2;  // key array address
3054     const Register keylen      = rax;
3055 
3056     const XMMRegister xmm_result = xmm0;
3057     const XMMRegister xmm_key_shuf_mask = xmm1;
3058     // On win64 xmm6-xmm15 must be preserved so don't use them.
3059     const XMMRegister xmm_temp1  = xmm2;
3060     const XMMRegister xmm_temp2  = xmm3;
3061     const XMMRegister xmm_temp3  = xmm4;
3062     const XMMRegister xmm_temp4  = xmm5;
3063 
3064     __ enter(); // required for proper stackwalking of RuntimeStub frame
3065 
3066     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3067     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3068 
3069     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3070     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
3071 
3072     // For encryption, the java expanded key ordering is just what we need
3073     // we don't know if the key is aligned, hence not using load-execute form
3074 
3075     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
3076     __ pxor(xmm_result, xmm_temp1);
3077 
3078     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3079     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3080     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3081     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3082 
3083     __ aesenc(xmm_result, xmm_temp1);
3084     __ aesenc(xmm_result, xmm_temp2);
3085     __ aesenc(xmm_result, xmm_temp3);
3086     __ aesenc(xmm_result, xmm_temp4);
3087 
3088     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3089     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3090     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3091     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3092 
3093     __ aesenc(xmm_result, xmm_temp1);
3094     __ aesenc(xmm_result, xmm_temp2);
3095     __ aesenc(xmm_result, xmm_temp3);
3096     __ aesenc(xmm_result, xmm_temp4);
3097 
3098     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3099     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3100 
3101     __ cmpl(keylen, 44);
3102     __ jccb(Assembler::equal, L_doLast);
3103 
3104     __ aesenc(xmm_result, xmm_temp1);
3105     __ aesenc(xmm_result, xmm_temp2);
3106 
3107     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3108     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3109 
3110     __ cmpl(keylen, 52);
3111     __ jccb(Assembler::equal, L_doLast);
3112 
3113     __ aesenc(xmm_result, xmm_temp1);
3114     __ aesenc(xmm_result, xmm_temp2);
3115 
3116     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3117     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3118 
3119     __ BIND(L_doLast);
3120     __ aesenc(xmm_result, xmm_temp1);
3121     __ aesenclast(xmm_result, xmm_temp2);
3122     __ movdqu(Address(to, 0), xmm_result);        // store the result
3123     __ xorptr(rax, rax); // return 0
3124     __ leave(); // required for proper stackwalking of RuntimeStub frame
3125     __ ret(0);
3126 
3127     return start;
3128   }
3129 
3130 
3131   // Arguments:
3132   //
3133   // Inputs:
3134   //   c_rarg0   - source byte array address
3135   //   c_rarg1   - destination byte array address
3136   //   c_rarg2   - K (key) in little endian int array
3137   //
3138   address generate_aescrypt_decryptBlock() {
3139     assert(UseAES, "need AES instructions and misaligned SSE support");
3140     __ align(CodeEntryAlignment);
3141     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
3142     Label L_doLast;
3143     address start = __ pc();
3144 
3145     const Register from        = c_rarg0;  // source array address
3146     const Register to          = c_rarg1;  // destination array address
3147     const Register key         = c_rarg2;  // key array address
3148     const Register keylen      = rax;
3149 
3150     const XMMRegister xmm_result = xmm0;
3151     const XMMRegister xmm_key_shuf_mask = xmm1;
3152     // On win64 xmm6-xmm15 must be preserved so don't use them.
3153     const XMMRegister xmm_temp1  = xmm2;
3154     const XMMRegister xmm_temp2  = xmm3;
3155     const XMMRegister xmm_temp3  = xmm4;
3156     const XMMRegister xmm_temp4  = xmm5;
3157 
3158     __ enter(); // required for proper stackwalking of RuntimeStub frame
3159 
3160     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3161     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3162 
3163     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3164     __ movdqu(xmm_result, Address(from, 0));
3165 
3166     // for decryption java expanded key ordering is rotated one position from what we want
3167     // so we start from 0x10 here and hit 0x00 last
3168     // we don't know if the key is aligned, hence not using load-execute form
3169     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3170     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3171     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3172     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3173 
3174     __ pxor  (xmm_result, xmm_temp1);
3175     __ aesdec(xmm_result, xmm_temp2);
3176     __ aesdec(xmm_result, xmm_temp3);
3177     __ aesdec(xmm_result, xmm_temp4);
3178 
3179     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3180     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3181     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3182     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3183 
3184     __ aesdec(xmm_result, xmm_temp1);
3185     __ aesdec(xmm_result, xmm_temp2);
3186     __ aesdec(xmm_result, xmm_temp3);
3187     __ aesdec(xmm_result, xmm_temp4);
3188 
3189     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3190     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3191     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
3192 
3193     __ cmpl(keylen, 44);
3194     __ jccb(Assembler::equal, L_doLast);
3195 
3196     __ aesdec(xmm_result, xmm_temp1);
3197     __ aesdec(xmm_result, xmm_temp2);
3198 
3199     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3200     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3201 
3202     __ cmpl(keylen, 52);
3203     __ jccb(Assembler::equal, L_doLast);
3204 
3205     __ aesdec(xmm_result, xmm_temp1);
3206     __ aesdec(xmm_result, xmm_temp2);
3207 
3208     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3209     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3210 
3211     __ BIND(L_doLast);
3212     __ aesdec(xmm_result, xmm_temp1);
3213     __ aesdec(xmm_result, xmm_temp2);
3214 
3215     // for decryption the aesdeclast operation is always on key+0x00
3216     __ aesdeclast(xmm_result, xmm_temp3);
3217     __ movdqu(Address(to, 0), xmm_result);  // store the result
3218     __ xorptr(rax, rax); // return 0
3219     __ leave(); // required for proper stackwalking of RuntimeStub frame
3220     __ ret(0);
3221 
3222     return start;
3223   }
3224 
3225 
3226   // Arguments:
3227   //
3228   // Inputs:
3229   //   c_rarg0   - source byte array address
3230   //   c_rarg1   - destination byte array address
3231   //   c_rarg2   - K (key) in little endian int array
3232   //   c_rarg3   - r vector byte array address
3233   //   c_rarg4   - input length
3234   //
3235   // Output:
3236   //   rax       - input length
3237   //
3238   address generate_cipherBlockChaining_encryptAESCrypt() {
3239     assert(UseAES, "need AES instructions and misaligned SSE support");
3240     __ align(CodeEntryAlignment);
3241     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
3242     address start = __ pc();
3243 
3244     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
3245     const Register from        = c_rarg0;  // source array address
3246     const Register to          = c_rarg1;  // destination array address
3247     const Register key         = c_rarg2;  // key array address
3248     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
3249                                            // and left with the results of the last encryption block
3250 #ifndef _WIN64
3251     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
3252 #else
3253     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
3254     const Register len_reg     = r10;      // pick the first volatile windows register
3255 #endif
3256     const Register pos         = rax;
3257 
3258     // xmm register assignments for the loops below
3259     const XMMRegister xmm_result = xmm0;
3260     const XMMRegister xmm_temp   = xmm1;
3261     // keys 0-10 preloaded into xmm2-xmm12
3262     const int XMM_REG_NUM_KEY_FIRST = 2;
3263     const int XMM_REG_NUM_KEY_LAST  = 15;
3264     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3265     const XMMRegister xmm_key10  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+10);
3266     const XMMRegister xmm_key11  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+11);
3267     const XMMRegister xmm_key12  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+12);
3268     const XMMRegister xmm_key13  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+13);
3269 
3270     __ enter(); // required for proper stackwalking of RuntimeStub frame
3271 
3272 #ifdef _WIN64
3273     // on win64, fill len_reg from stack position
3274     __ movl(len_reg, len_mem);
3275     // save the xmm registers which must be preserved 6-15
3276     __ subptr(rsp, -rsp_after_call_off * wordSize);
3277     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
3278       __ movdqu(xmm_save(i), as_XMMRegister(i));
3279     }
3280 #else
3281     __ push(len_reg); // Save
3282 #endif
3283 
3284     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
3285     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3286     // load up xmm regs xmm2 thru xmm12 with key 0x00 - 0xa0
3287     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_FIRST+10; rnum++) {
3288       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3289       offset += 0x10;
3290     }
3291     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
3292 
3293     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3294     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3295     __ cmpl(rax, 44);
3296     __ jcc(Assembler::notEqual, L_key_192_256);
3297 
3298     // 128 bit code follows here
3299     __ movptr(pos, 0);
3300     __ align(OptoLoopAlignment);
3301 
3302     __ BIND(L_loopTop_128);
3303     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3304     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3305     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3306     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 9; rnum++) {
3307       __ aesenc(xmm_result, as_XMMRegister(rnum));
3308     }
3309     __ aesenclast(xmm_result, xmm_key10);
3310     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3311     // no need to store r to memory until we exit
3312     __ addptr(pos, AESBlockSize);
3313     __ subptr(len_reg, AESBlockSize);
3314     __ jcc(Assembler::notEqual, L_loopTop_128);
3315 
3316     __ BIND(L_exit);
3317     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
3318 
3319 #ifdef _WIN64
3320     // restore xmm regs belonging to calling function
3321     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
3322       __ movdqu(as_XMMRegister(i), xmm_save(i));
3323     }
3324     __ movl(rax, len_mem);
3325 #else
3326     __ pop(rax); // return length
3327 #endif
3328     __ leave(); // required for proper stackwalking of RuntimeStub frame
3329     __ ret(0);
3330 
3331     __ BIND(L_key_192_256);
3332     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
3333     load_key(xmm_key11, key, 0xb0, xmm_key_shuf_mask);
3334     load_key(xmm_key12, key, 0xc0, xmm_key_shuf_mask);
3335     __ cmpl(rax, 52);
3336     __ jcc(Assembler::notEqual, L_key_256);
3337 
3338     // 192-bit code follows here (could be changed to use more xmm registers)
3339     __ movptr(pos, 0);
3340     __ align(OptoLoopAlignment);
3341 
3342     __ BIND(L_loopTop_192);
3343     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3344     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3345     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3346     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 11; rnum++) {
3347       __ aesenc(xmm_result, as_XMMRegister(rnum));
3348     }
3349     __ aesenclast(xmm_result, xmm_key12);
3350     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3351     // no need to store r to memory until we exit
3352     __ addptr(pos, AESBlockSize);
3353     __ subptr(len_reg, AESBlockSize);
3354     __ jcc(Assembler::notEqual, L_loopTop_192);
3355     __ jmp(L_exit);
3356 
3357     __ BIND(L_key_256);
3358     // 256-bit code follows here (could be changed to use more xmm registers)
3359     load_key(xmm_key13, key, 0xd0, xmm_key_shuf_mask);
3360     __ movptr(pos, 0);
3361     __ align(OptoLoopAlignment);
3362 
3363     __ BIND(L_loopTop_256);
3364     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3365     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3366     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3367     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 13; rnum++) {
3368       __ aesenc(xmm_result, as_XMMRegister(rnum));
3369     }
3370     load_key(xmm_temp, key, 0xe0);
3371     __ aesenclast(xmm_result, xmm_temp);
3372     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3373     // no need to store r to memory until we exit
3374     __ addptr(pos, AESBlockSize);
3375     __ subptr(len_reg, AESBlockSize);
3376     __ jcc(Assembler::notEqual, L_loopTop_256);
3377     __ jmp(L_exit);
3378 
3379     return start;
3380   }
3381 
3382   // Safefetch stubs.
3383   void generate_safefetch(const char* name, int size, address* entry,
3384                           address* fault_pc, address* continuation_pc) {
3385     // safefetch signatures:
3386     //   int      SafeFetch32(int*      adr, int      errValue);
3387     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3388     //
3389     // arguments:
3390     //   c_rarg0 = adr
3391     //   c_rarg1 = errValue
3392     //
3393     // result:
3394     //   PPC_RET  = *adr or errValue
3395 
3396     StubCodeMark mark(this, "StubRoutines", name);
3397 
3398     // Entry point, pc or function descriptor.
3399     *entry = __ pc();
3400 
3401     // Load *adr into c_rarg1, may fault.
3402     *fault_pc = __ pc();
3403     switch (size) {
3404       case 4:
3405         // int32_t
3406         __ movl(c_rarg1, Address(c_rarg0, 0));
3407         break;
3408       case 8:
3409         // int64_t
3410         __ movq(c_rarg1, Address(c_rarg0, 0));
3411         break;
3412       default:
3413         ShouldNotReachHere();
3414     }
3415 
3416     // return errValue or *adr
3417     *continuation_pc = __ pc();
3418     __ movq(rax, c_rarg1);
3419     __ ret(0);
3420   }
3421 
3422   // This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time
3423   // to hide instruction latency
3424   //
3425   // Arguments:
3426   //
3427   // Inputs:
3428   //   c_rarg0   - source byte array address
3429   //   c_rarg1   - destination byte array address
3430   //   c_rarg2   - K (key) in little endian int array
3431   //   c_rarg3   - r vector byte array address
3432   //   c_rarg4   - input length
3433   //
3434   // Output:
3435   //   rax       - input length
3436   //
3437 
3438   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
3439     assert(UseAES, "need AES instructions and misaligned SSE support");
3440     __ align(CodeEntryAlignment);
3441     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
3442     address start = __ pc();
3443 
3444     Label L_exit, L_key_192_256, L_key_256;
3445     Label L_singleBlock_loopTop_128, L_multiBlock_loopTop_128;
3446     Label L_singleBlock_loopTop_192, L_singleBlock_loopTop_256;
3447     const Register from        = c_rarg0;  // source array address
3448     const Register to          = c_rarg1;  // destination array address
3449     const Register key         = c_rarg2;  // key array address
3450     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
3451                                            // and left with the results of the last encryption block
3452 #ifndef _WIN64
3453     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
3454 #else
3455     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
3456     const Register len_reg     = r10;      // pick the first volatile windows register
3457 #endif
3458     const Register pos         = rax;
3459 
3460     // keys 0-10 preloaded into xmm2-xmm12
3461     const int XMM_REG_NUM_KEY_FIRST = 5;
3462     const int XMM_REG_NUM_KEY_LAST  = 15;
3463     const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3464     const XMMRegister xmm_key_last  = as_XMMRegister(XMM_REG_NUM_KEY_LAST);
3465 
3466     __ enter(); // required for proper stackwalking of RuntimeStub frame
3467 
3468 #ifdef _WIN64
3469     // on win64, fill len_reg from stack position
3470     __ movl(len_reg, len_mem);
3471     // save the xmm registers which must be preserved 6-15
3472     __ subptr(rsp, -rsp_after_call_off * wordSize);
3473     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
3474       __ movdqu(xmm_save(i), as_XMMRegister(i));
3475     }
3476 #else
3477     __ push(len_reg); // Save
3478 #endif
3479 
3480     // the java expanded key ordering is rotated one position from what we want
3481     // so we start from 0x10 here and hit 0x00 last
3482     const XMMRegister xmm_key_shuf_mask = xmm1;  // used temporarily to swap key bytes up front
3483     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3484     // load up xmm regs 5 thru 15 with key 0x10 - 0xa0 - 0x00
3485     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x10; rnum < XMM_REG_NUM_KEY_LAST; rnum++) {
3486       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3487       offset += 0x10;
3488     }
3489     load_key(xmm_key_last, key, 0x00, xmm_key_shuf_mask);
3490 
3491     const XMMRegister xmm_prev_block_cipher = xmm1;  // holds cipher of previous block
3492 
3493     // registers holding the four results in the parallelized loop
3494     const XMMRegister xmm_result0 = xmm0;
3495     const XMMRegister xmm_result1 = xmm2;
3496     const XMMRegister xmm_result2 = xmm3;
3497     const XMMRegister xmm_result3 = xmm4;
3498 
3499     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));   // initialize with initial rvec
3500 
3501     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3502     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3503     __ cmpl(rax, 44);
3504     __ jcc(Assembler::notEqual, L_key_192_256);
3505 
3506 
3507     // 128-bit code follows here, parallelized
3508     __ movptr(pos, 0);
3509     __ align(OptoLoopAlignment);
3510     __ BIND(L_multiBlock_loopTop_128);
3511     __ cmpptr(len_reg, 4*AESBlockSize);           // see if at least 4 blocks left
3512     __ jcc(Assembler::less, L_singleBlock_loopTop_128);
3513 
3514     __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0*AESBlockSize));   // get next 4 blocks into xmmresult registers
3515     __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1*AESBlockSize));
3516     __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2*AESBlockSize));
3517     __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3*AESBlockSize));
3518 
3519 #define DoFour(opc, src_reg)                    \
3520     __ opc(xmm_result0, src_reg);               \
3521     __ opc(xmm_result1, src_reg);               \
3522     __ opc(xmm_result2, src_reg);               \
3523     __ opc(xmm_result3, src_reg);
3524 
3525     DoFour(pxor, xmm_key_first);
3526     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
3527       DoFour(aesdec, as_XMMRegister(rnum));
3528     }
3529     DoFour(aesdeclast, xmm_key_last);
3530     // for each result, xor with the r vector of previous cipher block
3531     __ pxor(xmm_result0, xmm_prev_block_cipher);
3532     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0*AESBlockSize));
3533     __ pxor(xmm_result1, xmm_prev_block_cipher);
3534     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1*AESBlockSize));
3535     __ pxor(xmm_result2, xmm_prev_block_cipher);
3536     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2*AESBlockSize));
3537     __ pxor(xmm_result3, xmm_prev_block_cipher);
3538     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3*AESBlockSize));   // this will carry over to next set of blocks
3539 
3540     __ movdqu(Address(to, pos, Address::times_1, 0*AESBlockSize), xmm_result0);     // store 4 results into the next 64 bytes of output
3541     __ movdqu(Address(to, pos, Address::times_1, 1*AESBlockSize), xmm_result1);
3542     __ movdqu(Address(to, pos, Address::times_1, 2*AESBlockSize), xmm_result2);
3543     __ movdqu(Address(to, pos, Address::times_1, 3*AESBlockSize), xmm_result3);
3544 
3545     __ addptr(pos, 4*AESBlockSize);
3546     __ subptr(len_reg, 4*AESBlockSize);
3547     __ jmp(L_multiBlock_loopTop_128);
3548 
3549     // registers used in the non-parallelized loops
3550     // xmm register assignments for the loops below
3551     const XMMRegister xmm_result = xmm0;
3552     const XMMRegister xmm_prev_block_cipher_save = xmm2;
3553     const XMMRegister xmm_key11 = xmm3;
3554     const XMMRegister xmm_key12 = xmm4;
3555     const XMMRegister xmm_temp  = xmm4;
3556 
3557     __ align(OptoLoopAlignment);
3558     __ BIND(L_singleBlock_loopTop_128);
3559     __ cmpptr(len_reg, 0);           // any blocks left??
3560     __ jcc(Assembler::equal, L_exit);
3561     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of cipher input
3562     __ movdqa(xmm_prev_block_cipher_save, xmm_result);              // save for next r vector
3563     __ pxor  (xmm_result, xmm_key_first);               // do the aes dec rounds
3564     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
3565       __ aesdec(xmm_result, as_XMMRegister(rnum));
3566     }
3567     __ aesdeclast(xmm_result, xmm_key_last);
3568     __ pxor  (xmm_result, xmm_prev_block_cipher);               // xor with the current r vector
3569     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3570     // no need to store r to memory until we exit
3571     __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save);              // set up next r vector with cipher input from this block
3572 
3573     __ addptr(pos, AESBlockSize);
3574     __ subptr(len_reg, AESBlockSize);
3575     __ jmp(L_singleBlock_loopTop_128);
3576 
3577 
3578     __ BIND(L_exit);
3579     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher);     // final value of r stored in rvec of CipherBlockChaining object
3580 #ifdef _WIN64
3581     // restore regs belonging to calling function
3582     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
3583       __ movdqu(as_XMMRegister(i), xmm_save(i));
3584     }
3585     __ movl(rax, len_mem);
3586 #else
3587     __ pop(rax); // return length
3588 #endif
3589     __ leave(); // required for proper stackwalking of RuntimeStub frame
3590     __ ret(0);
3591 
3592 
3593     __ BIND(L_key_192_256);
3594     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
3595     load_key(xmm_key11, key, 0xb0);
3596     __ cmpl(rax, 52);
3597     __ jcc(Assembler::notEqual, L_key_256);
3598 
3599     // 192-bit code follows here (could be optimized to use parallelism)
3600     load_key(xmm_key12, key, 0xc0);     // 192-bit key goes up to c0
3601     __ movptr(pos, 0);
3602     __ align(OptoLoopAlignment);
3603 
3604     __ BIND(L_singleBlock_loopTop_192);
3605     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of cipher input
3606     __ movdqa(xmm_prev_block_cipher_save, xmm_result);              // save for next r vector
3607     __ pxor  (xmm_result, xmm_key_first);               // do the aes dec rounds
3608     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
3609       __ aesdec(xmm_result, as_XMMRegister(rnum));
3610     }
3611     __ aesdec(xmm_result, xmm_key11);
3612     __ aesdec(xmm_result, xmm_key12);
3613     __ aesdeclast(xmm_result, xmm_key_last);                    // xmm15 always came from key+0
3614     __ pxor  (xmm_result, xmm_prev_block_cipher);               // xor with the current r vector
3615     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);  // store into the next 16 bytes of output
3616     // no need to store r to memory until we exit
3617     __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save);  // set up next r vector with cipher input from this block
3618     __ addptr(pos, AESBlockSize);
3619     __ subptr(len_reg, AESBlockSize);
3620     __ jcc(Assembler::notEqual,L_singleBlock_loopTop_192);
3621     __ jmp(L_exit);
3622 
3623     __ BIND(L_key_256);
3624     // 256-bit code follows here (could be optimized to use parallelism)
3625     __ movptr(pos, 0);
3626     __ align(OptoLoopAlignment);
3627 
3628     __ BIND(L_singleBlock_loopTop_256);
3629     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
3630     __ movdqa(xmm_prev_block_cipher_save, xmm_result);              // save for next r vector
3631     __ pxor  (xmm_result, xmm_key_first);               // do the aes dec rounds
3632     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
3633       __ aesdec(xmm_result, as_XMMRegister(rnum));
3634     }
3635     __ aesdec(xmm_result, xmm_key11);
3636     load_key(xmm_temp, key, 0xc0);
3637     __ aesdec(xmm_result, xmm_temp);
3638     load_key(xmm_temp, key, 0xd0);
3639     __ aesdec(xmm_result, xmm_temp);
3640     load_key(xmm_temp, key, 0xe0);     // 256-bit key goes up to e0
3641     __ aesdec(xmm_result, xmm_temp);
3642     __ aesdeclast(xmm_result, xmm_key_last);          // xmm15 came from key+0
3643     __ pxor  (xmm_result, xmm_prev_block_cipher);               // xor with the current r vector
3644     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);  // store into the next 16 bytes of output
3645     // no need to store r to memory until we exit
3646     __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save);  // set up next r vector with cipher input from this block
3647     __ addptr(pos, AESBlockSize);
3648     __ subptr(len_reg, AESBlockSize);
3649     __ jcc(Assembler::notEqual,L_singleBlock_loopTop_256);
3650     __ jmp(L_exit);
3651 
3652     return start;
3653   }
3654 
3655 
3656   // byte swap x86 long
3657   address generate_ghash_long_swap_mask() {
3658     __ align(CodeEntryAlignment);
3659     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
3660     address start = __ pc();
3661     __ emit_data64(0x0f0e0d0c0b0a0908, relocInfo::none );
3662     __ emit_data64(0x0706050403020100, relocInfo::none );
3663   return start;
3664   }
3665 
3666   // byte swap x86 byte array
3667   address generate_ghash_byte_swap_mask() {
3668     __ align(CodeEntryAlignment);
3669     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
3670     address start = __ pc();
3671     __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none );
3672     __ emit_data64(0x0001020304050607, relocInfo::none );
3673   return start;
3674   }
3675 
3676   /* Single and multi-block ghash operations */
3677   address generate_ghash_processBlocks() {
3678     __ align(CodeEntryAlignment);
3679     Label L_ghash_loop, L_exit;
3680     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
3681     address start = __ pc();
3682 
3683     const Register state        = c_rarg0;
3684     const Register subkeyH      = c_rarg1;
3685     const Register data         = c_rarg2;
3686     const Register blocks       = c_rarg3;
3687 
3688 #ifdef _WIN64
3689     const int XMM_REG_LAST  = 10;
3690 #endif
3691 
3692     const XMMRegister xmm_temp0 = xmm0;
3693     const XMMRegister xmm_temp1 = xmm1;
3694     const XMMRegister xmm_temp2 = xmm2;
3695     const XMMRegister xmm_temp3 = xmm3;
3696     const XMMRegister xmm_temp4 = xmm4;
3697     const XMMRegister xmm_temp5 = xmm5;
3698     const XMMRegister xmm_temp6 = xmm6;
3699     const XMMRegister xmm_temp7 = xmm7;
3700     const XMMRegister xmm_temp8 = xmm8;
3701     const XMMRegister xmm_temp9 = xmm9;
3702     const XMMRegister xmm_temp10 = xmm10;
3703 
3704     __ enter();
3705 
3706 #ifdef _WIN64
3707     // save the xmm registers which must be preserved 6-10
3708     __ subptr(rsp, -rsp_after_call_off * wordSize);
3709     for (int i = 6; i <= XMM_REG_LAST; i++) {
3710       __ movdqu(xmm_save(i), as_XMMRegister(i));
3711     }
3712 #endif
3713 
3714     __ movdqu(xmm_temp10, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3715 
3716     __ movdqu(xmm_temp0, Address(state, 0));
3717     __ pshufb(xmm_temp0, xmm_temp10);
3718 
3719 
3720     __ BIND(L_ghash_loop);
3721     __ movdqu(xmm_temp2, Address(data, 0));
3722     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
3723 
3724     __ movdqu(xmm_temp1, Address(subkeyH, 0));
3725     __ pshufb(xmm_temp1, xmm_temp10);
3726 
3727     __ pxor(xmm_temp0, xmm_temp2);
3728 
3729     //
3730     // Multiply with the hash key
3731     //
3732     __ movdqu(xmm_temp3, xmm_temp0);
3733     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
3734     __ movdqu(xmm_temp4, xmm_temp0);
3735     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
3736 
3737     __ movdqu(xmm_temp5, xmm_temp0);
3738     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
3739     __ movdqu(xmm_temp6, xmm_temp0);
3740     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
3741 
3742     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
3743 
3744     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
3745     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
3746     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
3747     __ pxor(xmm_temp3, xmm_temp5);
3748     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
3749                                         // of the carry-less multiplication of
3750                                         // xmm0 by xmm1.
3751 
3752     // We shift the result of the multiplication by one bit position
3753     // to the left to cope for the fact that the bits are reversed.
3754     __ movdqu(xmm_temp7, xmm_temp3);
3755     __ movdqu(xmm_temp8, xmm_temp6);
3756     __ pslld(xmm_temp3, 1);
3757     __ pslld(xmm_temp6, 1);
3758     __ psrld(xmm_temp7, 31);
3759     __ psrld(xmm_temp8, 31);
3760     __ movdqu(xmm_temp9, xmm_temp7);
3761     __ pslldq(xmm_temp8, 4);
3762     __ pslldq(xmm_temp7, 4);
3763     __ psrldq(xmm_temp9, 12);
3764     __ por(xmm_temp3, xmm_temp7);
3765     __ por(xmm_temp6, xmm_temp8);
3766     __ por(xmm_temp6, xmm_temp9);
3767 
3768     //
3769     // First phase of the reduction
3770     //
3771     // Move xmm3 into xmm7, xmm8, xmm9 in order to perform the shifts
3772     // independently.
3773     __ movdqu(xmm_temp7, xmm_temp3);
3774     __ movdqu(xmm_temp8, xmm_temp3);
3775     __ movdqu(xmm_temp9, xmm_temp3);
3776     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
3777     __ pslld(xmm_temp8, 30);    // packed right shift shifting << 30
3778     __ pslld(xmm_temp9, 25);    // packed right shift shifting << 25
3779     __ pxor(xmm_temp7, xmm_temp8);      // xor the shifted versions
3780     __ pxor(xmm_temp7, xmm_temp9);
3781     __ movdqu(xmm_temp8, xmm_temp7);
3782     __ pslldq(xmm_temp7, 12);
3783     __ psrldq(xmm_temp8, 4);
3784     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
3785 
3786     //
3787     // Second phase of the reduction
3788     //
3789     // Make 3 copies of xmm3 in xmm2, xmm4, xmm5 for doing these
3790     // shift operations.
3791     __ movdqu(xmm_temp2, xmm_temp3);
3792     __ movdqu(xmm_temp4, xmm_temp3);
3793     __ movdqu(xmm_temp5, xmm_temp3);
3794     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
3795     __ psrld(xmm_temp4, 2);     // packed left shifting >> 2
3796     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
3797     __ pxor(xmm_temp2, xmm_temp4);      // xor the shifted versions
3798     __ pxor(xmm_temp2, xmm_temp5);
3799     __ pxor(xmm_temp2, xmm_temp8);
3800     __ pxor(xmm_temp3, xmm_temp2);
3801     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
3802 
3803     __ decrement(blocks);
3804     __ jcc(Assembler::zero, L_exit);
3805     __ movdqu(xmm_temp0, xmm_temp6);
3806     __ addptr(data, 16);
3807     __ jmp(L_ghash_loop);
3808 
3809     __ BIND(L_exit);
3810     __ pshufb(xmm_temp6, xmm_temp10);          // Byte swap 16-byte result
3811     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
3812 
3813 #ifdef _WIN64
3814     // restore xmm regs belonging to calling function
3815     for (int i = 6; i <= XMM_REG_LAST; i++) {
3816       __ movdqu(as_XMMRegister(i), xmm_save(i));
3817     }
3818 #endif
3819     __ leave();
3820     __ ret(0);
3821     return start;
3822   }
3823 
3824   /**
3825    *  Arguments:
3826    *
3827    * Inputs:
3828    *   c_rarg0   - int crc
3829    *   c_rarg1   - byte* buf
3830    *   c_rarg2   - int length
3831    *
3832    * Ouput:
3833    *       rax   - int crc result
3834    */
3835   address generate_updateBytesCRC32() {
3836     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
3837 
3838     __ align(CodeEntryAlignment);
3839     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
3840 
3841     address start = __ pc();
3842     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
3843     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
3844     // rscratch1: r10
3845     const Register crc   = c_rarg0;  // crc
3846     const Register buf   = c_rarg1;  // source java byte array address
3847     const Register len   = c_rarg2;  // length
3848     const Register table = c_rarg3;  // crc_table address (reuse register)
3849     const Register tmp   = r11;
3850     assert_different_registers(crc, buf, len, table, tmp, rax);
3851 
3852     BLOCK_COMMENT("Entry:");
3853     __ enter(); // required for proper stackwalking of RuntimeStub frame
3854 
3855     __ kernel_crc32(crc, buf, len, table, tmp);
3856 
3857     __ movl(rax, crc);
3858     __ leave(); // required for proper stackwalking of RuntimeStub frame
3859     __ ret(0);
3860 
3861     return start;
3862   }
3863 
3864 
3865   /**
3866    *  Arguments:
3867    *
3868    *  Input:
3869    *    c_rarg0   - x address
3870    *    c_rarg1   - x length
3871    *    c_rarg2   - y address
3872    *    c_rarg3   - y lenth
3873    * not Win64
3874    *    c_rarg4   - z address
3875    *    c_rarg5   - z length
3876    * Win64
3877    *    rsp+40    - z address
3878    *    rsp+48    - z length
3879    */
3880   address generate_multiplyToLen() {
3881     __ align(CodeEntryAlignment);
3882     StubCodeMark mark(this, "StubRoutines", "multiplyToLen");
3883 
3884     address start = __ pc();
3885     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
3886     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
3887     const Register x     = rdi;
3888     const Register xlen  = rax;
3889     const Register y     = rsi;
3890     const Register ylen  = rcx;
3891     const Register z     = r8;
3892     const Register zlen  = r11;
3893 
3894     // Next registers will be saved on stack in multiply_to_len().
3895     const Register tmp1  = r12;
3896     const Register tmp2  = r13;
3897     const Register tmp3  = r14;
3898     const Register tmp4  = r15;
3899     const Register tmp5  = rbx;
3900 
3901     BLOCK_COMMENT("Entry:");
3902     __ enter(); // required for proper stackwalking of RuntimeStub frame
3903 
3904 #ifndef _WIN64
3905     __ movptr(zlen, r9); // Save r9 in r11 - zlen
3906 #endif
3907     setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx
3908                        // ylen => rcx, z => r8, zlen => r11
3909                        // r9 and r10 may be used to save non-volatile registers
3910 #ifdef _WIN64
3911     // last 2 arguments (#4, #5) are on stack on Win64
3912     __ movptr(z, Address(rsp, 6 * wordSize));
3913     __ movptr(zlen, Address(rsp, 7 * wordSize));
3914 #endif
3915 
3916     __ movptr(xlen, rsi);
3917     __ movptr(y,    rdx);
3918     __ multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5);
3919 
3920     restore_arg_regs();
3921 
3922     __ leave(); // required for proper stackwalking of RuntimeStub frame
3923     __ ret(0);
3924 
3925     return start;
3926   }
3927 
3928 /**
3929    *  Arguments:
3930    *
3931   //  Input:
3932   //    c_rarg0   - x address
3933   //    c_rarg1   - x length
3934   //    c_rarg2   - z address
3935   //    c_rarg3   - z lenth
3936    *
3937    */
3938   address generate_squareToLen() {
3939 
3940     __ align(CodeEntryAlignment);
3941     StubCodeMark mark(this, "StubRoutines", "squareToLen");
3942 
3943     address start = __ pc();
3944     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
3945     // Unix:  rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...)
3946     const Register x      = rdi;
3947     const Register len    = rsi;
3948     const Register z      = r8;
3949     const Register zlen   = rcx;
3950 
3951    const Register tmp1      = r12;
3952    const Register tmp2      = r13;
3953    const Register tmp3      = r14;
3954    const Register tmp4      = r15;
3955    const Register tmp5      = rbx;
3956 
3957     BLOCK_COMMENT("Entry:");
3958     __ enter(); // required for proper stackwalking of RuntimeStub frame
3959 
3960        setup_arg_regs(4); // x => rdi, len => rsi, z => rdx
3961                           // zlen => rcx
3962                           // r9 and r10 may be used to save non-volatile registers
3963     __ movptr(r8, rdx);
3964     __ square_to_len(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
3965 
3966     restore_arg_regs();
3967 
3968     __ leave(); // required for proper stackwalking of RuntimeStub frame
3969     __ ret(0);
3970 
3971     return start;
3972   }
3973 
3974    /**
3975    *  Arguments:
3976    *
3977    *  Input:
3978    *    c_rarg0   - out address
3979    *    c_rarg1   - in address
3980    *    c_rarg2   - offset
3981    *    c_rarg3   - len
3982    * not Win64
3983    *    c_rarg4   - k
3984    * Win64
3985    *    rsp+40    - k
3986    */
3987   address generate_mulAdd() {
3988     __ align(CodeEntryAlignment);
3989     StubCodeMark mark(this, "StubRoutines", "mulAdd");
3990 
3991     address start = __ pc();
3992     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
3993     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
3994     const Register out     = rdi;
3995     const Register in      = rsi;
3996     const Register offset  = r11;
3997     const Register len     = rcx;
3998     const Register k       = r8;
3999 
4000     // Next registers will be saved on stack in mul_add().
4001     const Register tmp1  = r12;
4002     const Register tmp2  = r13;
4003     const Register tmp3  = r14;
4004     const Register tmp4  = r15;
4005     const Register tmp5  = rbx;
4006 
4007     BLOCK_COMMENT("Entry:");
4008     __ enter(); // required for proper stackwalking of RuntimeStub frame
4009 
4010     setup_arg_regs(4); // out => rdi, in => rsi, offset => rdx
4011                        // len => rcx, k => r8
4012                        // r9 and r10 may be used to save non-volatile registers
4013 #ifdef _WIN64
4014     // last argument is on stack on Win64
4015     __ movl(k, Address(rsp, 6 * wordSize));
4016 #endif
4017     __ movptr(r11, rdx);  // move offset in rdx to offset(r11)
4018     __ mul_add(out, in, offset, len, k, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
4019 
4020     restore_arg_regs();
4021 
4022     __ leave(); // required for proper stackwalking of RuntimeStub frame
4023     __ ret(0);
4024 
4025     return start;
4026   }
4027 
4028 
4029 #undef __
4030 #define __ masm->
4031 
4032   // Continuation point for throwing of implicit exceptions that are
4033   // not handled in the current activation. Fabricates an exception
4034   // oop and initiates normal exception dispatching in this
4035   // frame. Since we need to preserve callee-saved values (currently
4036   // only for C2, but done for C1 as well) we need a callee-saved oop
4037   // map and therefore have to make these stubs into RuntimeStubs
4038   // rather than BufferBlobs.  If the compiler needs all registers to
4039   // be preserved between the fault point and the exception handler
4040   // then it must assume responsibility for that in
4041   // AbstractCompiler::continuation_for_implicit_null_exception or
4042   // continuation_for_implicit_division_by_zero_exception. All other
4043   // implicit exceptions (e.g., NullPointerException or
4044   // AbstractMethodError on entry) are either at call sites or
4045   // otherwise assume that stack unwinding will be initiated, so
4046   // caller saved registers were assumed volatile in the compiler.
4047   address generate_throw_exception(const char* name,
4048                                    address runtime_entry,
4049                                    Register arg1 = noreg,
4050                                    Register arg2 = noreg) {
4051     // Information about frame layout at time of blocking runtime call.
4052     // Note that we only have to preserve callee-saved registers since
4053     // the compilers are responsible for supplying a continuation point
4054     // if they expect all registers to be preserved.
4055     enum layout {
4056       rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
4057       rbp_off2,
4058       return_off,
4059       return_off2,
4060       framesize // inclusive of return address
4061     };
4062 
4063     int insts_size = 512;
4064     int locs_size  = 64;
4065 
4066     CodeBuffer code(name, insts_size, locs_size);
4067     OopMapSet* oop_maps  = new OopMapSet();
4068     MacroAssembler* masm = new MacroAssembler(&code);
4069 
4070     address start = __ pc();
4071 
4072     // This is an inlined and slightly modified version of call_VM
4073     // which has the ability to fetch the return PC out of
4074     // thread-local storage and also sets up last_Java_sp slightly
4075     // differently than the real call_VM
4076 
4077     __ enter(); // required for proper stackwalking of RuntimeStub frame
4078 
4079     assert(is_even(framesize/2), "sp not 16-byte aligned");
4080 
4081     // return address and rbp are already in place
4082     __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
4083 
4084     int frame_complete = __ pc() - start;
4085 
4086     // Set up last_Java_sp and last_Java_fp
4087     address the_pc = __ pc();
4088     __ set_last_Java_frame(rsp, rbp, the_pc);
4089     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
4090 
4091     // Call runtime
4092     if (arg1 != noreg) {
4093       assert(arg2 != c_rarg1, "clobbered");
4094       __ movptr(c_rarg1, arg1);
4095     }
4096     if (arg2 != noreg) {
4097       __ movptr(c_rarg2, arg2);
4098     }
4099     __ movptr(c_rarg0, r15_thread);
4100     BLOCK_COMMENT("call runtime_entry");
4101     __ call(RuntimeAddress(runtime_entry));
4102 
4103     // Generate oop map
4104     OopMap* map = new OopMap(framesize, 0);
4105 
4106     oop_maps->add_gc_map(the_pc - start, map);
4107 
4108     __ reset_last_Java_frame(true);
4109 
4110     __ leave(); // required for proper stackwalking of RuntimeStub frame
4111 
4112     // check for pending exceptions
4113 #ifdef ASSERT
4114     Label L;
4115     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()),
4116             (int32_t) NULL_WORD);
4117     __ jcc(Assembler::notEqual, L);
4118     __ should_not_reach_here();
4119     __ bind(L);
4120 #endif // ASSERT
4121     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
4122 
4123 
4124     // codeBlob framesize is in words (not VMRegImpl::slot_size)
4125     RuntimeStub* stub =
4126       RuntimeStub::new_runtime_stub(name,
4127                                     &code,
4128                                     frame_complete,
4129                                     (framesize >> (LogBytesPerWord - LogBytesPerInt)),
4130                                     oop_maps, false);
4131     return stub->entry_point();
4132   }
4133 
4134   void create_control_words() {
4135     // Round to nearest, 53-bit mode, exceptions masked
4136     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
4137     // Round to zero, 53-bit mode, exception mased
4138     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
4139     // Round to nearest, 24-bit mode, exceptions masked
4140     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
4141     // Round to nearest, 64-bit mode, exceptions masked
4142     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;
4143     // Round to nearest, 64-bit mode, exceptions masked
4144     StubRoutines::_mxcsr_std           = 0x1F80;
4145     // Note: the following two constants are 80-bit values
4146     //       layout is critical for correct loading by FPU.
4147     // Bias for strict fp multiply/divide
4148     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
4149     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
4150     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
4151     // Un-Bias for strict fp multiply/divide
4152     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
4153     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
4154     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
4155   }
4156 
4157   // Initialization
4158   void generate_initial() {
4159     // Generates all stubs and initializes the entry points
4160 
4161     // This platform-specific settings are needed by generate_call_stub()
4162     create_control_words();
4163 
4164     // entry points that exist in all platforms Note: This is code
4165     // that could be shared among different platforms - however the
4166     // benefit seems to be smaller than the disadvantage of having a
4167     // much more complicated generator structure. See also comment in
4168     // stubRoutines.hpp.
4169 
4170     StubRoutines::_forward_exception_entry = generate_forward_exception();
4171 
4172     StubRoutines::_call_stub_entry =
4173       generate_call_stub(StubRoutines::_call_stub_return_address);
4174 
4175     // is referenced by megamorphic call
4176     StubRoutines::_catch_exception_entry = generate_catch_exception();
4177 
4178     // atomic calls
4179     StubRoutines::_atomic_xchg_entry         = generate_atomic_xchg();
4180     StubRoutines::_atomic_xchg_ptr_entry     = generate_atomic_xchg_ptr();
4181     StubRoutines::_atomic_cmpxchg_entry      = generate_atomic_cmpxchg();
4182     StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
4183     StubRoutines::_atomic_add_entry          = generate_atomic_add();
4184     StubRoutines::_atomic_add_ptr_entry      = generate_atomic_add_ptr();
4185     StubRoutines::_fence_entry               = generate_orderaccess_fence();
4186 
4187     StubRoutines::_handler_for_unsafe_access_entry =
4188       generate_handler_for_unsafe_access();
4189 
4190     // platform dependent
4191     StubRoutines::x86::_get_previous_fp_entry = generate_get_previous_fp();
4192     StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp();
4193 
4194     StubRoutines::x86::_verify_mxcsr_entry    = generate_verify_mxcsr();
4195 
4196     // Build this early so it's available for the interpreter.
4197     StubRoutines::_throw_StackOverflowError_entry =
4198       generate_throw_exception("StackOverflowError throw_exception",
4199                                CAST_FROM_FN_PTR(address,
4200                                                 SharedRuntime::
4201                                                 throw_StackOverflowError));
4202     if (UseCRC32Intrinsics) {
4203       // set table address before stub generation which use it
4204       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
4205       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
4206     }
4207   }
4208 
4209   void generate_all() {
4210     // Generates all stubs and initializes the entry points
4211 
4212     // These entry points require SharedInfo::stack0 to be set up in
4213     // non-core builds and need to be relocatable, so they each
4214     // fabricate a RuntimeStub internally.
4215     StubRoutines::_throw_AbstractMethodError_entry =
4216       generate_throw_exception("AbstractMethodError throw_exception",
4217                                CAST_FROM_FN_PTR(address,
4218                                                 SharedRuntime::
4219                                                 throw_AbstractMethodError));
4220 
4221     StubRoutines::_throw_IncompatibleClassChangeError_entry =
4222       generate_throw_exception("IncompatibleClassChangeError throw_exception",
4223                                CAST_FROM_FN_PTR(address,
4224                                                 SharedRuntime::
4225                                                 throw_IncompatibleClassChangeError));
4226 
4227     StubRoutines::_throw_NullPointerException_at_call_entry =
4228       generate_throw_exception("NullPointerException at call throw_exception",
4229                                CAST_FROM_FN_PTR(address,
4230                                                 SharedRuntime::
4231                                                 throw_NullPointerException_at_call));
4232 
4233     // entry points that are platform specific
4234     StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
4235     StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
4236     StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
4237     StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
4238 
4239     StubRoutines::x86::_float_sign_mask  = generate_fp_mask("float_sign_mask",  0x7FFFFFFF7FFFFFFF);
4240     StubRoutines::x86::_float_sign_flip  = generate_fp_mask("float_sign_flip",  0x8000000080000000);
4241     StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
4242     StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
4243 
4244     // support for verify_oop (must happen after universe_init)
4245     StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
4246 
4247     // arraycopy stubs used by compilers
4248     generate_arraycopy_stubs();
4249 
4250     generate_math_stubs();
4251 
4252     // don't bother generating these AES intrinsic stubs unless global flag is set
4253     if (UseAESIntrinsics) {
4254       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // needed by the others
4255 
4256       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
4257       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
4258       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
4259       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
4260     }
4261 
4262     // Generate GHASH intrinsics code
4263     if (UseGHASHIntrinsics) {
4264       StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
4265       StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
4266       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
4267     }
4268 
4269     // Safefetch stubs.
4270     generate_safefetch("SafeFetch32", sizeof(int),     &StubRoutines::_safefetch32_entry,
4271                                                        &StubRoutines::_safefetch32_fault_pc,
4272                                                        &StubRoutines::_safefetch32_continuation_pc);
4273     generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
4274                                                        &StubRoutines::_safefetchN_fault_pc,
4275                                                        &StubRoutines::_safefetchN_continuation_pc);
4276 #ifdef COMPILER2
4277     if (UseMultiplyToLenIntrinsic) {
4278       StubRoutines::_multiplyToLen = generate_multiplyToLen();
4279     }
4280     if (UseSquareToLenIntrinsic) {
4281       StubRoutines::_squareToLen = generate_squareToLen();
4282     }
4283     if (UseMulAddIntrinsic) {
4284       StubRoutines::_mulAdd = generate_mulAdd();
4285     }
4286 
4287 #ifndef _WINDOWS
4288     if (UseMontgomeryMultiplyIntrinsic) {
4289       StubRoutines::_montgomeryMultiply
4290         = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_multiply);
4291     }
4292     if (UseMontgomerySquareIntrinsic) {
4293       StubRoutines::_montgomerySquare
4294         = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square);
4295     }
4296 #endif // WINDOWS
4297 #endif // COMPILER2
4298   }
4299 
4300  public:
4301   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
4302     if (all) {
4303       generate_all();
4304     } else {
4305       generate_initial();
4306     }
4307   }
4308 }; // end class declaration
4309 
4310 void StubGenerator_generate(CodeBuffer* code, bool all) {
4311   StubGenerator g(code, all);
4312 }