1 /*
    2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "compiler/compiler_globals.hpp"
   29 #include "compiler/disassembler.hpp"
   30 #include "ci/ciInlineKlass.hpp"
   31 #include "crc32c.h"
   32 #include "gc/shared/barrierSet.hpp"
   33 #include "gc/shared/barrierSetAssembler.hpp"
   34 #include "gc/shared/collectedHeap.inline.hpp"
   35 #include "gc/shared/tlab_globals.hpp"
   36 #include "interpreter/bytecodeHistogram.hpp"
   37 #include "interpreter/interpreter.hpp"
   38 #include "jvm.h"
   39 #include "memory/resourceArea.hpp"
   40 #include "memory/universe.hpp"
   41 #include "oops/accessDecorators.hpp"
   42 #include "oops/compressedKlass.inline.hpp"
   43 #include "oops/compressedOops.inline.hpp"
   44 #include "oops/klass.inline.hpp"
   45 #include "oops/resolvedFieldEntry.hpp"
   46 #include "prims/methodHandles.hpp"
   47 #include "runtime/continuation.hpp"
   48 #include "runtime/interfaceSupport.inline.hpp"
   49 #include "runtime/javaThread.hpp"
   50 #include "runtime/jniHandles.hpp"
   51 #include "runtime/objectMonitor.hpp"
   52 #include "runtime/os.hpp"
   53 #include "runtime/safepoint.hpp"
   54 #include "runtime/safepointMechanism.hpp"
   55 #include "runtime/sharedRuntime.hpp"
   56 #include "runtime/signature_cc.hpp"
   57 #include "runtime/stubRoutines.hpp"
   58 #include "utilities/checkedCast.hpp"
   59 #include "utilities/macros.hpp"
   60 #include "vmreg_x86.inline.hpp"
   61 #ifdef COMPILER2
   62 #include "opto/output.hpp"
   63 #endif
   64 
   65 #ifdef PRODUCT
   66 #define BLOCK_COMMENT(str) /* nothing */
   67 #define STOP(error) stop(error)
   68 #else
   69 #define BLOCK_COMMENT(str) block_comment(str)
   70 #define STOP(error) block_comment(error); stop(error)
   71 #endif
   72 
   73 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   74 
   75 #ifdef ASSERT
   76 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   77 #endif
   78 
   79 static const Assembler::Condition reverse[] = {
   80     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   81     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   82     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   83     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
   84     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
   85     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
   86     Assembler::above          /* belowEqual    = 0x6 */ ,
   87     Assembler::belowEqual     /* above         = 0x7 */ ,
   88     Assembler::positive       /* negative      = 0x8 */ ,
   89     Assembler::negative       /* positive      = 0x9 */ ,
   90     Assembler::noParity       /* parity        = 0xa */ ,
   91     Assembler::parity         /* noParity      = 0xb */ ,
   92     Assembler::greaterEqual   /* less          = 0xc */ ,
   93     Assembler::less           /* greaterEqual  = 0xd */ ,
   94     Assembler::greater        /* lessEqual     = 0xe */ ,
   95     Assembler::lessEqual      /* greater       = 0xf, */
   96 
   97 };
   98 
   99 
  100 // Implementation of MacroAssembler
  101 
  102 // First all the versions that have distinct versions depending on 32/64 bit
  103 // Unless the difference is trivial (1 line or so).
  104 
  105 #ifndef _LP64
  106 
  107 // 32bit versions
  108 
  109 Address MacroAssembler::as_Address(AddressLiteral adr) {
  110   return Address(adr.target(), adr.rspec());
  111 }
  112 
  113 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  114   assert(rscratch == noreg, "");
  115   return Address::make_array(adr);
  116 }
  117 
  118 void MacroAssembler::call_VM_leaf_base(address entry_point,
  119                                        int number_of_arguments) {
  120   call(RuntimeAddress(entry_point));
  121   increment(rsp, number_of_arguments * wordSize);
  122 }
  123 
  124 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
  125   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  126 }
  127 
  128 
  129 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
  130   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  131 }
  132 
  133 void MacroAssembler::cmpoop(Address src1, jobject obj) {
  134   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  135 }
  136 
  137 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) {
  138   assert(rscratch == noreg, "redundant");
  139   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  140 }
  141 
  142 void MacroAssembler::extend_sign(Register hi, Register lo) {
  143   // According to Intel Doc. AP-526, "Integer Divide", p.18.
  144   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
  145     cdql();
  146   } else {
  147     movl(hi, lo);
  148     sarl(hi, 31);
  149   }
  150 }
  151 
  152 void MacroAssembler::jC2(Register tmp, Label& L) {
  153   // set parity bit if FPU flag C2 is set (via rax)
  154   save_rax(tmp);
  155   fwait(); fnstsw_ax();
  156   sahf();
  157   restore_rax(tmp);
  158   // branch
  159   jcc(Assembler::parity, L);
  160 }
  161 
  162 void MacroAssembler::jnC2(Register tmp, Label& L) {
  163   // set parity bit if FPU flag C2 is set (via rax)
  164   save_rax(tmp);
  165   fwait(); fnstsw_ax();
  166   sahf();
  167   restore_rax(tmp);
  168   // branch
  169   jcc(Assembler::noParity, L);
  170 }
  171 
  172 // 32bit can do a case table jump in one instruction but we no longer allow the base
  173 // to be installed in the Address class
  174 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  175   assert(rscratch == noreg, "not needed");
  176   jmp(as_Address(entry, noreg));
  177 }
  178 
  179 // Note: y_lo will be destroyed
  180 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  181   // Long compare for Java (semantics as described in JVM spec.)
  182   Label high, low, done;
  183 
  184   cmpl(x_hi, y_hi);
  185   jcc(Assembler::less, low);
  186   jcc(Assembler::greater, high);
  187   // x_hi is the return register
  188   xorl(x_hi, x_hi);
  189   cmpl(x_lo, y_lo);
  190   jcc(Assembler::below, low);
  191   jcc(Assembler::equal, done);
  192 
  193   bind(high);
  194   xorl(x_hi, x_hi);
  195   increment(x_hi);
  196   jmp(done);
  197 
  198   bind(low);
  199   xorl(x_hi, x_hi);
  200   decrementl(x_hi);
  201 
  202   bind(done);
  203 }
  204 
  205 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  206   mov_literal32(dst, (int32_t)src.target(), src.rspec());
  207 }
  208 
  209 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  210   assert(rscratch == noreg, "not needed");
  211 
  212   // leal(dst, as_Address(adr));
  213   // see note in movl as to why we must use a move
  214   mov_literal32(dst, (int32_t)adr.target(), adr.rspec());
  215 }
  216 
  217 void MacroAssembler::leave() {
  218   mov(rsp, rbp);
  219   pop(rbp);
  220 }
  221 
  222 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
  223   // Multiplication of two Java long values stored on the stack
  224   // as illustrated below. Result is in rdx:rax.
  225   //
  226   // rsp ---> [  ??  ] \               \
  227   //            ....    | y_rsp_offset  |
  228   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
  229   //          [ y_hi ]                  | (in bytes)
  230   //            ....                    |
  231   //          [ x_lo ]                 /
  232   //          [ x_hi ]
  233   //            ....
  234   //
  235   // Basic idea: lo(result) = lo(x_lo * y_lo)
  236   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
  237   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
  238   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
  239   Label quick;
  240   // load x_hi, y_hi and check if quick
  241   // multiplication is possible
  242   movl(rbx, x_hi);
  243   movl(rcx, y_hi);
  244   movl(rax, rbx);
  245   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
  246   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
  247   // do full multiplication
  248   // 1st step
  249   mull(y_lo);                                    // x_hi * y_lo
  250   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
  251   // 2nd step
  252   movl(rax, x_lo);
  253   mull(rcx);                                     // x_lo * y_hi
  254   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
  255   // 3rd step
  256   bind(quick);                                   // note: rbx, = 0 if quick multiply!
  257   movl(rax, x_lo);
  258   mull(y_lo);                                    // x_lo * y_lo
  259   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
  260 }
  261 
  262 void MacroAssembler::lneg(Register hi, Register lo) {
  263   negl(lo);
  264   adcl(hi, 0);
  265   negl(hi);
  266 }
  267 
  268 void MacroAssembler::lshl(Register hi, Register lo) {
  269   // Java shift left long support (semantics as described in JVM spec., p.305)
  270   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
  271   // shift value is in rcx !
  272   assert(hi != rcx, "must not use rcx");
  273   assert(lo != rcx, "must not use rcx");
  274   const Register s = rcx;                        // shift count
  275   const int      n = BitsPerWord;
  276   Label L;
  277   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  278   cmpl(s, n);                                    // if (s < n)
  279   jcc(Assembler::less, L);                       // else (s >= n)
  280   movl(hi, lo);                                  // x := x << n
  281   xorl(lo, lo);
  282   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  283   bind(L);                                       // s (mod n) < n
  284   shldl(hi, lo);                                 // x := x << s
  285   shll(lo);
  286 }
  287 
  288 
  289 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
  290   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
  291   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
  292   assert(hi != rcx, "must not use rcx");
  293   assert(lo != rcx, "must not use rcx");
  294   const Register s = rcx;                        // shift count
  295   const int      n = BitsPerWord;
  296   Label L;
  297   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  298   cmpl(s, n);                                    // if (s < n)
  299   jcc(Assembler::less, L);                       // else (s >= n)
  300   movl(lo, hi);                                  // x := x >> n
  301   if (sign_extension) sarl(hi, 31);
  302   else                xorl(hi, hi);
  303   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  304   bind(L);                                       // s (mod n) < n
  305   shrdl(lo, hi);                                 // x := x >> s
  306   if (sign_extension) sarl(hi);
  307   else                shrl(hi);
  308 }
  309 
  310 void MacroAssembler::movoop(Register dst, jobject obj) {
  311   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  312 }
  313 
  314 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  315   assert(rscratch == noreg, "redundant");
  316   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  317 }
  318 
  319 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  320   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  321 }
  322 
  323 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  324   assert(rscratch == noreg, "redundant");
  325   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  326 }
  327 
  328 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  329   if (src.is_lval()) {
  330     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
  331   } else {
  332     movl(dst, as_Address(src));
  333   }
  334 }
  335 
  336 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  337   assert(rscratch == noreg, "redundant");
  338   movl(as_Address(dst, noreg), src);
  339 }
  340 
  341 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  342   movl(dst, as_Address(src, noreg));
  343 }
  344 
  345 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  346   assert(rscratch == noreg, "redundant");
  347   movl(dst, src);
  348 }
  349 
  350 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  351   assert(rscratch == noreg, "redundant");
  352   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
  353 }
  354 
  355 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  356   assert(rscratch == noreg, "redundant");
  357   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
  358 }
  359 
  360 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  361   assert(rscratch == noreg, "redundant");
  362   if (src.is_lval()) {
  363     push_literal32((int32_t)src.target(), src.rspec());
  364   } else {
  365     pushl(as_Address(src));
  366   }
  367 }
  368 
  369 static void pass_arg0(MacroAssembler* masm, Register arg) {
  370   masm->push(arg);
  371 }
  372 
  373 static void pass_arg1(MacroAssembler* masm, Register arg) {
  374   masm->push(arg);
  375 }
  376 
  377 static void pass_arg2(MacroAssembler* masm, Register arg) {
  378   masm->push(arg);
  379 }
  380 
  381 static void pass_arg3(MacroAssembler* masm, Register arg) {
  382   masm->push(arg);
  383 }
  384 
  385 #ifndef PRODUCT
  386 extern "C" void findpc(intptr_t x);
  387 #endif
  388 
  389 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
  390   // In order to get locks to work, we need to fake a in_VM state
  391   JavaThread* thread = JavaThread::current();
  392   JavaThreadState saved_state = thread->thread_state();
  393   thread->set_thread_state(_thread_in_vm);
  394   if (ShowMessageBoxOnError) {
  395     JavaThread* thread = JavaThread::current();
  396     JavaThreadState saved_state = thread->thread_state();
  397     thread->set_thread_state(_thread_in_vm);
  398     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  399       ttyLocker ttyl;
  400       BytecodeCounter::print();
  401     }
  402     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  403     // This is the value of eip which points to where verify_oop will return.
  404     if (os::message_box(msg, "Execution stopped, print registers?")) {
  405       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
  406       BREAKPOINT;
  407     }
  408   }
  409   fatal("DEBUG MESSAGE: %s", msg);
  410 }
  411 
  412 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
  413   ttyLocker ttyl;
  414   DebuggingContext debugging{};
  415   tty->print_cr("eip = 0x%08x", eip);
  416 #ifndef PRODUCT
  417   if ((WizardMode || Verbose) && PrintMiscellaneous) {
  418     tty->cr();
  419     findpc(eip);
  420     tty->cr();
  421   }
  422 #endif
  423 #define PRINT_REG(rax) \
  424   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
  425   PRINT_REG(rax);
  426   PRINT_REG(rbx);
  427   PRINT_REG(rcx);
  428   PRINT_REG(rdx);
  429   PRINT_REG(rdi);
  430   PRINT_REG(rsi);
  431   PRINT_REG(rbp);
  432   PRINT_REG(rsp);
  433 #undef PRINT_REG
  434   // Print some words near top of staack.
  435   int* dump_sp = (int*) rsp;
  436   for (int col1 = 0; col1 < 8; col1++) {
  437     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  438     os::print_location(tty, *dump_sp++);
  439   }
  440   for (int row = 0; row < 16; row++) {
  441     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  442     for (int col = 0; col < 8; col++) {
  443       tty->print(" 0x%08x", *dump_sp++);
  444     }
  445     tty->cr();
  446   }
  447   // Print some instructions around pc:
  448   Disassembler::decode((address)eip-64, (address)eip);
  449   tty->print_cr("--------");
  450   Disassembler::decode((address)eip, (address)eip+32);
  451 }
  452 
  453 void MacroAssembler::stop(const char* msg) {
  454   // push address of message
  455   ExternalAddress message((address)msg);
  456   pushptr(message.addr(), noreg);
  457   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  458   pusha();                                            // push registers
  459   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
  460   hlt();
  461 }
  462 
  463 void MacroAssembler::warn(const char* msg) {
  464   push_CPU_state();
  465 
  466   // push address of message
  467   ExternalAddress message((address)msg);
  468   pushptr(message.addr(), noreg);
  469 
  470   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  471   addl(rsp, wordSize);       // discard argument
  472   pop_CPU_state();
  473 }
  474 
  475 void MacroAssembler::print_state() {
  476   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  477   pusha();                                            // push registers
  478 
  479   push_CPU_state();
  480   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
  481   pop_CPU_state();
  482 
  483   popa();
  484   addl(rsp, wordSize);
  485 }
  486 
  487 #else // _LP64
  488 
  489 // 64 bit versions
  490 
  491 Address MacroAssembler::as_Address(AddressLiteral adr) {
  492   // amd64 always does this as a pc-rel
  493   // we can be absolute or disp based on the instruction type
  494   // jmp/call are displacements others are absolute
  495   assert(!adr.is_lval(), "must be rval");
  496   assert(reachable(adr), "must be");
  497   return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc());
  498 
  499 }
  500 
  501 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  502   AddressLiteral base = adr.base();
  503   lea(rscratch, base);
  504   Address index = adr.index();
  505   assert(index._disp == 0, "must not have disp"); // maybe it can?
  506   Address array(rscratch, index._index, index._scale, index._disp);
  507   return array;
  508 }
  509 
  510 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
  511   Label L, E;
  512 
  513 #ifdef _WIN64
  514   // Windows always allocates space for it's register args
  515   assert(num_args <= 4, "only register arguments supported");
  516   subq(rsp,  frame::arg_reg_save_area_bytes);
  517 #endif
  518 
  519   // Align stack if necessary
  520   testl(rsp, 15);
  521   jcc(Assembler::zero, L);
  522 
  523   subq(rsp, 8);
  524   call(RuntimeAddress(entry_point));
  525   addq(rsp, 8);
  526   jmp(E);
  527 
  528   bind(L);
  529   call(RuntimeAddress(entry_point));
  530 
  531   bind(E);
  532 
  533 #ifdef _WIN64
  534   // restore stack pointer
  535   addq(rsp, frame::arg_reg_save_area_bytes);
  536 #endif
  537 
  538 }
  539 
  540 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) {
  541   assert(!src2.is_lval(), "should use cmpptr");
  542   assert(rscratch != noreg || always_reachable(src2), "missing");
  543 
  544   if (reachable(src2)) {
  545     cmpq(src1, as_Address(src2));
  546   } else {
  547     lea(rscratch, src2);
  548     Assembler::cmpq(src1, Address(rscratch, 0));
  549   }
  550 }
  551 
  552 int MacroAssembler::corrected_idivq(Register reg) {
  553   // Full implementation of Java ldiv and lrem; checks for special
  554   // case as described in JVM spec., p.243 & p.271.  The function
  555   // returns the (pc) offset of the idivl instruction - may be needed
  556   // for implicit exceptions.
  557   //
  558   //         normal case                           special case
  559   //
  560   // input : rax: dividend                         min_long
  561   //         reg: divisor   (may not be eax/edx)   -1
  562   //
  563   // output: rax: quotient  (= rax idiv reg)       min_long
  564   //         rdx: remainder (= rax irem reg)       0
  565   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
  566   static const int64_t min_long = 0x8000000000000000;
  567   Label normal_case, special_case;
  568 
  569   // check for special case
  570   cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/);
  571   jcc(Assembler::notEqual, normal_case);
  572   xorl(rdx, rdx); // prepare rdx for possible special case (where
  573                   // remainder = 0)
  574   cmpq(reg, -1);
  575   jcc(Assembler::equal, special_case);
  576 
  577   // handle normal case
  578   bind(normal_case);
  579   cdqq();
  580   int idivq_offset = offset();
  581   idivq(reg);
  582 
  583   // normal and special case exit
  584   bind(special_case);
  585 
  586   return idivq_offset;
  587 }
  588 
  589 void MacroAssembler::decrementq(Register reg, int value) {
  590   if (value == min_jint) { subq(reg, value); return; }
  591   if (value <  0) { incrementq(reg, -value); return; }
  592   if (value == 0) {                        ; return; }
  593   if (value == 1 && UseIncDec) { decq(reg) ; return; }
  594   /* else */      { subq(reg, value)       ; return; }
  595 }
  596 
  597 void MacroAssembler::decrementq(Address dst, int value) {
  598   if (value == min_jint) { subq(dst, value); return; }
  599   if (value <  0) { incrementq(dst, -value); return; }
  600   if (value == 0) {                        ; return; }
  601   if (value == 1 && UseIncDec) { decq(dst) ; return; }
  602   /* else */      { subq(dst, value)       ; return; }
  603 }
  604 
  605 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) {
  606   assert(rscratch != noreg || always_reachable(dst), "missing");
  607 
  608   if (reachable(dst)) {
  609     incrementq(as_Address(dst));
  610   } else {
  611     lea(rscratch, dst);
  612     incrementq(Address(rscratch, 0));
  613   }
  614 }
  615 
  616 void MacroAssembler::incrementq(Register reg, int value) {
  617   if (value == min_jint) { addq(reg, value); return; }
  618   if (value <  0) { decrementq(reg, -value); return; }
  619   if (value == 0) {                        ; return; }
  620   if (value == 1 && UseIncDec) { incq(reg) ; return; }
  621   /* else */      { addq(reg, value)       ; return; }
  622 }
  623 
  624 void MacroAssembler::incrementq(Address dst, int value) {
  625   if (value == min_jint) { addq(dst, value); return; }
  626   if (value <  0) { decrementq(dst, -value); return; }
  627   if (value == 0) {                        ; return; }
  628   if (value == 1 && UseIncDec) { incq(dst) ; return; }
  629   /* else */      { addq(dst, value)       ; return; }
  630 }
  631 
  632 // 32bit can do a case table jump in one instruction but we no longer allow the base
  633 // to be installed in the Address class
  634 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  635   lea(rscratch, entry.base());
  636   Address dispatch = entry.index();
  637   assert(dispatch._base == noreg, "must be");
  638   dispatch._base = rscratch;
  639   jmp(dispatch);
  640 }
  641 
  642 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  643   ShouldNotReachHere(); // 64bit doesn't use two regs
  644   cmpq(x_lo, y_lo);
  645 }
  646 
  647 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  648   mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  649 }
  650 
  651 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  652   lea(rscratch, adr);
  653   movptr(dst, rscratch);
  654 }
  655 
  656 void MacroAssembler::leave() {
  657   // %%% is this really better? Why not on 32bit too?
  658   emit_int8((unsigned char)0xC9); // LEAVE
  659 }
  660 
  661 void MacroAssembler::lneg(Register hi, Register lo) {
  662   ShouldNotReachHere(); // 64bit doesn't use two regs
  663   negq(lo);
  664 }
  665 
  666 void MacroAssembler::movoop(Register dst, jobject obj) {
  667   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  668 }
  669 
  670 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  671   mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  672   movq(dst, rscratch);
  673 }
  674 
  675 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  676   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  677 }
  678 
  679 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  680   mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  681   movq(dst, rscratch);
  682 }
  683 
  684 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  685   if (src.is_lval()) {
  686     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  687   } else {
  688     if (reachable(src)) {
  689       movq(dst, as_Address(src));
  690     } else {
  691       lea(dst, src);
  692       movq(dst, Address(dst, 0));
  693     }
  694   }
  695 }
  696 
  697 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  698   movq(as_Address(dst, rscratch), src);
  699 }
  700 
  701 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  702   movq(dst, as_Address(src, dst /*rscratch*/));
  703 }
  704 
  705 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
  706 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  707   if (is_simm32(src)) {
  708     movptr(dst, checked_cast<int32_t>(src));
  709   } else {
  710     mov64(rscratch, src);
  711     movq(dst, rscratch);
  712   }
  713 }
  714 
  715 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  716   movoop(rscratch, obj);
  717   push(rscratch);
  718 }
  719 
  720 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  721   mov_metadata(rscratch, obj);
  722   push(rscratch);
  723 }
  724 
  725 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  726   lea(rscratch, src);
  727   if (src.is_lval()) {
  728     push(rscratch);
  729   } else {
  730     pushq(Address(rscratch, 0));
  731   }
  732 }
  733 
  734 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
  735   reset_last_Java_frame(r15_thread, clear_fp);
  736 }
  737 
  738 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
  739                                          Register last_java_fp,
  740                                          address  last_java_pc,
  741                                          Register rscratch) {
  742   set_last_Java_frame(r15_thread, last_java_sp, last_java_fp, last_java_pc, rscratch);
  743 }
  744 
  745 static void pass_arg0(MacroAssembler* masm, Register arg) {
  746   if (c_rarg0 != arg ) {
  747     masm->mov(c_rarg0, arg);
  748   }
  749 }
  750 
  751 static void pass_arg1(MacroAssembler* masm, Register arg) {
  752   if (c_rarg1 != arg ) {
  753     masm->mov(c_rarg1, arg);
  754   }
  755 }
  756 
  757 static void pass_arg2(MacroAssembler* masm, Register arg) {
  758   if (c_rarg2 != arg ) {
  759     masm->mov(c_rarg2, arg);
  760   }
  761 }
  762 
  763 static void pass_arg3(MacroAssembler* masm, Register arg) {
  764   if (c_rarg3 != arg ) {
  765     masm->mov(c_rarg3, arg);
  766   }
  767 }
  768 
  769 void MacroAssembler::stop(const char* msg) {
  770   if (ShowMessageBoxOnError) {
  771     address rip = pc();
  772     pusha(); // get regs on stack
  773     lea(c_rarg1, InternalAddress(rip));
  774     movq(c_rarg2, rsp); // pass pointer to regs array
  775   }
  776   lea(c_rarg0, ExternalAddress((address) msg));
  777   andq(rsp, -16); // align stack as required by ABI
  778   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
  779   hlt();
  780 }
  781 
  782 void MacroAssembler::warn(const char* msg) {
  783   push(rbp);
  784   movq(rbp, rsp);
  785   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  786   push_CPU_state();   // keeps alignment at 16 bytes
  787 
  788   lea(c_rarg0, ExternalAddress((address) msg));
  789   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  790 
  791   pop_CPU_state();
  792   mov(rsp, rbp);
  793   pop(rbp);
  794 }
  795 
  796 void MacroAssembler::print_state() {
  797   address rip = pc();
  798   pusha();            // get regs on stack
  799   push(rbp);
  800   movq(rbp, rsp);
  801   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  802   push_CPU_state();   // keeps alignment at 16 bytes
  803 
  804   lea(c_rarg0, InternalAddress(rip));
  805   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
  806   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
  807 
  808   pop_CPU_state();
  809   mov(rsp, rbp);
  810   pop(rbp);
  811   popa();
  812 }
  813 
  814 #ifndef PRODUCT
  815 extern "C" void findpc(intptr_t x);
  816 #endif
  817 
  818 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
  819   // In order to get locks to work, we need to fake a in_VM state
  820   if (ShowMessageBoxOnError) {
  821     JavaThread* thread = JavaThread::current();
  822     JavaThreadState saved_state = thread->thread_state();
  823     thread->set_thread_state(_thread_in_vm);
  824 #ifndef PRODUCT
  825     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  826       ttyLocker ttyl;
  827       BytecodeCounter::print();
  828     }
  829 #endif
  830     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  831     // XXX correct this offset for amd64
  832     // This is the value of eip which points to where verify_oop will return.
  833     if (os::message_box(msg, "Execution stopped, print registers?")) {
  834       print_state64(pc, regs);
  835       BREAKPOINT;
  836     }
  837   }
  838   fatal("DEBUG MESSAGE: %s", msg);
  839 }
  840 
  841 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
  842   ttyLocker ttyl;
  843   DebuggingContext debugging{};
  844   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
  845 #ifndef PRODUCT
  846   tty->cr();
  847   findpc(pc);
  848   tty->cr();
  849 #endif
  850 #define PRINT_REG(rax, value) \
  851   { tty->print("%s = ", #rax); os::print_location(tty, value); }
  852   PRINT_REG(rax, regs[15]);
  853   PRINT_REG(rbx, regs[12]);
  854   PRINT_REG(rcx, regs[14]);
  855   PRINT_REG(rdx, regs[13]);
  856   PRINT_REG(rdi, regs[8]);
  857   PRINT_REG(rsi, regs[9]);
  858   PRINT_REG(rbp, regs[10]);
  859   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
  860   PRINT_REG(rsp, (intptr_t)(&regs[16]));
  861   PRINT_REG(r8 , regs[7]);
  862   PRINT_REG(r9 , regs[6]);
  863   PRINT_REG(r10, regs[5]);
  864   PRINT_REG(r11, regs[4]);
  865   PRINT_REG(r12, regs[3]);
  866   PRINT_REG(r13, regs[2]);
  867   PRINT_REG(r14, regs[1]);
  868   PRINT_REG(r15, regs[0]);
  869 #undef PRINT_REG
  870   // Print some words near the top of the stack.
  871   int64_t* rsp = &regs[16];
  872   int64_t* dump_sp = rsp;
  873   for (int col1 = 0; col1 < 8; col1++) {
  874     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  875     os::print_location(tty, *dump_sp++);
  876   }
  877   for (int row = 0; row < 25; row++) {
  878     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  879     for (int col = 0; col < 4; col++) {
  880       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
  881     }
  882     tty->cr();
  883   }
  884   // Print some instructions around pc:
  885   Disassembler::decode((address)pc-64, (address)pc);
  886   tty->print_cr("--------");
  887   Disassembler::decode((address)pc, (address)pc+32);
  888 }
  889 
  890 // The java_calling_convention describes stack locations as ideal slots on
  891 // a frame with no abi restrictions. Since we must observe abi restrictions
  892 // (like the placement of the register window) the slots must be biased by
  893 // the following value.
  894 static int reg2offset_in(VMReg r) {
  895   // Account for saved rbp and return address
  896   // This should really be in_preserve_stack_slots
  897   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
  898 }
  899 
  900 static int reg2offset_out(VMReg r) {
  901   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
  902 }
  903 
  904 // A long move
  905 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  906 
  907   // The calling conventions assures us that each VMregpair is either
  908   // all really one physical register or adjacent stack slots.
  909 
  910   if (src.is_single_phys_reg() ) {
  911     if (dst.is_single_phys_reg()) {
  912       if (dst.first() != src.first()) {
  913         mov(dst.first()->as_Register(), src.first()->as_Register());
  914       }
  915     } else {
  916       assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)",
  917              src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name());
  918       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  919     }
  920   } else if (dst.is_single_phys_reg()) {
  921     assert(src.is_single_reg(),  "not a stack pair");
  922     movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  923   } else {
  924     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  925     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  926     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  927   }
  928 }
  929 
  930 // A double move
  931 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  932 
  933   // The calling conventions assures us that each VMregpair is either
  934   // all really one physical register or adjacent stack slots.
  935 
  936   if (src.is_single_phys_reg() ) {
  937     if (dst.is_single_phys_reg()) {
  938       // In theory these overlap but the ordering is such that this is likely a nop
  939       if ( src.first() != dst.first()) {
  940         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
  941       }
  942     } else {
  943       assert(dst.is_single_reg(), "not a stack pair");
  944       movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  945     }
  946   } else if (dst.is_single_phys_reg()) {
  947     assert(src.is_single_reg(),  "not a stack pair");
  948     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  949   } else {
  950     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  951     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  952     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  953   }
  954 }
  955 
  956 
  957 // A float arg may have to do float reg int reg conversion
  958 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  959   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
  960 
  961   // The calling conventions assures us that each VMregpair is either
  962   // all really one physical register or adjacent stack slots.
  963 
  964   if (src.first()->is_stack()) {
  965     if (dst.first()->is_stack()) {
  966       movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  967       movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  968     } else {
  969       // stack to reg
  970       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  971       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  972     }
  973   } else if (dst.first()->is_stack()) {
  974     // reg to stack
  975     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  976     movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  977   } else {
  978     // reg to reg
  979     // In theory these overlap but the ordering is such that this is likely a nop
  980     if ( src.first() != dst.first()) {
  981       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
  982     }
  983   }
  984 }
  985 
  986 // On 64 bit we will store integer like items to the stack as
  987 // 64 bits items (x86_32/64 abi) even though java would only store
  988 // 32bits for a parameter. On 32bit it will simply be 32 bits
  989 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
  990 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  991   if (src.first()->is_stack()) {
  992     if (dst.first()->is_stack()) {
  993       // stack to stack
  994       movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  995       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  996     } else {
  997       // stack to reg
  998       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  999     }
 1000   } else if (dst.first()->is_stack()) {
 1001     // reg to stack
 1002     // Do we really have to sign extend???
 1003     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
 1004     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
 1005   } else {
 1006     // Do we really have to sign extend???
 1007     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
 1008     if (dst.first() != src.first()) {
 1009       movq(dst.first()->as_Register(), src.first()->as_Register());
 1010     }
 1011   }
 1012 }
 1013 
 1014 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
 1015   if (src.first()->is_stack()) {
 1016     if (dst.first()->is_stack()) {
 1017       // stack to stack
 1018       movq(rax, Address(rbp, reg2offset_in(src.first())));
 1019       movq(Address(rsp, reg2offset_out(dst.first())), rax);
 1020     } else {
 1021       // stack to reg
 1022       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 1023     }
 1024   } else if (dst.first()->is_stack()) {
 1025     // reg to stack
 1026     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 1027   } else {
 1028     if (dst.first() != src.first()) {
 1029       movq(dst.first()->as_Register(), src.first()->as_Register());
 1030     }
 1031   }
 1032 }
 1033 
 1034 // An oop arg. Must pass a handle not the oop itself
 1035 void MacroAssembler::object_move(OopMap* map,
 1036                         int oop_handle_offset,
 1037                         int framesize_in_slots,
 1038                         VMRegPair src,
 1039                         VMRegPair dst,
 1040                         bool is_receiver,
 1041                         int* receiver_offset) {
 1042 
 1043   // must pass a handle. First figure out the location we use as a handle
 1044 
 1045   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
 1046 
 1047   // See if oop is null if it is we need no handle
 1048 
 1049   if (src.first()->is_stack()) {
 1050 
 1051     // Oop is already on the stack as an argument
 1052     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 1053     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 1054     if (is_receiver) {
 1055       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 1056     }
 1057 
 1058     cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD);
 1059     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
 1060     // conditionally move a null
 1061     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
 1062   } else {
 1063 
 1064     // Oop is in a register we must store it to the space we reserve
 1065     // on the stack for oop_handles and pass a handle if oop is non-null
 1066 
 1067     const Register rOop = src.first()->as_Register();
 1068     int oop_slot;
 1069     if (rOop == j_rarg0)
 1070       oop_slot = 0;
 1071     else if (rOop == j_rarg1)
 1072       oop_slot = 1;
 1073     else if (rOop == j_rarg2)
 1074       oop_slot = 2;
 1075     else if (rOop == j_rarg3)
 1076       oop_slot = 3;
 1077     else if (rOop == j_rarg4)
 1078       oop_slot = 4;
 1079     else {
 1080       assert(rOop == j_rarg5, "wrong register");
 1081       oop_slot = 5;
 1082     }
 1083 
 1084     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 1085     int offset = oop_slot*VMRegImpl::stack_slot_size;
 1086 
 1087     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 1088     // Store oop in handle area, may be null
 1089     movptr(Address(rsp, offset), rOop);
 1090     if (is_receiver) {
 1091       *receiver_offset = offset;
 1092     }
 1093 
 1094     cmpptr(rOop, NULL_WORD);
 1095     lea(rHandle, Address(rsp, offset));
 1096     // conditionally move a null from the handle area where it was just stored
 1097     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
 1098   }
 1099 
 1100   // If arg is on the stack then place it otherwise it is already in correct reg.
 1101   if (dst.first()->is_stack()) {
 1102     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
 1103   }
 1104 }
 1105 
 1106 #endif // _LP64
 1107 
 1108 // Now versions that are common to 32/64 bit
 1109 
 1110 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 1111   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 1112 }
 1113 
 1114 void MacroAssembler::addptr(Register dst, Register src) {
 1115   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1116 }
 1117 
 1118 void MacroAssembler::addptr(Address dst, Register src) {
 1119   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1120 }
 1121 
 1122 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1123   assert(rscratch != noreg || always_reachable(src), "missing");
 1124 
 1125   if (reachable(src)) {
 1126     Assembler::addsd(dst, as_Address(src));
 1127   } else {
 1128     lea(rscratch, src);
 1129     Assembler::addsd(dst, Address(rscratch, 0));
 1130   }
 1131 }
 1132 
 1133 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1134   assert(rscratch != noreg || always_reachable(src), "missing");
 1135 
 1136   if (reachable(src)) {
 1137     addss(dst, as_Address(src));
 1138   } else {
 1139     lea(rscratch, src);
 1140     addss(dst, Address(rscratch, 0));
 1141   }
 1142 }
 1143 
 1144 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1145   assert(rscratch != noreg || always_reachable(src), "missing");
 1146 
 1147   if (reachable(src)) {
 1148     Assembler::addpd(dst, as_Address(src));
 1149   } else {
 1150     lea(rscratch, src);
 1151     Assembler::addpd(dst, Address(rscratch, 0));
 1152   }
 1153 }
 1154 
 1155 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
 1156 // Stub code is generated once and never copied.
 1157 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
 1158 void MacroAssembler::align64() {
 1159   align(64, (unsigned long long) pc());
 1160 }
 1161 
 1162 void MacroAssembler::align32() {
 1163   align(32, (unsigned long long) pc());
 1164 }
 1165 
 1166 void MacroAssembler::align(int modulus) {
 1167   // 8273459: Ensure alignment is possible with current segment alignment
 1168   assert(modulus <= CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
 1169   align(modulus, offset());
 1170 }
 1171 
 1172 void MacroAssembler::align(int modulus, int target) {
 1173   if (target % modulus != 0) {
 1174     nop(modulus - (target % modulus));
 1175   }
 1176 }
 1177 
 1178 void MacroAssembler::push_f(XMMRegister r) {
 1179   subptr(rsp, wordSize);
 1180   movflt(Address(rsp, 0), r);
 1181 }
 1182 
 1183 void MacroAssembler::pop_f(XMMRegister r) {
 1184   movflt(r, Address(rsp, 0));
 1185   addptr(rsp, wordSize);
 1186 }
 1187 
 1188 void MacroAssembler::push_d(XMMRegister r) {
 1189   subptr(rsp, 2 * wordSize);
 1190   movdbl(Address(rsp, 0), r);
 1191 }
 1192 
 1193 void MacroAssembler::pop_d(XMMRegister r) {
 1194   movdbl(r, Address(rsp, 0));
 1195   addptr(rsp, 2 * Interpreter::stackElementSize);
 1196 }
 1197 
 1198 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1199   // Used in sign-masking with aligned address.
 1200   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1201   assert(rscratch != noreg || always_reachable(src), "missing");
 1202 
 1203   if (reachable(src)) {
 1204     Assembler::andpd(dst, as_Address(src));
 1205   } else {
 1206     lea(rscratch, src);
 1207     Assembler::andpd(dst, Address(rscratch, 0));
 1208   }
 1209 }
 1210 
 1211 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1212   // Used in sign-masking with aligned address.
 1213   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1214   assert(rscratch != noreg || always_reachable(src), "missing");
 1215 
 1216   if (reachable(src)) {
 1217     Assembler::andps(dst, as_Address(src));
 1218   } else {
 1219     lea(rscratch, src);
 1220     Assembler::andps(dst, Address(rscratch, 0));
 1221   }
 1222 }
 1223 
 1224 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 1225   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 1226 }
 1227 
 1228 #ifdef _LP64
 1229 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) {
 1230   assert(rscratch != noreg || always_reachable(src), "missing");
 1231 
 1232   if (reachable(src)) {
 1233     andq(dst, as_Address(src));
 1234   } else {
 1235     lea(rscratch, src);
 1236     andq(dst, Address(rscratch, 0));
 1237   }
 1238 }
 1239 #endif
 1240 
 1241 void MacroAssembler::atomic_incl(Address counter_addr) {
 1242   lock();
 1243   incrementl(counter_addr);
 1244 }
 1245 
 1246 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) {
 1247   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1248 
 1249   if (reachable(counter_addr)) {
 1250     atomic_incl(as_Address(counter_addr));
 1251   } else {
 1252     lea(rscratch, counter_addr);
 1253     atomic_incl(Address(rscratch, 0));
 1254   }
 1255 }
 1256 
 1257 #ifdef _LP64
 1258 void MacroAssembler::atomic_incq(Address counter_addr) {
 1259   lock();
 1260   incrementq(counter_addr);
 1261 }
 1262 
 1263 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) {
 1264   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1265 
 1266   if (reachable(counter_addr)) {
 1267     atomic_incq(as_Address(counter_addr));
 1268   } else {
 1269     lea(rscratch, counter_addr);
 1270     atomic_incq(Address(rscratch, 0));
 1271   }
 1272 }
 1273 #endif
 1274 
 1275 // Writes to stack successive pages until offset reached to check for
 1276 // stack overflow + shadow pages.  This clobbers tmp.
 1277 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 1278   movptr(tmp, rsp);
 1279   // Bang stack for total size given plus shadow page size.
 1280   // Bang one page at a time because large size can bang beyond yellow and
 1281   // red zones.
 1282   Label loop;
 1283   bind(loop);
 1284   movl(Address(tmp, (-(int)os::vm_page_size())), size );
 1285   subptr(tmp, (int)os::vm_page_size());
 1286   subl(size, (int)os::vm_page_size());
 1287   jcc(Assembler::greater, loop);
 1288 
 1289   // Bang down shadow pages too.
 1290   // At this point, (tmp-0) is the last address touched, so don't
 1291   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
 1292   // was post-decremented.)  Skip this address by starting at i=1, and
 1293   // touch a few more pages below.  N.B.  It is important to touch all
 1294   // the way down including all pages in the shadow zone.
 1295   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) {
 1296     // this could be any sized move but this is can be a debugging crumb
 1297     // so the bigger the better.
 1298     movptr(Address(tmp, (-i*(int)os::vm_page_size())), size );
 1299   }
 1300 }
 1301 
 1302 void MacroAssembler::reserved_stack_check() {
 1303   // testing if reserved zone needs to be enabled
 1304   Label no_reserved_zone_enabling;
 1305   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 1306   NOT_LP64(get_thread(rsi);)
 1307 
 1308   cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
 1309   jcc(Assembler::below, no_reserved_zone_enabling);
 1310 
 1311   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
 1312   jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
 1313   should_not_reach_here();
 1314 
 1315   bind(no_reserved_zone_enabling);
 1316 }
 1317 
 1318 void MacroAssembler::c2bool(Register x) {
 1319   // implements x == 0 ? 0 : 1
 1320   // note: must only look at least-significant byte of x
 1321   //       since C-style booleans are stored in one byte
 1322   //       only! (was bug)
 1323   andl(x, 0xFF);
 1324   setb(Assembler::notZero, x);
 1325 }
 1326 
 1327 // Wouldn't need if AddressLiteral version had new name
 1328 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
 1329   Assembler::call(L, rtype);
 1330 }
 1331 
 1332 void MacroAssembler::call(Register entry) {
 1333   Assembler::call(entry);
 1334 }
 1335 
 1336 void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
 1337   assert(rscratch != noreg || always_reachable(entry), "missing");
 1338 
 1339   if (reachable(entry)) {
 1340     Assembler::call_literal(entry.target(), entry.rspec());
 1341   } else {
 1342     lea(rscratch, entry);
 1343     Assembler::call(rscratch);
 1344   }
 1345 }
 1346 
 1347 void MacroAssembler::ic_call(address entry, jint method_index) {
 1348   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 1349 #ifdef _LP64
 1350   // Needs full 64-bit immediate for later patching.
 1351   mov64(rax, (intptr_t)Universe::non_oop_word());
 1352 #else
 1353   movptr(rax, (intptr_t)Universe::non_oop_word());
 1354 #endif
 1355   call(AddressLiteral(entry, rh));
 1356 }
 1357 
 1358 void MacroAssembler::emit_static_call_stub() {
 1359   // Static stub relocation also tags the Method* in the code-stream.
 1360   mov_metadata(rbx, (Metadata*) nullptr);  // Method is zapped till fixup time.
 1361   // This is recognized as unresolved by relocs/nativeinst/ic code.
 1362   jump(RuntimeAddress(pc()));
 1363 }
 1364 
 1365 // Implementation of call_VM versions
 1366 
 1367 void MacroAssembler::call_VM(Register oop_result,
 1368                              address entry_point,
 1369                              bool check_exceptions) {
 1370   Label C, E;
 1371   call(C, relocInfo::none);
 1372   jmp(E);
 1373 
 1374   bind(C);
 1375   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 1376   ret(0);
 1377 
 1378   bind(E);
 1379 }
 1380 
 1381 void MacroAssembler::call_VM(Register oop_result,
 1382                              address entry_point,
 1383                              Register arg_1,
 1384                              bool check_exceptions) {
 1385   Label C, E;
 1386   call(C, relocInfo::none);
 1387   jmp(E);
 1388 
 1389   bind(C);
 1390   pass_arg1(this, arg_1);
 1391   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 1392   ret(0);
 1393 
 1394   bind(E);
 1395 }
 1396 
 1397 void MacroAssembler::call_VM(Register oop_result,
 1398                              address entry_point,
 1399                              Register arg_1,
 1400                              Register arg_2,
 1401                              bool check_exceptions) {
 1402   Label C, E;
 1403   call(C, relocInfo::none);
 1404   jmp(E);
 1405 
 1406   bind(C);
 1407 
 1408   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1409 
 1410   pass_arg2(this, arg_2);
 1411   pass_arg1(this, arg_1);
 1412   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 1413   ret(0);
 1414 
 1415   bind(E);
 1416 }
 1417 
 1418 void MacroAssembler::call_VM(Register oop_result,
 1419                              address entry_point,
 1420                              Register arg_1,
 1421                              Register arg_2,
 1422                              Register arg_3,
 1423                              bool check_exceptions) {
 1424   Label C, E;
 1425   call(C, relocInfo::none);
 1426   jmp(E);
 1427 
 1428   bind(C);
 1429 
 1430   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1431   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1432   pass_arg3(this, arg_3);
 1433   pass_arg2(this, arg_2);
 1434   pass_arg1(this, arg_1);
 1435   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 1436   ret(0);
 1437 
 1438   bind(E);
 1439 }
 1440 
 1441 void MacroAssembler::call_VM(Register oop_result,
 1442                              Register last_java_sp,
 1443                              address entry_point,
 1444                              int number_of_arguments,
 1445                              bool check_exceptions) {
 1446   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1447   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1448 }
 1449 
 1450 void MacroAssembler::call_VM(Register oop_result,
 1451                              Register last_java_sp,
 1452                              address entry_point,
 1453                              Register arg_1,
 1454                              bool check_exceptions) {
 1455   pass_arg1(this, arg_1);
 1456   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1457 }
 1458 
 1459 void MacroAssembler::call_VM(Register oop_result,
 1460                              Register last_java_sp,
 1461                              address entry_point,
 1462                              Register arg_1,
 1463                              Register arg_2,
 1464                              bool check_exceptions) {
 1465 
 1466   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1467   pass_arg2(this, arg_2);
 1468   pass_arg1(this, arg_1);
 1469   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1470 }
 1471 
 1472 void MacroAssembler::call_VM(Register oop_result,
 1473                              Register last_java_sp,
 1474                              address entry_point,
 1475                              Register arg_1,
 1476                              Register arg_2,
 1477                              Register arg_3,
 1478                              bool check_exceptions) {
 1479   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1480   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1481   pass_arg3(this, arg_3);
 1482   pass_arg2(this, arg_2);
 1483   pass_arg1(this, arg_1);
 1484   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1485 }
 1486 
 1487 void MacroAssembler::super_call_VM(Register oop_result,
 1488                                    Register last_java_sp,
 1489                                    address entry_point,
 1490                                    int number_of_arguments,
 1491                                    bool check_exceptions) {
 1492   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
 1493   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1494 }
 1495 
 1496 void MacroAssembler::super_call_VM(Register oop_result,
 1497                                    Register last_java_sp,
 1498                                    address entry_point,
 1499                                    Register arg_1,
 1500                                    bool check_exceptions) {
 1501   pass_arg1(this, arg_1);
 1502   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1503 }
 1504 
 1505 void MacroAssembler::super_call_VM(Register oop_result,
 1506                                    Register last_java_sp,
 1507                                    address entry_point,
 1508                                    Register arg_1,
 1509                                    Register arg_2,
 1510                                    bool check_exceptions) {
 1511 
 1512   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1513   pass_arg2(this, arg_2);
 1514   pass_arg1(this, arg_1);
 1515   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1516 }
 1517 
 1518 void MacroAssembler::super_call_VM(Register oop_result,
 1519                                    Register last_java_sp,
 1520                                    address entry_point,
 1521                                    Register arg_1,
 1522                                    Register arg_2,
 1523                                    Register arg_3,
 1524                                    bool check_exceptions) {
 1525   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1526   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1527   pass_arg3(this, arg_3);
 1528   pass_arg2(this, arg_2);
 1529   pass_arg1(this, arg_1);
 1530   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1531 }
 1532 
 1533 void MacroAssembler::call_VM_base(Register oop_result,
 1534                                   Register java_thread,
 1535                                   Register last_java_sp,
 1536                                   address  entry_point,
 1537                                   int      number_of_arguments,
 1538                                   bool     check_exceptions) {
 1539   // determine java_thread register
 1540   if (!java_thread->is_valid()) {
 1541 #ifdef _LP64
 1542     java_thread = r15_thread;
 1543 #else
 1544     java_thread = rdi;
 1545     get_thread(java_thread);
 1546 #endif // LP64
 1547   }
 1548   // determine last_java_sp register
 1549   if (!last_java_sp->is_valid()) {
 1550     last_java_sp = rsp;
 1551   }
 1552   // debugging support
 1553   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 1554   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
 1555 #ifdef ASSERT
 1556   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
 1557   // r12 is the heapbase.
 1558   LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
 1559 #endif // ASSERT
 1560 
 1561   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 1562   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 1563 
 1564   // push java thread (becomes first argument of C function)
 1565 
 1566   NOT_LP64(push(java_thread); number_of_arguments++);
 1567   LP64_ONLY(mov(c_rarg0, r15_thread));
 1568 
 1569   // set last Java frame before call
 1570   assert(last_java_sp != rbp, "can't use ebp/rbp");
 1571 
 1572   // Only interpreter should have to set fp
 1573   set_last_Java_frame(java_thread, last_java_sp, rbp, nullptr, rscratch1);
 1574 
 1575   // do the call, remove parameters
 1576   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
 1577 
 1578   // restore the thread (cannot use the pushed argument since arguments
 1579   // may be overwritten by C code generated by an optimizing compiler);
 1580   // however can use the register value directly if it is callee saved.
 1581   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
 1582     // rdi & rsi (also r15) are callee saved -> nothing to do
 1583 #ifdef ASSERT
 1584     guarantee(java_thread != rax, "change this code");
 1585     push(rax);
 1586     { Label L;
 1587       get_thread(rax);
 1588       cmpptr(java_thread, rax);
 1589       jcc(Assembler::equal, L);
 1590       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
 1591       bind(L);
 1592     }
 1593     pop(rax);
 1594 #endif
 1595   } else {
 1596     get_thread(java_thread);
 1597   }
 1598   // reset last Java frame
 1599   // Only interpreter should have to clear fp
 1600   reset_last_Java_frame(java_thread, true);
 1601 
 1602    // C++ interp handles this in the interpreter
 1603   check_and_handle_popframe(java_thread);
 1604   check_and_handle_earlyret(java_thread);
 1605 
 1606   if (check_exceptions) {
 1607     // check for pending exceptions (java_thread is set upon return)
 1608     cmpptr(Address(java_thread, Thread::pending_exception_offset()), NULL_WORD);
 1609 #ifndef _LP64
 1610     jump_cc(Assembler::notEqual,
 1611             RuntimeAddress(StubRoutines::forward_exception_entry()));
 1612 #else
 1613     // This used to conditionally jump to forward_exception however it is
 1614     // possible if we relocate that the branch will not reach. So we must jump
 1615     // around so we can always reach
 1616 
 1617     Label ok;
 1618     jcc(Assembler::equal, ok);
 1619     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 1620     bind(ok);
 1621 #endif // LP64
 1622   }
 1623 
 1624   // get oop result if there is one and reset the value in the thread
 1625   if (oop_result->is_valid()) {
 1626     get_vm_result(oop_result, java_thread);
 1627   }
 1628 }
 1629 
 1630 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 1631 
 1632   // Calculate the value for last_Java_sp
 1633   // somewhat subtle. call_VM does an intermediate call
 1634   // which places a return address on the stack just under the
 1635   // stack pointer as the user finished with it. This allows
 1636   // use to retrieve last_Java_pc from last_Java_sp[-1].
 1637   // On 32bit we then have to push additional args on the stack to accomplish
 1638   // the actual requested call. On 64bit call_VM only can use register args
 1639   // so the only extra space is the return address that call_VM created.
 1640   // This hopefully explains the calculations here.
 1641 
 1642 #ifdef _LP64
 1643   // We've pushed one address, correct last_Java_sp
 1644   lea(rax, Address(rsp, wordSize));
 1645 #else
 1646   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
 1647 #endif // LP64
 1648 
 1649   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
 1650 
 1651 }
 1652 
 1653 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
 1654 void MacroAssembler::call_VM_leaf0(address entry_point) {
 1655   MacroAssembler::call_VM_leaf_base(entry_point, 0);
 1656 }
 1657 
 1658 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
 1659   call_VM_leaf_base(entry_point, number_of_arguments);
 1660 }
 1661 
 1662 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
 1663   pass_arg0(this, arg_0);
 1664   call_VM_leaf(entry_point, 1);
 1665 }
 1666 
 1667 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1668 
 1669   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1670   pass_arg1(this, arg_1);
 1671   pass_arg0(this, arg_0);
 1672   call_VM_leaf(entry_point, 2);
 1673 }
 1674 
 1675 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1676   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1677   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1678   pass_arg2(this, arg_2);
 1679   pass_arg1(this, arg_1);
 1680   pass_arg0(this, arg_0);
 1681   call_VM_leaf(entry_point, 3);
 1682 }
 1683 
 1684 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1685   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1686   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1687   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1688   pass_arg3(this, arg_3);
 1689   pass_arg2(this, arg_2);
 1690   pass_arg1(this, arg_1);
 1691   pass_arg0(this, arg_0);
 1692   call_VM_leaf(entry_point, 3);
 1693 }
 1694 
 1695 void MacroAssembler::super_call_VM_leaf(address entry_point) {
 1696   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1697 }
 1698 
 1699 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1700   pass_arg0(this, arg_0);
 1701   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1702 }
 1703 
 1704 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1705   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1706   pass_arg1(this, arg_1);
 1707   pass_arg0(this, arg_0);
 1708   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1709 }
 1710 
 1711 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1712   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1713   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1714   pass_arg2(this, arg_2);
 1715   pass_arg1(this, arg_1);
 1716   pass_arg0(this, arg_0);
 1717   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 1718 }
 1719 
 1720 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1721   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1722   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1723   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1724   pass_arg3(this, arg_3);
 1725   pass_arg2(this, arg_2);
 1726   pass_arg1(this, arg_1);
 1727   pass_arg0(this, arg_0);
 1728   MacroAssembler::call_VM_leaf_base(entry_point, 4);
 1729 }
 1730 
 1731 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
 1732   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 1733   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
 1734   verify_oop_msg(oop_result, "broken oop in call_VM_base");
 1735 }
 1736 
 1737 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 1738   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 1739   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
 1740 }
 1741 
 1742 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
 1743 }
 1744 
 1745 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
 1746 }
 1747 
 1748 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) {
 1749   assert(rscratch != noreg || always_reachable(src1), "missing");
 1750 
 1751   if (reachable(src1)) {
 1752     cmpl(as_Address(src1), imm);
 1753   } else {
 1754     lea(rscratch, src1);
 1755     cmpl(Address(rscratch, 0), imm);
 1756   }
 1757 }
 1758 
 1759 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) {
 1760   assert(!src2.is_lval(), "use cmpptr");
 1761   assert(rscratch != noreg || always_reachable(src2), "missing");
 1762 
 1763   if (reachable(src2)) {
 1764     cmpl(src1, as_Address(src2));
 1765   } else {
 1766     lea(rscratch, src2);
 1767     cmpl(src1, Address(rscratch, 0));
 1768   }
 1769 }
 1770 
 1771 void MacroAssembler::cmp32(Register src1, int32_t imm) {
 1772   Assembler::cmpl(src1, imm);
 1773 }
 1774 
 1775 void MacroAssembler::cmp32(Register src1, Address src2) {
 1776   Assembler::cmpl(src1, src2);
 1777 }
 1778 
 1779 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1780   ucomisd(opr1, opr2);
 1781 
 1782   Label L;
 1783   if (unordered_is_less) {
 1784     movl(dst, -1);
 1785     jcc(Assembler::parity, L);
 1786     jcc(Assembler::below , L);
 1787     movl(dst, 0);
 1788     jcc(Assembler::equal , L);
 1789     increment(dst);
 1790   } else { // unordered is greater
 1791     movl(dst, 1);
 1792     jcc(Assembler::parity, L);
 1793     jcc(Assembler::above , L);
 1794     movl(dst, 0);
 1795     jcc(Assembler::equal , L);
 1796     decrementl(dst);
 1797   }
 1798   bind(L);
 1799 }
 1800 
 1801 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1802   ucomiss(opr1, opr2);
 1803 
 1804   Label L;
 1805   if (unordered_is_less) {
 1806     movl(dst, -1);
 1807     jcc(Assembler::parity, L);
 1808     jcc(Assembler::below , L);
 1809     movl(dst, 0);
 1810     jcc(Assembler::equal , L);
 1811     increment(dst);
 1812   } else { // unordered is greater
 1813     movl(dst, 1);
 1814     jcc(Assembler::parity, L);
 1815     jcc(Assembler::above , L);
 1816     movl(dst, 0);
 1817     jcc(Assembler::equal , L);
 1818     decrementl(dst);
 1819   }
 1820   bind(L);
 1821 }
 1822 
 1823 
 1824 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) {
 1825   assert(rscratch != noreg || always_reachable(src1), "missing");
 1826 
 1827   if (reachable(src1)) {
 1828     cmpb(as_Address(src1), imm);
 1829   } else {
 1830     lea(rscratch, src1);
 1831     cmpb(Address(rscratch, 0), imm);
 1832   }
 1833 }
 1834 
 1835 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) {
 1836 #ifdef _LP64
 1837   assert(rscratch != noreg || always_reachable(src2), "missing");
 1838 
 1839   if (src2.is_lval()) {
 1840     movptr(rscratch, src2);
 1841     Assembler::cmpq(src1, rscratch);
 1842   } else if (reachable(src2)) {
 1843     cmpq(src1, as_Address(src2));
 1844   } else {
 1845     lea(rscratch, src2);
 1846     Assembler::cmpq(src1, Address(rscratch, 0));
 1847   }
 1848 #else
 1849   assert(rscratch == noreg, "not needed");
 1850   if (src2.is_lval()) {
 1851     cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1852   } else {
 1853     cmpl(src1, as_Address(src2));
 1854   }
 1855 #endif // _LP64
 1856 }
 1857 
 1858 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) {
 1859   assert(src2.is_lval(), "not a mem-mem compare");
 1860 #ifdef _LP64
 1861   // moves src2's literal address
 1862   movptr(rscratch, src2);
 1863   Assembler::cmpq(src1, rscratch);
 1864 #else
 1865   assert(rscratch == noreg, "not needed");
 1866   cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1867 #endif // _LP64
 1868 }
 1869 
 1870 void MacroAssembler::cmpoop(Register src1, Register src2) {
 1871   cmpptr(src1, src2);
 1872 }
 1873 
 1874 void MacroAssembler::cmpoop(Register src1, Address src2) {
 1875   cmpptr(src1, src2);
 1876 }
 1877 
 1878 #ifdef _LP64
 1879 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
 1880   movoop(rscratch, src2);
 1881   cmpptr(src1, rscratch);
 1882 }
 1883 #endif
 1884 
 1885 void MacroAssembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 1886   if ((UseAVX > 0) && (dst != src)) {
 1887     xorpd(dst, dst);
 1888   }
 1889   Assembler::cvtss2sd(dst, src);
 1890 }
 1891 
 1892 void MacroAssembler::cvtss2sd(XMMRegister dst, Address src) {
 1893   if (UseAVX > 0) {
 1894     xorpd(dst, dst);
 1895   }
 1896   Assembler::cvtss2sd(dst, src);
 1897 }
 1898 
 1899 void MacroAssembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 1900   if ((UseAVX > 0) && (dst != src)) {
 1901     xorps(dst, dst);
 1902   }
 1903   Assembler::cvtsd2ss(dst, src);
 1904 }
 1905 
 1906 void MacroAssembler::cvtsd2ss(XMMRegister dst, Address src) {
 1907   if (UseAVX > 0) {
 1908     xorps(dst, dst);
 1909   }
 1910   Assembler::cvtsd2ss(dst, src);
 1911 }
 1912 
 1913 void MacroAssembler::cvtsi2sdl(XMMRegister dst, Register src) {
 1914   if (UseAVX > 0) {
 1915     xorpd(dst, dst);
 1916   }
 1917   Assembler::cvtsi2sdl(dst, src);
 1918 }
 1919 
 1920 void MacroAssembler::cvtsi2sdl(XMMRegister dst, Address src) {
 1921   if (UseAVX > 0) {
 1922     xorpd(dst, dst);
 1923   }
 1924   Assembler::cvtsi2sdl(dst, src);
 1925 }
 1926 
 1927 void MacroAssembler::cvtsi2ssl(XMMRegister dst, Register src) {
 1928   if (UseAVX > 0) {
 1929     xorps(dst, dst);
 1930   }
 1931   Assembler::cvtsi2ssl(dst, src);
 1932 }
 1933 
 1934 void MacroAssembler::cvtsi2ssl(XMMRegister dst, Address src) {
 1935   if (UseAVX > 0) {
 1936     xorps(dst, dst);
 1937   }
 1938   Assembler::cvtsi2ssl(dst, src);
 1939 }
 1940 
 1941 #ifdef _LP64
 1942 void MacroAssembler::cvtsi2sdq(XMMRegister dst, Register src) {
 1943   if (UseAVX > 0) {
 1944     xorpd(dst, dst);
 1945   }
 1946   Assembler::cvtsi2sdq(dst, src);
 1947 }
 1948 
 1949 void MacroAssembler::cvtsi2sdq(XMMRegister dst, Address src) {
 1950   if (UseAVX > 0) {
 1951     xorpd(dst, dst);
 1952   }
 1953   Assembler::cvtsi2sdq(dst, src);
 1954 }
 1955 
 1956 void MacroAssembler::cvtsi2ssq(XMMRegister dst, Register src) {
 1957   if (UseAVX > 0) {
 1958     xorps(dst, dst);
 1959   }
 1960   Assembler::cvtsi2ssq(dst, src);
 1961 }
 1962 
 1963 void MacroAssembler::cvtsi2ssq(XMMRegister dst, Address src) {
 1964   if (UseAVX > 0) {
 1965     xorps(dst, dst);
 1966   }
 1967   Assembler::cvtsi2ssq(dst, src);
 1968 }
 1969 #endif  // _LP64
 1970 
 1971 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
 1972   assert(rscratch != noreg || always_reachable(adr), "missing");
 1973 
 1974   if (reachable(adr)) {
 1975     lock();
 1976     cmpxchgptr(reg, as_Address(adr));
 1977   } else {
 1978     lea(rscratch, adr);
 1979     lock();
 1980     cmpxchgptr(reg, Address(rscratch, 0));
 1981   }
 1982 }
 1983 
 1984 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
 1985   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
 1986 }
 1987 
 1988 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1989   assert(rscratch != noreg || always_reachable(src), "missing");
 1990 
 1991   if (reachable(src)) {
 1992     Assembler::comisd(dst, as_Address(src));
 1993   } else {
 1994     lea(rscratch, src);
 1995     Assembler::comisd(dst, Address(rscratch, 0));
 1996   }
 1997 }
 1998 
 1999 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2000   assert(rscratch != noreg || always_reachable(src), "missing");
 2001 
 2002   if (reachable(src)) {
 2003     Assembler::comiss(dst, as_Address(src));
 2004   } else {
 2005     lea(rscratch, src);
 2006     Assembler::comiss(dst, Address(rscratch, 0));
 2007   }
 2008 }
 2009 
 2010 
 2011 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) {
 2012   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 2013 
 2014   Condition negated_cond = negate_condition(cond);
 2015   Label L;
 2016   jcc(negated_cond, L);
 2017   pushf(); // Preserve flags
 2018   atomic_incl(counter_addr, rscratch);
 2019   popf();
 2020   bind(L);
 2021 }
 2022 
 2023 int MacroAssembler::corrected_idivl(Register reg) {
 2024   // Full implementation of Java idiv and irem; checks for
 2025   // special case as described in JVM spec., p.243 & p.271.
 2026   // The function returns the (pc) offset of the idivl
 2027   // instruction - may be needed for implicit exceptions.
 2028   //
 2029   //         normal case                           special case
 2030   //
 2031   // input : rax,: dividend                         min_int
 2032   //         reg: divisor   (may not be rax,/rdx)   -1
 2033   //
 2034   // output: rax,: quotient  (= rax, idiv reg)       min_int
 2035   //         rdx: remainder (= rax, irem reg)       0
 2036   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
 2037   const int min_int = 0x80000000;
 2038   Label normal_case, special_case;
 2039 
 2040   // check for special case
 2041   cmpl(rax, min_int);
 2042   jcc(Assembler::notEqual, normal_case);
 2043   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
 2044   cmpl(reg, -1);
 2045   jcc(Assembler::equal, special_case);
 2046 
 2047   // handle normal case
 2048   bind(normal_case);
 2049   cdql();
 2050   int idivl_offset = offset();
 2051   idivl(reg);
 2052 
 2053   // normal and special case exit
 2054   bind(special_case);
 2055 
 2056   return idivl_offset;
 2057 }
 2058 
 2059 
 2060 
 2061 void MacroAssembler::decrementl(Register reg, int value) {
 2062   if (value == min_jint) {subl(reg, value) ; return; }
 2063   if (value <  0) { incrementl(reg, -value); return; }
 2064   if (value == 0) {                        ; return; }
 2065   if (value == 1 && UseIncDec) { decl(reg) ; return; }
 2066   /* else */      { subl(reg, value)       ; return; }
 2067 }
 2068 
 2069 void MacroAssembler::decrementl(Address dst, int value) {
 2070   if (value == min_jint) {subl(dst, value) ; return; }
 2071   if (value <  0) { incrementl(dst, -value); return; }
 2072   if (value == 0) {                        ; return; }
 2073   if (value == 1 && UseIncDec) { decl(dst) ; return; }
 2074   /* else */      { subl(dst, value)       ; return; }
 2075 }
 2076 
 2077 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
 2078   assert(shift_value > 0, "illegal shift value");
 2079   Label _is_positive;
 2080   testl (reg, reg);
 2081   jcc (Assembler::positive, _is_positive);
 2082   int offset = (1 << shift_value) - 1 ;
 2083 
 2084   if (offset == 1) {
 2085     incrementl(reg);
 2086   } else {
 2087     addl(reg, offset);
 2088   }
 2089 
 2090   bind (_is_positive);
 2091   sarl(reg, shift_value);
 2092 }
 2093 
 2094 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2095   assert(rscratch != noreg || always_reachable(src), "missing");
 2096 
 2097   if (reachable(src)) {
 2098     Assembler::divsd(dst, as_Address(src));
 2099   } else {
 2100     lea(rscratch, src);
 2101     Assembler::divsd(dst, Address(rscratch, 0));
 2102   }
 2103 }
 2104 
 2105 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2106   assert(rscratch != noreg || always_reachable(src), "missing");
 2107 
 2108   if (reachable(src)) {
 2109     Assembler::divss(dst, as_Address(src));
 2110   } else {
 2111     lea(rscratch, src);
 2112     Assembler::divss(dst, Address(rscratch, 0));
 2113   }
 2114 }
 2115 
 2116 void MacroAssembler::enter() {
 2117   push(rbp);
 2118   mov(rbp, rsp);
 2119 }
 2120 
 2121 void MacroAssembler::post_call_nop() {
 2122   if (!Continuations::enabled()) {
 2123     return;
 2124   }
 2125   InstructionMark im(this);
 2126   relocate(post_call_nop_Relocation::spec());
 2127   InlineSkippedInstructionsCounter skipCounter(this);
 2128   emit_int8((uint8_t)0x0f);
 2129   emit_int8((uint8_t)0x1f);
 2130   emit_int8((uint8_t)0x84);
 2131   emit_int8((uint8_t)0x00);
 2132   emit_int32(0x00);
 2133 }
 2134 
 2135 // A 5 byte nop that is safe for patching (see patch_verified_entry)
 2136 void MacroAssembler::fat_nop() {
 2137   if (UseAddressNop) {
 2138     addr_nop_5();
 2139   } else {
 2140     emit_int8((uint8_t)0x26); // es:
 2141     emit_int8((uint8_t)0x2e); // cs:
 2142     emit_int8((uint8_t)0x64); // fs:
 2143     emit_int8((uint8_t)0x65); // gs:
 2144     emit_int8((uint8_t)0x90);
 2145   }
 2146 }
 2147 
 2148 #ifndef _LP64
 2149 void MacroAssembler::fcmp(Register tmp) {
 2150   fcmp(tmp, 1, true, true);
 2151 }
 2152 
 2153 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
 2154   assert(!pop_right || pop_left, "usage error");
 2155   if (VM_Version::supports_cmov()) {
 2156     assert(tmp == noreg, "unneeded temp");
 2157     if (pop_left) {
 2158       fucomip(index);
 2159     } else {
 2160       fucomi(index);
 2161     }
 2162     if (pop_right) {
 2163       fpop();
 2164     }
 2165   } else {
 2166     assert(tmp != noreg, "need temp");
 2167     if (pop_left) {
 2168       if (pop_right) {
 2169         fcompp();
 2170       } else {
 2171         fcomp(index);
 2172       }
 2173     } else {
 2174       fcom(index);
 2175     }
 2176     // convert FPU condition into eflags condition via rax,
 2177     save_rax(tmp);
 2178     fwait(); fnstsw_ax();
 2179     sahf();
 2180     restore_rax(tmp);
 2181   }
 2182   // condition codes set as follows:
 2183   //
 2184   // CF (corresponds to C0) if x < y
 2185   // PF (corresponds to C2) if unordered
 2186   // ZF (corresponds to C3) if x = y
 2187 }
 2188 
 2189 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
 2190   fcmp2int(dst, unordered_is_less, 1, true, true);
 2191 }
 2192 
 2193 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
 2194   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
 2195   Label L;
 2196   if (unordered_is_less) {
 2197     movl(dst, -1);
 2198     jcc(Assembler::parity, L);
 2199     jcc(Assembler::below , L);
 2200     movl(dst, 0);
 2201     jcc(Assembler::equal , L);
 2202     increment(dst);
 2203   } else { // unordered is greater
 2204     movl(dst, 1);
 2205     jcc(Assembler::parity, L);
 2206     jcc(Assembler::above , L);
 2207     movl(dst, 0);
 2208     jcc(Assembler::equal , L);
 2209     decrementl(dst);
 2210   }
 2211   bind(L);
 2212 }
 2213 
 2214 void MacroAssembler::fld_d(AddressLiteral src) {
 2215   fld_d(as_Address(src));
 2216 }
 2217 
 2218 void MacroAssembler::fld_s(AddressLiteral src) {
 2219   fld_s(as_Address(src));
 2220 }
 2221 
 2222 void MacroAssembler::fldcw(AddressLiteral src) {
 2223   fldcw(as_Address(src));
 2224 }
 2225 
 2226 void MacroAssembler::fpop() {
 2227   ffree();
 2228   fincstp();
 2229 }
 2230 
 2231 void MacroAssembler::fremr(Register tmp) {
 2232   save_rax(tmp);
 2233   { Label L;
 2234     bind(L);
 2235     fprem();
 2236     fwait(); fnstsw_ax();
 2237     sahf();
 2238     jcc(Assembler::parity, L);
 2239   }
 2240   restore_rax(tmp);
 2241   // Result is in ST0.
 2242   // Note: fxch & fpop to get rid of ST1
 2243   // (otherwise FPU stack could overflow eventually)
 2244   fxch(1);
 2245   fpop();
 2246 }
 2247 
 2248 void MacroAssembler::empty_FPU_stack() {
 2249   if (VM_Version::supports_mmx()) {
 2250     emms();
 2251   } else {
 2252     for (int i = 8; i-- > 0; ) ffree(i);
 2253   }
 2254 }
 2255 #endif // !LP64
 2256 
 2257 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2258   assert(rscratch != noreg || always_reachable(src), "missing");
 2259   if (reachable(src)) {
 2260     Assembler::mulpd(dst, as_Address(src));
 2261   } else {
 2262     lea(rscratch, src);
 2263     Assembler::mulpd(dst, Address(rscratch, 0));
 2264   }
 2265 }
 2266 
 2267 void MacroAssembler::load_float(Address src) {
 2268 #ifdef _LP64
 2269   movflt(xmm0, src);
 2270 #else
 2271   if (UseSSE >= 1) {
 2272     movflt(xmm0, src);
 2273   } else {
 2274     fld_s(src);
 2275   }
 2276 #endif // LP64
 2277 }
 2278 
 2279 void MacroAssembler::store_float(Address dst) {
 2280 #ifdef _LP64
 2281   movflt(dst, xmm0);
 2282 #else
 2283   if (UseSSE >= 1) {
 2284     movflt(dst, xmm0);
 2285   } else {
 2286     fstp_s(dst);
 2287   }
 2288 #endif // LP64
 2289 }
 2290 
 2291 void MacroAssembler::load_double(Address src) {
 2292 #ifdef _LP64
 2293   movdbl(xmm0, src);
 2294 #else
 2295   if (UseSSE >= 2) {
 2296     movdbl(xmm0, src);
 2297   } else {
 2298     fld_d(src);
 2299   }
 2300 #endif // LP64
 2301 }
 2302 
 2303 void MacroAssembler::store_double(Address dst) {
 2304 #ifdef _LP64
 2305   movdbl(dst, xmm0);
 2306 #else
 2307   if (UseSSE >= 2) {
 2308     movdbl(dst, xmm0);
 2309   } else {
 2310     fstp_d(dst);
 2311   }
 2312 #endif // LP64
 2313 }
 2314 
 2315 // dst = c = a * b + c
 2316 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2317   Assembler::vfmadd231sd(c, a, b);
 2318   if (dst != c) {
 2319     movdbl(dst, c);
 2320   }
 2321 }
 2322 
 2323 // dst = c = a * b + c
 2324 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2325   Assembler::vfmadd231ss(c, a, b);
 2326   if (dst != c) {
 2327     movflt(dst, c);
 2328   }
 2329 }
 2330 
 2331 // dst = c = a * b + c
 2332 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2333   Assembler::vfmadd231pd(c, a, b, vector_len);
 2334   if (dst != c) {
 2335     vmovdqu(dst, c);
 2336   }
 2337 }
 2338 
 2339 // dst = c = a * b + c
 2340 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2341   Assembler::vfmadd231ps(c, a, b, vector_len);
 2342   if (dst != c) {
 2343     vmovdqu(dst, c);
 2344   }
 2345 }
 2346 
 2347 // dst = c = a * b + c
 2348 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2349   Assembler::vfmadd231pd(c, a, b, vector_len);
 2350   if (dst != c) {
 2351     vmovdqu(dst, c);
 2352   }
 2353 }
 2354 
 2355 // dst = c = a * b + c
 2356 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2357   Assembler::vfmadd231ps(c, a, b, vector_len);
 2358   if (dst != c) {
 2359     vmovdqu(dst, c);
 2360   }
 2361 }
 2362 
 2363 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) {
 2364   assert(rscratch != noreg || always_reachable(dst), "missing");
 2365 
 2366   if (reachable(dst)) {
 2367     incrementl(as_Address(dst));
 2368   } else {
 2369     lea(rscratch, dst);
 2370     incrementl(Address(rscratch, 0));
 2371   }
 2372 }
 2373 
 2374 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) {
 2375   incrementl(as_Address(dst, rscratch));
 2376 }
 2377 
 2378 void MacroAssembler::incrementl(Register reg, int value) {
 2379   if (value == min_jint) {addl(reg, value) ; return; }
 2380   if (value <  0) { decrementl(reg, -value); return; }
 2381   if (value == 0) {                        ; return; }
 2382   if (value == 1 && UseIncDec) { incl(reg) ; return; }
 2383   /* else */      { addl(reg, value)       ; return; }
 2384 }
 2385 
 2386 void MacroAssembler::incrementl(Address dst, int value) {
 2387   if (value == min_jint) {addl(dst, value) ; return; }
 2388   if (value <  0) { decrementl(dst, -value); return; }
 2389   if (value == 0) {                        ; return; }
 2390   if (value == 1 && UseIncDec) { incl(dst) ; return; }
 2391   /* else */      { addl(dst, value)       ; return; }
 2392 }
 2393 
 2394 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) {
 2395   assert(rscratch != noreg || always_reachable(dst), "missing");
 2396 
 2397   if (reachable(dst)) {
 2398     jmp_literal(dst.target(), dst.rspec());
 2399   } else {
 2400     lea(rscratch, dst);
 2401     jmp(rscratch);
 2402   }
 2403 }
 2404 
 2405 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) {
 2406   assert(rscratch != noreg || always_reachable(dst), "missing");
 2407 
 2408   if (reachable(dst)) {
 2409     InstructionMark im(this);
 2410     relocate(dst.reloc());
 2411     const int short_size = 2;
 2412     const int long_size = 6;
 2413     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
 2414     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
 2415       // 0111 tttn #8-bit disp
 2416       emit_int8(0x70 | cc);
 2417       emit_int8((offs - short_size) & 0xFF);
 2418     } else {
 2419       // 0000 1111 1000 tttn #32-bit disp
 2420       emit_int8(0x0F);
 2421       emit_int8((unsigned char)(0x80 | cc));
 2422       emit_int32(offs - long_size);
 2423     }
 2424   } else {
 2425 #ifdef ASSERT
 2426     warning("reversing conditional branch");
 2427 #endif /* ASSERT */
 2428     Label skip;
 2429     jccb(reverse[cc], skip);
 2430     lea(rscratch, dst);
 2431     Assembler::jmp(rscratch);
 2432     bind(skip);
 2433   }
 2434 }
 2435 
 2436 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) {
 2437   assert(rscratch != noreg || always_reachable(src), "missing");
 2438 
 2439   if (reachable(src)) {
 2440     Assembler::ldmxcsr(as_Address(src));
 2441   } else {
 2442     lea(rscratch, src);
 2443     Assembler::ldmxcsr(Address(rscratch, 0));
 2444   }
 2445 }
 2446 
 2447 int MacroAssembler::load_signed_byte(Register dst, Address src) {
 2448   int off;
 2449   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2450     off = offset();
 2451     movsbl(dst, src); // movsxb
 2452   } else {
 2453     off = load_unsigned_byte(dst, src);
 2454     shll(dst, 24);
 2455     sarl(dst, 24);
 2456   }
 2457   return off;
 2458 }
 2459 
 2460 // Note: load_signed_short used to be called load_signed_word.
 2461 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
 2462 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
 2463 // The term "word" in HotSpot means a 32- or 64-bit machine word.
 2464 int MacroAssembler::load_signed_short(Register dst, Address src) {
 2465   int off;
 2466   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2467     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
 2468     // version but this is what 64bit has always done. This seems to imply
 2469     // that users are only using 32bits worth.
 2470     off = offset();
 2471     movswl(dst, src); // movsxw
 2472   } else {
 2473     off = load_unsigned_short(dst, src);
 2474     shll(dst, 16);
 2475     sarl(dst, 16);
 2476   }
 2477   return off;
 2478 }
 2479 
 2480 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
 2481   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2482   // and "3.9 Partial Register Penalties", p. 22).
 2483   int off;
 2484   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
 2485     off = offset();
 2486     movzbl(dst, src); // movzxb
 2487   } else {
 2488     xorl(dst, dst);
 2489     off = offset();
 2490     movb(dst, src);
 2491   }
 2492   return off;
 2493 }
 2494 
 2495 // Note: load_unsigned_short used to be called load_unsigned_word.
 2496 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
 2497   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2498   // and "3.9 Partial Register Penalties", p. 22).
 2499   int off;
 2500   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
 2501     off = offset();
 2502     movzwl(dst, src); // movzxw
 2503   } else {
 2504     xorl(dst, dst);
 2505     off = offset();
 2506     movw(dst, src);
 2507   }
 2508   return off;
 2509 }
 2510 
 2511 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
 2512   switch (size_in_bytes) {
 2513 #ifndef _LP64
 2514   case  8:
 2515     assert(dst2 != noreg, "second dest register required");
 2516     movl(dst,  src);
 2517     movl(dst2, src.plus_disp(BytesPerInt));
 2518     break;
 2519 #else
 2520   case  8:  movq(dst, src); break;
 2521 #endif
 2522   case  4:  movl(dst, src); break;
 2523   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
 2524   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
 2525   default:  ShouldNotReachHere();
 2526   }
 2527 }
 2528 
 2529 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
 2530   switch (size_in_bytes) {
 2531 #ifndef _LP64
 2532   case  8:
 2533     assert(src2 != noreg, "second source register required");
 2534     movl(dst,                        src);
 2535     movl(dst.plus_disp(BytesPerInt), src2);
 2536     break;
 2537 #else
 2538   case  8:  movq(dst, src); break;
 2539 #endif
 2540   case  4:  movl(dst, src); break;
 2541   case  2:  movw(dst, src); break;
 2542   case  1:  movb(dst, src); break;
 2543   default:  ShouldNotReachHere();
 2544   }
 2545 }
 2546 
 2547 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) {
 2548   assert(rscratch != noreg || always_reachable(dst), "missing");
 2549 
 2550   if (reachable(dst)) {
 2551     movl(as_Address(dst), src);
 2552   } else {
 2553     lea(rscratch, dst);
 2554     movl(Address(rscratch, 0), src);
 2555   }
 2556 }
 2557 
 2558 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
 2559   if (reachable(src)) {
 2560     movl(dst, as_Address(src));
 2561   } else {
 2562     lea(dst, src);
 2563     movl(dst, Address(dst, 0));
 2564   }
 2565 }
 2566 
 2567 // C++ bool manipulation
 2568 
 2569 void MacroAssembler::movbool(Register dst, Address src) {
 2570   if(sizeof(bool) == 1)
 2571     movb(dst, src);
 2572   else if(sizeof(bool) == 2)
 2573     movw(dst, src);
 2574   else if(sizeof(bool) == 4)
 2575     movl(dst, src);
 2576   else
 2577     // unsupported
 2578     ShouldNotReachHere();
 2579 }
 2580 
 2581 void MacroAssembler::movbool(Address dst, bool boolconst) {
 2582   if(sizeof(bool) == 1)
 2583     movb(dst, (int) boolconst);
 2584   else if(sizeof(bool) == 2)
 2585     movw(dst, (int) boolconst);
 2586   else if(sizeof(bool) == 4)
 2587     movl(dst, (int) boolconst);
 2588   else
 2589     // unsupported
 2590     ShouldNotReachHere();
 2591 }
 2592 
 2593 void MacroAssembler::movbool(Address dst, Register src) {
 2594   if(sizeof(bool) == 1)
 2595     movb(dst, src);
 2596   else if(sizeof(bool) == 2)
 2597     movw(dst, src);
 2598   else if(sizeof(bool) == 4)
 2599     movl(dst, src);
 2600   else
 2601     // unsupported
 2602     ShouldNotReachHere();
 2603 }
 2604 
 2605 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2606   assert(rscratch != noreg || always_reachable(src), "missing");
 2607 
 2608   if (reachable(src)) {
 2609     movdl(dst, as_Address(src));
 2610   } else {
 2611     lea(rscratch, src);
 2612     movdl(dst, Address(rscratch, 0));
 2613   }
 2614 }
 2615 
 2616 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2617   assert(rscratch != noreg || always_reachable(src), "missing");
 2618 
 2619   if (reachable(src)) {
 2620     movq(dst, as_Address(src));
 2621   } else {
 2622     lea(rscratch, src);
 2623     movq(dst, Address(rscratch, 0));
 2624   }
 2625 }
 2626 
 2627 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2628   assert(rscratch != noreg || always_reachable(src), "missing");
 2629 
 2630   if (reachable(src)) {
 2631     if (UseXmmLoadAndClearUpper) {
 2632       movsd (dst, as_Address(src));
 2633     } else {
 2634       movlpd(dst, as_Address(src));
 2635     }
 2636   } else {
 2637     lea(rscratch, src);
 2638     if (UseXmmLoadAndClearUpper) {
 2639       movsd (dst, Address(rscratch, 0));
 2640     } else {
 2641       movlpd(dst, Address(rscratch, 0));
 2642     }
 2643   }
 2644 }
 2645 
 2646 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2647   assert(rscratch != noreg || always_reachable(src), "missing");
 2648 
 2649   if (reachable(src)) {
 2650     movss(dst, as_Address(src));
 2651   } else {
 2652     lea(rscratch, src);
 2653     movss(dst, Address(rscratch, 0));
 2654   }
 2655 }
 2656 
 2657 void MacroAssembler::movptr(Register dst, Register src) {
 2658   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2659 }
 2660 
 2661 void MacroAssembler::movptr(Register dst, Address src) {
 2662   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2663 }
 2664 
 2665 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 2666 void MacroAssembler::movptr(Register dst, intptr_t src) {
 2667 #ifdef _LP64
 2668   if (is_simm32(src)) {
 2669     movq(dst, checked_cast<int32_t>(src));
 2670   } else {
 2671     mov64(dst, src);
 2672   }
 2673 #else
 2674   movl(dst, src);
 2675 #endif
 2676 }
 2677 
 2678 void MacroAssembler::movptr(Address dst, Register src) {
 2679   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2680 }
 2681 
 2682 void MacroAssembler::movptr(Address dst, int32_t src) {
 2683   LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src));
 2684 }
 2685 
 2686 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
 2687   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2688   Assembler::movdqu(dst, src);
 2689 }
 2690 
 2691 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
 2692   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2693   Assembler::movdqu(dst, src);
 2694 }
 2695 
 2696 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
 2697   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2698   Assembler::movdqu(dst, src);
 2699 }
 2700 
 2701 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2702   assert(rscratch != noreg || always_reachable(src), "missing");
 2703 
 2704   if (reachable(src)) {
 2705     movdqu(dst, as_Address(src));
 2706   } else {
 2707     lea(rscratch, src);
 2708     movdqu(dst, Address(rscratch, 0));
 2709   }
 2710 }
 2711 
 2712 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
 2713   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2714   Assembler::vmovdqu(dst, src);
 2715 }
 2716 
 2717 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
 2718   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2719   Assembler::vmovdqu(dst, src);
 2720 }
 2721 
 2722 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2723   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2724   Assembler::vmovdqu(dst, src);
 2725 }
 2726 
 2727 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2728   assert(rscratch != noreg || always_reachable(src), "missing");
 2729 
 2730   if (reachable(src)) {
 2731     vmovdqu(dst, as_Address(src));
 2732   }
 2733   else {
 2734     lea(rscratch, src);
 2735     vmovdqu(dst, Address(rscratch, 0));
 2736   }
 2737 }
 2738 
 2739 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2740   assert(rscratch != noreg || always_reachable(src), "missing");
 2741 
 2742   if (vector_len == AVX_512bit) {
 2743     evmovdquq(dst, src, AVX_512bit, rscratch);
 2744   } else if (vector_len == AVX_256bit) {
 2745     vmovdqu(dst, src, rscratch);
 2746   } else {
 2747     movdqu(dst, src, rscratch);
 2748   }
 2749 }
 2750 
 2751 void MacroAssembler::kmov(KRegister dst, Address src) {
 2752   if (VM_Version::supports_avx512bw()) {
 2753     kmovql(dst, src);
 2754   } else {
 2755     assert(VM_Version::supports_evex(), "");
 2756     kmovwl(dst, src);
 2757   }
 2758 }
 2759 
 2760 void MacroAssembler::kmov(Address dst, KRegister src) {
 2761   if (VM_Version::supports_avx512bw()) {
 2762     kmovql(dst, src);
 2763   } else {
 2764     assert(VM_Version::supports_evex(), "");
 2765     kmovwl(dst, src);
 2766   }
 2767 }
 2768 
 2769 void MacroAssembler::kmov(KRegister dst, KRegister src) {
 2770   if (VM_Version::supports_avx512bw()) {
 2771     kmovql(dst, src);
 2772   } else {
 2773     assert(VM_Version::supports_evex(), "");
 2774     kmovwl(dst, src);
 2775   }
 2776 }
 2777 
 2778 void MacroAssembler::kmov(Register dst, KRegister src) {
 2779   if (VM_Version::supports_avx512bw()) {
 2780     kmovql(dst, src);
 2781   } else {
 2782     assert(VM_Version::supports_evex(), "");
 2783     kmovwl(dst, src);
 2784   }
 2785 }
 2786 
 2787 void MacroAssembler::kmov(KRegister dst, Register src) {
 2788   if (VM_Version::supports_avx512bw()) {
 2789     kmovql(dst, src);
 2790   } else {
 2791     assert(VM_Version::supports_evex(), "");
 2792     kmovwl(dst, src);
 2793   }
 2794 }
 2795 
 2796 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) {
 2797   assert(rscratch != noreg || always_reachable(src), "missing");
 2798 
 2799   if (reachable(src)) {
 2800     kmovql(dst, as_Address(src));
 2801   } else {
 2802     lea(rscratch, src);
 2803     kmovql(dst, Address(rscratch, 0));
 2804   }
 2805 }
 2806 
 2807 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) {
 2808   assert(rscratch != noreg || always_reachable(src), "missing");
 2809 
 2810   if (reachable(src)) {
 2811     kmovwl(dst, as_Address(src));
 2812   } else {
 2813     lea(rscratch, src);
 2814     kmovwl(dst, Address(rscratch, 0));
 2815   }
 2816 }
 2817 
 2818 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2819                                int vector_len, Register rscratch) {
 2820   assert(rscratch != noreg || always_reachable(src), "missing");
 2821 
 2822   if (reachable(src)) {
 2823     Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
 2824   } else {
 2825     lea(rscratch, src);
 2826     Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len);
 2827   }
 2828 }
 2829 
 2830 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2831                                int vector_len, Register rscratch) {
 2832   assert(rscratch != noreg || always_reachable(src), "missing");
 2833 
 2834   if (reachable(src)) {
 2835     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
 2836   } else {
 2837     lea(rscratch, src);
 2838     Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len);
 2839   }
 2840 }
 2841 
 2842 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2843   assert(rscratch != noreg || always_reachable(src), "missing");
 2844 
 2845   if (reachable(src)) {
 2846     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
 2847   } else {
 2848     lea(rscratch, src);
 2849     Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len);
 2850   }
 2851 }
 2852 
 2853 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2854   assert(rscratch != noreg || always_reachable(src), "missing");
 2855 
 2856   if (reachable(src)) {
 2857     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
 2858   } else {
 2859     lea(rscratch, src);
 2860     Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len);
 2861   }
 2862 }
 2863 
 2864 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2865   assert(rscratch != noreg || always_reachable(src), "missing");
 2866 
 2867   if (reachable(src)) {
 2868     Assembler::evmovdquq(dst, as_Address(src), vector_len);
 2869   } else {
 2870     lea(rscratch, src);
 2871     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
 2872   }
 2873 }
 2874 
 2875 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2876   assert(rscratch != noreg || always_reachable(src), "missing");
 2877 
 2878   if (reachable(src)) {
 2879     Assembler::movdqa(dst, as_Address(src));
 2880   } else {
 2881     lea(rscratch, src);
 2882     Assembler::movdqa(dst, Address(rscratch, 0));
 2883   }
 2884 }
 2885 
 2886 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2887   assert(rscratch != noreg || always_reachable(src), "missing");
 2888 
 2889   if (reachable(src)) {
 2890     Assembler::movsd(dst, as_Address(src));
 2891   } else {
 2892     lea(rscratch, src);
 2893     Assembler::movsd(dst, Address(rscratch, 0));
 2894   }
 2895 }
 2896 
 2897 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2898   assert(rscratch != noreg || always_reachable(src), "missing");
 2899 
 2900   if (reachable(src)) {
 2901     Assembler::movss(dst, as_Address(src));
 2902   } else {
 2903     lea(rscratch, src);
 2904     Assembler::movss(dst, Address(rscratch, 0));
 2905   }
 2906 }
 2907 
 2908 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2909   assert(rscratch != noreg || always_reachable(src), "missing");
 2910 
 2911   if (reachable(src)) {
 2912     Assembler::movddup(dst, as_Address(src));
 2913   } else {
 2914     lea(rscratch, src);
 2915     Assembler::movddup(dst, Address(rscratch, 0));
 2916   }
 2917 }
 2918 
 2919 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2920   assert(rscratch != noreg || always_reachable(src), "missing");
 2921 
 2922   if (reachable(src)) {
 2923     Assembler::vmovddup(dst, as_Address(src), vector_len);
 2924   } else {
 2925     lea(rscratch, src);
 2926     Assembler::vmovddup(dst, Address(rscratch, 0), vector_len);
 2927   }
 2928 }
 2929 
 2930 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2931   assert(rscratch != noreg || always_reachable(src), "missing");
 2932 
 2933   if (reachable(src)) {
 2934     Assembler::mulsd(dst, as_Address(src));
 2935   } else {
 2936     lea(rscratch, src);
 2937     Assembler::mulsd(dst, Address(rscratch, 0));
 2938   }
 2939 }
 2940 
 2941 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2942   assert(rscratch != noreg || always_reachable(src), "missing");
 2943 
 2944   if (reachable(src)) {
 2945     Assembler::mulss(dst, as_Address(src));
 2946   } else {
 2947     lea(rscratch, src);
 2948     Assembler::mulss(dst, Address(rscratch, 0));
 2949   }
 2950 }
 2951 
 2952 void MacroAssembler::null_check(Register reg, int offset) {
 2953   if (needs_explicit_null_check(offset)) {
 2954     // provoke OS null exception if reg is null by
 2955     // accessing M[reg] w/o changing any (non-CC) registers
 2956     // NOTE: cmpl is plenty here to provoke a segv
 2957     cmpptr(rax, Address(reg, 0));
 2958     // Note: should probably use testl(rax, Address(reg, 0));
 2959     //       may be shorter code (however, this version of
 2960     //       testl needs to be implemented first)
 2961   } else {
 2962     // nothing to do, (later) access of M[reg + offset]
 2963     // will provoke OS null exception if reg is null
 2964   }
 2965 }
 2966 
 2967 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
 2968   andptr(markword, markWord::inline_type_mask_in_place);
 2969   cmpptr(markword, markWord::inline_type_pattern);
 2970   jcc(Assembler::equal, is_inline_type);
 2971 }
 2972 
 2973 void MacroAssembler::test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type) {
 2974   movl(temp_reg, Address(klass, Klass::access_flags_offset()));
 2975   testl(temp_reg, JVM_ACC_IDENTITY);
 2976   jcc(Assembler::zero, is_inline_type);
 2977 }
 2978 
 2979 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type) {
 2980   testptr(object, object);
 2981   jcc(Assembler::zero, not_inline_type);
 2982   const int is_inline_type_mask = markWord::inline_type_pattern;
 2983   movptr(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
 2984   andptr(tmp, is_inline_type_mask);
 2985   cmpptr(tmp, is_inline_type_mask);
 2986   jcc(Assembler::notEqual, not_inline_type);
 2987 }
 2988 
 2989 void MacroAssembler::test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type) {
 2990 #ifdef ASSERT
 2991   {
 2992     Label done_check;
 2993     test_klass_is_inline_type(klass, temp_reg, done_check);
 2994     stop("test_klass_is_empty_inline_type with non inline type klass");
 2995     bind(done_check);
 2996   }
 2997 #endif
 2998   movl(temp_reg, Address(klass, InstanceKlass::misc_flags_offset()));
 2999   testl(temp_reg, InstanceKlassFlags::is_empty_inline_type_value());
 3000   jcc(Assembler::notZero, is_empty_inline_type);
 3001 }
 3002 
 3003 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
 3004   movl(temp_reg, flags);
 3005   shrl(temp_reg, ResolvedFieldEntry::is_null_free_inline_type_shift);
 3006   andl(temp_reg, 0x1);
 3007   testl(temp_reg, temp_reg);
 3008   jcc(Assembler::notZero, is_null_free_inline_type);
 3009 }
 3010 
 3011 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
 3012   movl(temp_reg, flags);
 3013   shrl(temp_reg, ResolvedFieldEntry::is_null_free_inline_type_shift);
 3014   andl(temp_reg, 0x1);
 3015   testl(temp_reg, temp_reg);
 3016   jcc(Assembler::zero, not_null_free_inline_type);
 3017 }
 3018 
 3019 void MacroAssembler::test_field_is_flat(Register flags, Register temp_reg, Label& is_flat) {
 3020   movl(temp_reg, flags);
 3021   shrl(temp_reg, ResolvedFieldEntry::is_flat_shift);
 3022   andl(temp_reg, 0x1);
 3023   testl(temp_reg, temp_reg);
 3024   jcc(Assembler::notZero, is_flat);
 3025 }
 3026 
 3027 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) {
 3028   Label test_mark_word;
 3029   // load mark word
 3030   movptr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
 3031   // check displaced
 3032   testl(temp_reg, markWord::unlocked_value);
 3033   jccb(Assembler::notZero, test_mark_word);
 3034   // slow path use klass prototype
 3035   push(rscratch1);
 3036   load_prototype_header(temp_reg, oop, rscratch1);
 3037   pop(rscratch1);
 3038 
 3039   bind(test_mark_word);
 3040   testl(temp_reg, test_bit);
 3041   jcc((jmp_set) ? Assembler::notZero : Assembler::zero, jmp_label);
 3042 }
 3043 
 3044 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg,
 3045                                          Label& is_flat_array) {
 3046 #ifdef _LP64
 3047   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array);
 3048 #else
 3049   load_klass(temp_reg, oop, noreg);
 3050   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 3051   test_flat_array_layout(temp_reg, is_flat_array);
 3052 #endif
 3053 }
 3054 
 3055 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg,
 3056                                              Label& is_non_flat_array) {
 3057 #ifdef _LP64
 3058   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
 3059 #else
 3060   load_klass(temp_reg, oop, noreg);
 3061   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 3062   test_non_flat_array_layout(temp_reg, is_non_flat_array);
 3063 #endif
 3064 }
 3065 
 3066 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label&is_null_free_array) {
 3067 #ifdef _LP64
 3068   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
 3069 #else
 3070   load_klass(temp_reg, oop, noreg);
 3071   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 3072   test_null_free_array_layout(temp_reg, is_null_free_array);
 3073 #endif
 3074 }
 3075 
 3076 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
 3077 #ifdef _LP64
 3078   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
 3079 #else
 3080   load_klass(temp_reg, oop, noreg);
 3081   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 3082   test_non_null_free_array_layout(temp_reg, is_non_null_free_array);
 3083 #endif
 3084 }
 3085 
 3086 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
 3087   testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
 3088   jcc(Assembler::notZero, is_flat_array);
 3089 }
 3090 
 3091 void MacroAssembler::test_non_flat_array_layout(Register lh, Label& is_non_flat_array) {
 3092   testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
 3093   jcc(Assembler::zero, is_non_flat_array);
 3094 }
 3095 
 3096 void MacroAssembler::test_null_free_array_layout(Register lh, Label& is_null_free_array) {
 3097   testl(lh, Klass::_lh_null_free_array_bit_inplace);
 3098   jcc(Assembler::notZero, is_null_free_array);
 3099 }
 3100 
 3101 void MacroAssembler::test_non_null_free_array_layout(Register lh, Label& is_non_null_free_array) {
 3102   testl(lh, Klass::_lh_null_free_array_bit_inplace);
 3103   jcc(Assembler::zero, is_non_null_free_array);
 3104 }
 3105 
 3106 
 3107 void MacroAssembler::os_breakpoint() {
 3108   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 3109   // (e.g., MSVC can't call ps() otherwise)
 3110   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 3111 }
 3112 
 3113 void MacroAssembler::unimplemented(const char* what) {
 3114   const char* buf = nullptr;
 3115   {
 3116     ResourceMark rm;
 3117     stringStream ss;
 3118     ss.print("unimplemented: %s", what);
 3119     buf = code_string(ss.as_string());
 3120   }
 3121   stop(buf);
 3122 }
 3123 
 3124 #ifdef _LP64
 3125 #define XSTATE_BV 0x200
 3126 #endif
 3127 
 3128 void MacroAssembler::pop_CPU_state() {
 3129   pop_FPU_state();
 3130   pop_IU_state();
 3131 }
 3132 
 3133 void MacroAssembler::pop_FPU_state() {
 3134 #ifndef _LP64
 3135   frstor(Address(rsp, 0));
 3136 #else
 3137   fxrstor(Address(rsp, 0));
 3138 #endif
 3139   addptr(rsp, FPUStateSizeInWords * wordSize);
 3140 }
 3141 
 3142 void MacroAssembler::pop_IU_state() {
 3143   popa();
 3144   LP64_ONLY(addq(rsp, 8));
 3145   popf();
 3146 }
 3147 
 3148 // Save Integer and Float state
 3149 // Warning: Stack must be 16 byte aligned (64bit)
 3150 void MacroAssembler::push_CPU_state() {
 3151   push_IU_state();
 3152   push_FPU_state();
 3153 }
 3154 
 3155 void MacroAssembler::push_FPU_state() {
 3156   subptr(rsp, FPUStateSizeInWords * wordSize);
 3157 #ifndef _LP64
 3158   fnsave(Address(rsp, 0));
 3159   fwait();
 3160 #else
 3161   fxsave(Address(rsp, 0));
 3162 #endif // LP64
 3163 }
 3164 
 3165 void MacroAssembler::push_IU_state() {
 3166   // Push flags first because pusha kills them
 3167   pushf();
 3168   // Make sure rsp stays 16-byte aligned
 3169   LP64_ONLY(subq(rsp, 8));
 3170   pusha();
 3171 }
 3172 
 3173 void MacroAssembler::push_cont_fastpath() {
 3174   if (!Continuations::enabled()) return;
 3175 
 3176 #ifndef _LP64
 3177   Register rthread = rax;
 3178   Register rrealsp = rbx;
 3179   push(rthread);
 3180   push(rrealsp);
 3181 
 3182   get_thread(rthread);
 3183 
 3184   // The code below wants the original RSP.
 3185   // Move it back after the pushes above.
 3186   movptr(rrealsp, rsp);
 3187   addptr(rrealsp, 2*wordSize);
 3188 #else
 3189   Register rthread = r15_thread;
 3190   Register rrealsp = rsp;
 3191 #endif
 3192 
 3193   Label done;
 3194   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3195   jccb(Assembler::belowEqual, done);
 3196   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp);
 3197   bind(done);
 3198 
 3199 #ifndef _LP64
 3200   pop(rrealsp);
 3201   pop(rthread);
 3202 #endif
 3203 }
 3204 
 3205 void MacroAssembler::pop_cont_fastpath() {
 3206   if (!Continuations::enabled()) return;
 3207 
 3208 #ifndef _LP64
 3209   Register rthread = rax;
 3210   Register rrealsp = rbx;
 3211   push(rthread);
 3212   push(rrealsp);
 3213 
 3214   get_thread(rthread);
 3215 
 3216   // The code below wants the original RSP.
 3217   // Move it back after the pushes above.
 3218   movptr(rrealsp, rsp);
 3219   addptr(rrealsp, 2*wordSize);
 3220 #else
 3221   Register rthread = r15_thread;
 3222   Register rrealsp = rsp;
 3223 #endif
 3224 
 3225   Label done;
 3226   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3227   jccb(Assembler::below, done);
 3228   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0);
 3229   bind(done);
 3230 
 3231 #ifndef _LP64
 3232   pop(rrealsp);
 3233   pop(rthread);
 3234 #endif
 3235 }
 3236 
 3237 void MacroAssembler::inc_held_monitor_count() {
 3238 #ifndef _LP64
 3239   Register thread = rax;
 3240   push(thread);
 3241   get_thread(thread);
 3242   incrementl(Address(thread, JavaThread::held_monitor_count_offset()));
 3243   pop(thread);
 3244 #else // LP64
 3245   incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3246 #endif
 3247 }
 3248 
 3249 void MacroAssembler::dec_held_monitor_count() {
 3250 #ifndef _LP64
 3251   Register thread = rax;
 3252   push(thread);
 3253   get_thread(thread);
 3254   decrementl(Address(thread, JavaThread::held_monitor_count_offset()));
 3255   pop(thread);
 3256 #else // LP64
 3257   decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3258 #endif
 3259 }
 3260 
 3261 #ifdef ASSERT
 3262 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
 3263 #ifdef _LP64
 3264   Label no_cont;
 3265   movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset()));
 3266   testl(cont, cont);
 3267   jcc(Assembler::zero, no_cont);
 3268   stop(name);
 3269   bind(no_cont);
 3270 #else
 3271   Unimplemented();
 3272 #endif
 3273 }
 3274 #endif
 3275 
 3276 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
 3277   if (!java_thread->is_valid()) {
 3278     java_thread = rdi;
 3279     get_thread(java_thread);
 3280   }
 3281   // we must set sp to zero to clear frame
 3282   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 3283   // must clear fp, so that compiled frames are not confused; it is
 3284   // possible that we need it only for debugging
 3285   if (clear_fp) {
 3286     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 3287   }
 3288   // Always clear the pc because it could have been set by make_walkable()
 3289   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 3290   vzeroupper();
 3291 }
 3292 
 3293 void MacroAssembler::restore_rax(Register tmp) {
 3294   if (tmp == noreg) pop(rax);
 3295   else if (tmp != rax) mov(rax, tmp);
 3296 }
 3297 
 3298 void MacroAssembler::round_to(Register reg, int modulus) {
 3299   addptr(reg, modulus - 1);
 3300   andptr(reg, -modulus);
 3301 }
 3302 
 3303 void MacroAssembler::save_rax(Register tmp) {
 3304   if (tmp == noreg) push(rax);
 3305   else if (tmp != rax) mov(tmp, rax);
 3306 }
 3307 
 3308 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, bool at_return, bool in_nmethod) {
 3309   if (at_return) {
 3310     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
 3311     // we may safely use rsp instead to perform the stack watermark check.
 3312     cmpptr(in_nmethod ? rsp : rbp, Address(thread_reg, JavaThread::polling_word_offset()));
 3313     jcc(Assembler::above, slow_path);
 3314     return;
 3315   }
 3316   testb(Address(thread_reg, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
 3317   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
 3318 }
 3319 
 3320 // Calls to C land
 3321 //
 3322 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
 3323 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 3324 // has to be reset to 0. This is required to allow proper stack traversal.
 3325 void MacroAssembler::set_last_Java_frame(Register java_thread,
 3326                                          Register last_java_sp,
 3327                                          Register last_java_fp,
 3328                                          address  last_java_pc,
 3329                                          Register rscratch) {
 3330   vzeroupper();
 3331   // determine java_thread register
 3332   if (!java_thread->is_valid()) {
 3333     java_thread = rdi;
 3334     get_thread(java_thread);
 3335   }
 3336   // determine last_java_sp register
 3337   if (!last_java_sp->is_valid()) {
 3338     last_java_sp = rsp;
 3339   }
 3340   // last_java_fp is optional
 3341   if (last_java_fp->is_valid()) {
 3342     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
 3343   }
 3344   // last_java_pc is optional
 3345   if (last_java_pc != nullptr) {
 3346     Address java_pc(java_thread,
 3347                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 3348     lea(java_pc, InternalAddress(last_java_pc), rscratch);
 3349   }
 3350   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 3351 }
 3352 
 3353 void MacroAssembler::shlptr(Register dst, int imm8) {
 3354   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
 3355 }
 3356 
 3357 void MacroAssembler::shrptr(Register dst, int imm8) {
 3358   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
 3359 }
 3360 
 3361 void MacroAssembler::sign_extend_byte(Register reg) {
 3362   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
 3363     movsbl(reg, reg); // movsxb
 3364   } else {
 3365     shll(reg, 24);
 3366     sarl(reg, 24);
 3367   }
 3368 }
 3369 
 3370 void MacroAssembler::sign_extend_short(Register reg) {
 3371   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 3372     movswl(reg, reg); // movsxw
 3373   } else {
 3374     shll(reg, 16);
 3375     sarl(reg, 16);
 3376   }
 3377 }
 3378 
 3379 void MacroAssembler::testl(Address dst, int32_t imm32) {
 3380   if (imm32 >= 0 && is8bit(imm32)) {
 3381     testb(dst, imm32);
 3382   } else {
 3383     Assembler::testl(dst, imm32);
 3384   }
 3385 }
 3386 
 3387 void MacroAssembler::testl(Register dst, int32_t imm32) {
 3388   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
 3389     testb(dst, imm32);
 3390   } else {
 3391     Assembler::testl(dst, imm32);
 3392   }
 3393 }
 3394 
 3395 void MacroAssembler::testl(Register dst, AddressLiteral src) {
 3396   assert(always_reachable(src), "Address should be reachable");
 3397   testl(dst, as_Address(src));
 3398 }
 3399 
 3400 #ifdef _LP64
 3401 
 3402 void MacroAssembler::testq(Address dst, int32_t imm32) {
 3403   if (imm32 >= 0) {
 3404     testl(dst, imm32);
 3405   } else {
 3406     Assembler::testq(dst, imm32);
 3407   }
 3408 }
 3409 
 3410 void MacroAssembler::testq(Register dst, int32_t imm32) {
 3411   if (imm32 >= 0) {
 3412     testl(dst, imm32);
 3413   } else {
 3414     Assembler::testq(dst, imm32);
 3415   }
 3416 }
 3417 
 3418 #endif
 3419 
 3420 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 3421   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3422   Assembler::pcmpeqb(dst, src);
 3423 }
 3424 
 3425 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 3426   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3427   Assembler::pcmpeqw(dst, src);
 3428 }
 3429 
 3430 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 3431   assert((dst->encoding() < 16),"XMM register should be 0-15");
 3432   Assembler::pcmpestri(dst, src, imm8);
 3433 }
 3434 
 3435 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 3436   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3437   Assembler::pcmpestri(dst, src, imm8);
 3438 }
 3439 
 3440 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 3441   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3442   Assembler::pmovzxbw(dst, src);
 3443 }
 3444 
 3445 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
 3446   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3447   Assembler::pmovzxbw(dst, src);
 3448 }
 3449 
 3450 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
 3451   assert((src->encoding() < 16),"XMM register should be 0-15");
 3452   Assembler::pmovmskb(dst, src);
 3453 }
 3454 
 3455 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
 3456   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3457   Assembler::ptest(dst, src);
 3458 }
 3459 
 3460 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3461   assert(rscratch != noreg || always_reachable(src), "missing");
 3462 
 3463   if (reachable(src)) {
 3464     Assembler::sqrtss(dst, as_Address(src));
 3465   } else {
 3466     lea(rscratch, src);
 3467     Assembler::sqrtss(dst, Address(rscratch, 0));
 3468   }
 3469 }
 3470 
 3471 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3472   assert(rscratch != noreg || always_reachable(src), "missing");
 3473 
 3474   if (reachable(src)) {
 3475     Assembler::subsd(dst, as_Address(src));
 3476   } else {
 3477     lea(rscratch, src);
 3478     Assembler::subsd(dst, Address(rscratch, 0));
 3479   }
 3480 }
 3481 
 3482 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) {
 3483   assert(rscratch != noreg || always_reachable(src), "missing");
 3484 
 3485   if (reachable(src)) {
 3486     Assembler::roundsd(dst, as_Address(src), rmode);
 3487   } else {
 3488     lea(rscratch, src);
 3489     Assembler::roundsd(dst, Address(rscratch, 0), rmode);
 3490   }
 3491 }
 3492 
 3493 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3494   assert(rscratch != noreg || always_reachable(src), "missing");
 3495 
 3496   if (reachable(src)) {
 3497     Assembler::subss(dst, as_Address(src));
 3498   } else {
 3499     lea(rscratch, src);
 3500     Assembler::subss(dst, Address(rscratch, 0));
 3501   }
 3502 }
 3503 
 3504 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3505   assert(rscratch != noreg || always_reachable(src), "missing");
 3506 
 3507   if (reachable(src)) {
 3508     Assembler::ucomisd(dst, as_Address(src));
 3509   } else {
 3510     lea(rscratch, src);
 3511     Assembler::ucomisd(dst, Address(rscratch, 0));
 3512   }
 3513 }
 3514 
 3515 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3516   assert(rscratch != noreg || always_reachable(src), "missing");
 3517 
 3518   if (reachable(src)) {
 3519     Assembler::ucomiss(dst, as_Address(src));
 3520   } else {
 3521     lea(rscratch, src);
 3522     Assembler::ucomiss(dst, Address(rscratch, 0));
 3523   }
 3524 }
 3525 
 3526 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3527   assert(rscratch != noreg || always_reachable(src), "missing");
 3528 
 3529   // Used in sign-bit flipping with aligned address.
 3530   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3531   if (reachable(src)) {
 3532     Assembler::xorpd(dst, as_Address(src));
 3533   } else {
 3534     lea(rscratch, src);
 3535     Assembler::xorpd(dst, Address(rscratch, 0));
 3536   }
 3537 }
 3538 
 3539 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
 3540   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
 3541     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3542   }
 3543   else {
 3544     Assembler::xorpd(dst, src);
 3545   }
 3546 }
 3547 
 3548 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
 3549   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
 3550     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3551   } else {
 3552     Assembler::xorps(dst, src);
 3553   }
 3554 }
 3555 
 3556 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3557   assert(rscratch != noreg || always_reachable(src), "missing");
 3558 
 3559   // Used in sign-bit flipping with aligned address.
 3560   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3561   if (reachable(src)) {
 3562     Assembler::xorps(dst, as_Address(src));
 3563   } else {
 3564     lea(rscratch, src);
 3565     Assembler::xorps(dst, Address(rscratch, 0));
 3566   }
 3567 }
 3568 
 3569 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3570   assert(rscratch != noreg || always_reachable(src), "missing");
 3571 
 3572   // Used in sign-bit flipping with aligned address.
 3573   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
 3574   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
 3575   if (reachable(src)) {
 3576     Assembler::pshufb(dst, as_Address(src));
 3577   } else {
 3578     lea(rscratch, src);
 3579     Assembler::pshufb(dst, Address(rscratch, 0));
 3580   }
 3581 }
 3582 
 3583 // AVX 3-operands instructions
 3584 
 3585 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3586   assert(rscratch != noreg || always_reachable(src), "missing");
 3587 
 3588   if (reachable(src)) {
 3589     vaddsd(dst, nds, as_Address(src));
 3590   } else {
 3591     lea(rscratch, src);
 3592     vaddsd(dst, nds, Address(rscratch, 0));
 3593   }
 3594 }
 3595 
 3596 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3597   assert(rscratch != noreg || always_reachable(src), "missing");
 3598 
 3599   if (reachable(src)) {
 3600     vaddss(dst, nds, as_Address(src));
 3601   } else {
 3602     lea(rscratch, src);
 3603     vaddss(dst, nds, Address(rscratch, 0));
 3604   }
 3605 }
 3606 
 3607 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3608   assert(UseAVX > 0, "requires some form of AVX");
 3609   assert(rscratch != noreg || always_reachable(src), "missing");
 3610 
 3611   if (reachable(src)) {
 3612     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
 3613   } else {
 3614     lea(rscratch, src);
 3615     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
 3616   }
 3617 }
 3618 
 3619 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3620   assert(UseAVX > 0, "requires some form of AVX");
 3621   assert(rscratch != noreg || always_reachable(src), "missing");
 3622 
 3623   if (reachable(src)) {
 3624     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
 3625   } else {
 3626     lea(rscratch, src);
 3627     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
 3628   }
 3629 }
 3630 
 3631 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3632   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3633   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3634 
 3635   vandps(dst, nds, negate_field, vector_len, rscratch);
 3636 }
 3637 
 3638 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3639   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3640   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3641 
 3642   vandpd(dst, nds, negate_field, vector_len, rscratch);
 3643 }
 3644 
 3645 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3646   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3647   Assembler::vpaddb(dst, nds, src, vector_len);
 3648 }
 3649 
 3650 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3651   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3652   Assembler::vpaddb(dst, nds, src, vector_len);
 3653 }
 3654 
 3655 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3656   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3657   Assembler::vpaddw(dst, nds, src, vector_len);
 3658 }
 3659 
 3660 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3661   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3662   Assembler::vpaddw(dst, nds, src, vector_len);
 3663 }
 3664 
 3665 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3666   assert(rscratch != noreg || always_reachable(src), "missing");
 3667 
 3668   if (reachable(src)) {
 3669     Assembler::vpand(dst, nds, as_Address(src), vector_len);
 3670   } else {
 3671     lea(rscratch, src);
 3672     Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len);
 3673   }
 3674 }
 3675 
 3676 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3677   assert(rscratch != noreg || always_reachable(src), "missing");
 3678 
 3679   if (reachable(src)) {
 3680     Assembler::vpbroadcastd(dst, as_Address(src), vector_len);
 3681   } else {
 3682     lea(rscratch, src);
 3683     Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len);
 3684   }
 3685 }
 3686 
 3687 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3688   assert(rscratch != noreg || always_reachable(src), "missing");
 3689 
 3690   if (reachable(src)) {
 3691     Assembler::vpbroadcastq(dst, as_Address(src), vector_len);
 3692   } else {
 3693     lea(rscratch, src);
 3694     Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len);
 3695   }
 3696 }
 3697 
 3698 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3699   assert(rscratch != noreg || always_reachable(src), "missing");
 3700 
 3701   if (reachable(src)) {
 3702     Assembler::vbroadcastsd(dst, as_Address(src), vector_len);
 3703   } else {
 3704     lea(rscratch, src);
 3705     Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len);
 3706   }
 3707 }
 3708 
 3709 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3710   assert(rscratch != noreg || always_reachable(src), "missing");
 3711 
 3712   if (reachable(src)) {
 3713     Assembler::vbroadcastss(dst, as_Address(src), vector_len);
 3714   } else {
 3715     lea(rscratch, src);
 3716     Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len);
 3717   }
 3718 }
 3719 
 3720 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3721   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3722   Assembler::vpcmpeqb(dst, nds, src, vector_len);
 3723 }
 3724 
 3725 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3726   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3727   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3728 }
 3729 
 3730 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3731   assert(rscratch != noreg || always_reachable(src), "missing");
 3732 
 3733   if (reachable(src)) {
 3734     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
 3735   } else {
 3736     lea(rscratch, src);
 3737     Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len);
 3738   }
 3739 }
 3740 
 3741 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3742                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3743   assert(rscratch != noreg || always_reachable(src), "missing");
 3744 
 3745   if (reachable(src)) {
 3746     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3747   } else {
 3748     lea(rscratch, src);
 3749     Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3750   }
 3751 }
 3752 
 3753 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3754                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3755   assert(rscratch != noreg || always_reachable(src), "missing");
 3756 
 3757   if (reachable(src)) {
 3758     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3759   } else {
 3760     lea(rscratch, src);
 3761     Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3762   }
 3763 }
 3764 
 3765 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3766                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3767   assert(rscratch != noreg || always_reachable(src), "missing");
 3768 
 3769   if (reachable(src)) {
 3770     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3771   } else {
 3772     lea(rscratch, src);
 3773     Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3774   }
 3775 }
 3776 
 3777 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3778                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3779   assert(rscratch != noreg || always_reachable(src), "missing");
 3780 
 3781   if (reachable(src)) {
 3782     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3783   } else {
 3784     lea(rscratch, src);
 3785     Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3786   }
 3787 }
 3788 
 3789 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
 3790   if (width == Assembler::Q) {
 3791     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
 3792   } else {
 3793     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
 3794   }
 3795 }
 3796 
 3797 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) {
 3798   int eq_cond_enc = 0x29;
 3799   int gt_cond_enc = 0x37;
 3800   if (width != Assembler::Q) {
 3801     eq_cond_enc = 0x74 + width;
 3802     gt_cond_enc = 0x64 + width;
 3803   }
 3804   switch (cond) {
 3805   case eq:
 3806     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3807     break;
 3808   case neq:
 3809     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3810     vallones(xtmp, vector_len);
 3811     vpxor(dst, xtmp, dst, vector_len);
 3812     break;
 3813   case le:
 3814     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3815     vallones(xtmp, vector_len);
 3816     vpxor(dst, xtmp, dst, vector_len);
 3817     break;
 3818   case nlt:
 3819     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3820     vallones(xtmp, vector_len);
 3821     vpxor(dst, xtmp, dst, vector_len);
 3822     break;
 3823   case lt:
 3824     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3825     break;
 3826   case nle:
 3827     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3828     break;
 3829   default:
 3830     assert(false, "Should not reach here");
 3831   }
 3832 }
 3833 
 3834 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 3835   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3836   Assembler::vpmovzxbw(dst, src, vector_len);
 3837 }
 3838 
 3839 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
 3840   assert((src->encoding() < 16),"XMM register should be 0-15");
 3841   Assembler::vpmovmskb(dst, src, vector_len);
 3842 }
 3843 
 3844 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3845   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3846   Assembler::vpmullw(dst, nds, src, vector_len);
 3847 }
 3848 
 3849 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3850   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3851   Assembler::vpmullw(dst, nds, src, vector_len);
 3852 }
 3853 
 3854 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3855   assert((UseAVX > 0), "AVX support is needed");
 3856   assert(rscratch != noreg || always_reachable(src), "missing");
 3857 
 3858   if (reachable(src)) {
 3859     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
 3860   } else {
 3861     lea(rscratch, src);
 3862     Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len);
 3863   }
 3864 }
 3865 
 3866 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3867   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3868   Assembler::vpsubb(dst, nds, src, vector_len);
 3869 }
 3870 
 3871 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3872   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3873   Assembler::vpsubb(dst, nds, src, vector_len);
 3874 }
 3875 
 3876 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3877   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3878   Assembler::vpsubw(dst, nds, src, vector_len);
 3879 }
 3880 
 3881 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3882   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3883   Assembler::vpsubw(dst, nds, src, vector_len);
 3884 }
 3885 
 3886 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3887   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3888   Assembler::vpsraw(dst, nds, shift, vector_len);
 3889 }
 3890 
 3891 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3892   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3893   Assembler::vpsraw(dst, nds, shift, vector_len);
 3894 }
 3895 
 3896 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3897   assert(UseAVX > 2,"");
 3898   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3899      vector_len = 2;
 3900   }
 3901   Assembler::evpsraq(dst, nds, shift, vector_len);
 3902 }
 3903 
 3904 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3905   assert(UseAVX > 2,"");
 3906   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3907      vector_len = 2;
 3908   }
 3909   Assembler::evpsraq(dst, nds, shift, vector_len);
 3910 }
 3911 
 3912 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3913   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3914   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3915 }
 3916 
 3917 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3918   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3919   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3920 }
 3921 
 3922 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3923   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3924   Assembler::vpsllw(dst, nds, shift, vector_len);
 3925 }
 3926 
 3927 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3928   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3929   Assembler::vpsllw(dst, nds, shift, vector_len);
 3930 }
 3931 
 3932 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
 3933   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3934   Assembler::vptest(dst, src);
 3935 }
 3936 
 3937 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 3938   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3939   Assembler::punpcklbw(dst, src);
 3940 }
 3941 
 3942 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
 3943   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 3944   Assembler::pshufd(dst, src, mode);
 3945 }
 3946 
 3947 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 3948   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3949   Assembler::pshuflw(dst, src, mode);
 3950 }
 3951 
 3952 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3953   assert(rscratch != noreg || always_reachable(src), "missing");
 3954 
 3955   if (reachable(src)) {
 3956     vandpd(dst, nds, as_Address(src), vector_len);
 3957   } else {
 3958     lea(rscratch, src);
 3959     vandpd(dst, nds, Address(rscratch, 0), vector_len);
 3960   }
 3961 }
 3962 
 3963 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3964   assert(rscratch != noreg || always_reachable(src), "missing");
 3965 
 3966   if (reachable(src)) {
 3967     vandps(dst, nds, as_Address(src), vector_len);
 3968   } else {
 3969     lea(rscratch, src);
 3970     vandps(dst, nds, Address(rscratch, 0), vector_len);
 3971   }
 3972 }
 3973 
 3974 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3975                             bool merge, int vector_len, Register rscratch) {
 3976   assert(rscratch != noreg || always_reachable(src), "missing");
 3977 
 3978   if (reachable(src)) {
 3979     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
 3980   } else {
 3981     lea(rscratch, src);
 3982     Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 3983   }
 3984 }
 3985 
 3986 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3987   assert(rscratch != noreg || always_reachable(src), "missing");
 3988 
 3989   if (reachable(src)) {
 3990     vdivsd(dst, nds, as_Address(src));
 3991   } else {
 3992     lea(rscratch, src);
 3993     vdivsd(dst, nds, Address(rscratch, 0));
 3994   }
 3995 }
 3996 
 3997 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3998   assert(rscratch != noreg || always_reachable(src), "missing");
 3999 
 4000   if (reachable(src)) {
 4001     vdivss(dst, nds, as_Address(src));
 4002   } else {
 4003     lea(rscratch, src);
 4004     vdivss(dst, nds, Address(rscratch, 0));
 4005   }
 4006 }
 4007 
 4008 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4009   assert(rscratch != noreg || always_reachable(src), "missing");
 4010 
 4011   if (reachable(src)) {
 4012     vmulsd(dst, nds, as_Address(src));
 4013   } else {
 4014     lea(rscratch, src);
 4015     vmulsd(dst, nds, Address(rscratch, 0));
 4016   }
 4017 }
 4018 
 4019 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4020   assert(rscratch != noreg || always_reachable(src), "missing");
 4021 
 4022   if (reachable(src)) {
 4023     vmulss(dst, nds, as_Address(src));
 4024   } else {
 4025     lea(rscratch, src);
 4026     vmulss(dst, nds, Address(rscratch, 0));
 4027   }
 4028 }
 4029 
 4030 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4031   assert(rscratch != noreg || always_reachable(src), "missing");
 4032 
 4033   if (reachable(src)) {
 4034     vsubsd(dst, nds, as_Address(src));
 4035   } else {
 4036     lea(rscratch, src);
 4037     vsubsd(dst, nds, Address(rscratch, 0));
 4038   }
 4039 }
 4040 
 4041 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4042   assert(rscratch != noreg || always_reachable(src), "missing");
 4043 
 4044   if (reachable(src)) {
 4045     vsubss(dst, nds, as_Address(src));
 4046   } else {
 4047     lea(rscratch, src);
 4048     vsubss(dst, nds, Address(rscratch, 0));
 4049   }
 4050 }
 4051 
 4052 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4053   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 4054   assert(rscratch != noreg || always_reachable(src), "missing");
 4055 
 4056   vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch);
 4057 }
 4058 
 4059 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 4060   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 4061   assert(rscratch != noreg || always_reachable(src), "missing");
 4062 
 4063   vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch);
 4064 }
 4065 
 4066 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4067   assert(rscratch != noreg || always_reachable(src), "missing");
 4068 
 4069   if (reachable(src)) {
 4070     vxorpd(dst, nds, as_Address(src), vector_len);
 4071   } else {
 4072     lea(rscratch, src);
 4073     vxorpd(dst, nds, Address(rscratch, 0), vector_len);
 4074   }
 4075 }
 4076 
 4077 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4078   assert(rscratch != noreg || always_reachable(src), "missing");
 4079 
 4080   if (reachable(src)) {
 4081     vxorps(dst, nds, as_Address(src), vector_len);
 4082   } else {
 4083     lea(rscratch, src);
 4084     vxorps(dst, nds, Address(rscratch, 0), vector_len);
 4085   }
 4086 }
 4087 
 4088 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4089   assert(rscratch != noreg || always_reachable(src), "missing");
 4090 
 4091   if (UseAVX > 1 || (vector_len < 1)) {
 4092     if (reachable(src)) {
 4093       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
 4094     } else {
 4095       lea(rscratch, src);
 4096       Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len);
 4097     }
 4098   } else {
 4099     MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch);
 4100   }
 4101 }
 4102 
 4103 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4104   assert(rscratch != noreg || always_reachable(src), "missing");
 4105 
 4106   if (reachable(src)) {
 4107     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
 4108   } else {
 4109     lea(rscratch, src);
 4110     Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len);
 4111   }
 4112 }
 4113 
 4114 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) {
 4115   const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask);
 4116   STATIC_ASSERT(inverted_mask == -4); // otherwise check this code
 4117   // The inverted mask is sign-extended
 4118   andptr(possibly_non_local, inverted_mask);
 4119 }
 4120 
 4121 void MacroAssembler::resolve_jobject(Register value,
 4122                                      Register thread,
 4123                                      Register tmp) {
 4124   assert_different_registers(value, thread, tmp);
 4125   Label done, tagged, weak_tagged;
 4126   testptr(value, value);
 4127   jcc(Assembler::zero, done);           // Use null as-is.
 4128   testptr(value, JNIHandles::tag_mask); // Test for tag.
 4129   jcc(Assembler::notZero, tagged);
 4130 
 4131   // Resolve local handle
 4132   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread);
 4133   verify_oop(value);
 4134   jmp(done);
 4135 
 4136   bind(tagged);
 4137   testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag.
 4138   jcc(Assembler::notZero, weak_tagged);
 4139 
 4140   // Resolve global handle
 4141   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4142   verify_oop(value);
 4143   jmp(done);
 4144 
 4145   bind(weak_tagged);
 4146   // Resolve jweak.
 4147   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 4148                  value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread);
 4149   verify_oop(value);
 4150 
 4151   bind(done);
 4152 }
 4153 
 4154 void MacroAssembler::resolve_global_jobject(Register value,
 4155                                             Register thread,
 4156                                             Register tmp) {
 4157   assert_different_registers(value, thread, tmp);
 4158   Label done;
 4159 
 4160   testptr(value, value);
 4161   jcc(Assembler::zero, done);           // Use null as-is.
 4162 
 4163 #ifdef ASSERT
 4164   {
 4165     Label valid_global_tag;
 4166     testptr(value, JNIHandles::TypeTag::global); // Test for global tag.
 4167     jcc(Assembler::notZero, valid_global_tag);
 4168     stop("non global jobject using resolve_global_jobject");
 4169     bind(valid_global_tag);
 4170   }
 4171 #endif
 4172 
 4173   // Resolve global handle
 4174   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4175   verify_oop(value);
 4176 
 4177   bind(done);
 4178 }
 4179 
 4180 void MacroAssembler::subptr(Register dst, int32_t imm32) {
 4181   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
 4182 }
 4183 
 4184 // Force generation of a 4 byte immediate value even if it fits into 8bit
 4185 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
 4186   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
 4187 }
 4188 
 4189 void MacroAssembler::subptr(Register dst, Register src) {
 4190   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
 4191 }
 4192 
 4193 // C++ bool manipulation
 4194 void MacroAssembler::testbool(Register dst) {
 4195   if(sizeof(bool) == 1)
 4196     testb(dst, 0xff);
 4197   else if(sizeof(bool) == 2) {
 4198     // testw implementation needed for two byte bools
 4199     ShouldNotReachHere();
 4200   } else if(sizeof(bool) == 4)
 4201     testl(dst, dst);
 4202   else
 4203     // unsupported
 4204     ShouldNotReachHere();
 4205 }
 4206 
 4207 void MacroAssembler::testptr(Register dst, Register src) {
 4208   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 4209 }
 4210 
 4211 // Object / value buffer allocation...
 4212 //
 4213 // Kills klass and rsi on LP64
 4214 void MacroAssembler::allocate_instance(Register klass, Register new_obj,
 4215                                        Register t1, Register t2,
 4216                                        bool clear_fields, Label& alloc_failed)
 4217 {
 4218   Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop;
 4219   Register layout_size = t1;
 4220   assert(new_obj == rax, "needs to be rax");
 4221   assert_different_registers(klass, new_obj, t1, t2);
 4222 
 4223   // get instance_size in InstanceKlass (scaled to a count of bytes)
 4224   movl(layout_size, Address(klass, Klass::layout_helper_offset()));
 4225   // test to see if it has a finalizer or is malformed in some way
 4226   testl(layout_size, Klass::_lh_instance_slow_path_bit);
 4227   jcc(Assembler::notZero, slow_case_no_pop);
 4228 
 4229   // Allocate the instance:
 4230   //  If TLAB is enabled:
 4231   //    Try to allocate in the TLAB.
 4232   //    If fails, go to the slow path.
 4233   //  Else If inline contiguous allocations are enabled:
 4234   //    Try to allocate in eden.
 4235   //    If fails due to heap end, go to slow path.
 4236   //
 4237   //  If TLAB is enabled OR inline contiguous is enabled:
 4238   //    Initialize the allocation.
 4239   //    Exit.
 4240   //
 4241   //  Go to slow path.
 4242 
 4243   push(klass);
 4244   const Register thread = LP64_ONLY(r15_thread) NOT_LP64(klass);
 4245 #ifndef _LP64
 4246   if (UseTLAB) {
 4247     get_thread(thread);
 4248   }
 4249 #endif // _LP64
 4250 
 4251   if (UseTLAB) {
 4252     tlab_allocate(thread, new_obj, layout_size, 0, klass, t2, slow_case);
 4253     if (ZeroTLAB || (!clear_fields)) {
 4254       // the fields have been already cleared
 4255       jmp(initialize_header);
 4256     } else {
 4257       // initialize both the header and fields
 4258       jmp(initialize_object);
 4259     }
 4260   } else {
 4261     jmp(slow_case);
 4262   }
 4263 
 4264   // If UseTLAB is true, the object is created above and there is an initialize need.
 4265   // Otherwise, skip and go to the slow path.
 4266   if (UseTLAB) {
 4267     if (clear_fields) {
 4268       // The object is initialized before the header.  If the object size is
 4269       // zero, go directly to the header initialization.
 4270       bind(initialize_object);
 4271       decrement(layout_size, sizeof(oopDesc));
 4272       jcc(Assembler::zero, initialize_header);
 4273 
 4274       // Initialize topmost object field, divide size by 8, check if odd and
 4275       // test if zero.
 4276       Register zero = klass;
 4277       xorl(zero, zero);    // use zero reg to clear memory (shorter code)
 4278       shrl(layout_size, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
 4279 
 4280   #ifdef ASSERT
 4281       // make sure instance_size was multiple of 8
 4282       Label L;
 4283       // Ignore partial flag stall after shrl() since it is debug VM
 4284       jcc(Assembler::carryClear, L);
 4285       stop("object size is not multiple of 2 - adjust this code");
 4286       bind(L);
 4287       // must be > 0, no extra check needed here
 4288   #endif
 4289 
 4290       // initialize remaining object fields: instance_size was a multiple of 8
 4291       {
 4292         Label loop;
 4293         bind(loop);
 4294         movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 1*oopSize), zero);
 4295         NOT_LP64(movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 2*oopSize), zero));
 4296         decrement(layout_size);
 4297         jcc(Assembler::notZero, loop);
 4298       }
 4299     } // clear_fields
 4300 
 4301     // initialize object header only.
 4302     bind(initialize_header);
 4303     pop(klass);
 4304     Register mark_word = t2;
 4305     movptr(mark_word, Address(klass, Klass::prototype_header_offset()));
 4306     movptr(Address(new_obj, oopDesc::mark_offset_in_bytes ()), mark_word);
 4307 #ifdef _LP64
 4308     xorl(rsi, rsi);                 // use zero reg to clear memory (shorter code)
 4309     store_klass_gap(new_obj, rsi);  // zero klass gap for compressed oops
 4310 #endif
 4311     movptr(t2, klass);         // preserve klass
 4312     store_klass(new_obj, t2, rscratch1);  // src klass reg is potentially compressed
 4313 
 4314     jmp(done);
 4315   }
 4316 
 4317   bind(slow_case);
 4318   pop(klass);
 4319   bind(slow_case_no_pop);
 4320   jmp(alloc_failed);
 4321 
 4322   bind(done);
 4323 }
 4324 
 4325 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 4326 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 4327                                    Register var_size_in_bytes,
 4328                                    int con_size_in_bytes,
 4329                                    Register t1,
 4330                                    Register t2,
 4331                                    Label& slow_case) {
 4332   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4333   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 4334 }
 4335 
 4336 RegSet MacroAssembler::call_clobbered_gp_registers() {
 4337   RegSet regs;
 4338 #ifdef _LP64
 4339   regs += RegSet::of(rax, rcx, rdx);
 4340 #ifndef WINDOWS
 4341   regs += RegSet::of(rsi, rdi);
 4342 #endif
 4343   regs += RegSet::range(r8, r11);
 4344 #else
 4345   regs += RegSet::of(rax, rcx, rdx);
 4346 #endif
 4347   return regs;
 4348 }
 4349 
 4350 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() {
 4351   int num_xmm_registers = XMMRegister::available_xmm_registers();
 4352 #if defined(WINDOWS) && defined(_LP64)
 4353   XMMRegSet result = XMMRegSet::range(xmm0, xmm5);
 4354   if (num_xmm_registers > 16) {
 4355      result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1));
 4356   }
 4357   return result;
 4358 #else
 4359   return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1));
 4360 #endif
 4361 }
 4362 
 4363 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor
 4364 
 4365 #ifndef _LP64
 4366 static bool use_x87_registers() { return UseSSE < 2; }
 4367 #endif
 4368 static bool use_xmm_registers() { return UseSSE >= 1; }
 4369 
 4370 // C1 only ever uses the first double/float of the XMM register.
 4371 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); }
 4372 
 4373 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4374   if (UseSSE == 1) {
 4375     masm->movflt(Address(rsp, offset), reg);
 4376   } else {
 4377     masm->movdbl(Address(rsp, offset), reg);
 4378   }
 4379 }
 4380 
 4381 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4382   if (UseSSE == 1) {
 4383     masm->movflt(reg, Address(rsp, offset));
 4384   } else {
 4385     masm->movdbl(reg, Address(rsp, offset));
 4386   }
 4387 }
 4388 
 4389 int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers, bool save_fpu,
 4390                            int& gp_area_size, int& fp_area_size, int& xmm_area_size) {
 4391 
 4392   gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
 4393                          StackAlignmentInBytes);
 4394 #ifdef _LP64
 4395   fp_area_size = 0;
 4396 #else
 4397   fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0;
 4398 #endif
 4399   xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0;
 4400 
 4401   return gp_area_size + fp_area_size + xmm_area_size;
 4402 }
 4403 
 4404 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) {
 4405   block_comment("push_call_clobbered_registers start");
 4406   // Regular registers
 4407   RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude;
 4408 
 4409   int gp_area_size;
 4410   int fp_area_size;
 4411   int xmm_area_size;
 4412   int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu,
 4413                                                gp_area_size, fp_area_size, xmm_area_size);
 4414   subptr(rsp, total_save_size);
 4415 
 4416   push_set(gp_registers_to_push, 0);
 4417 
 4418 #ifndef _LP64
 4419   if (save_fpu && use_x87_registers()) {
 4420     fnsave(Address(rsp, gp_area_size));
 4421     fwait();
 4422   }
 4423 #endif
 4424   if (save_fpu && use_xmm_registers()) {
 4425     push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4426   }
 4427 
 4428   block_comment("push_call_clobbered_registers end");
 4429 }
 4430 
 4431 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) {
 4432   block_comment("pop_call_clobbered_registers start");
 4433 
 4434   RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude;
 4435 
 4436   int gp_area_size;
 4437   int fp_area_size;
 4438   int xmm_area_size;
 4439   int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu,
 4440                                                gp_area_size, fp_area_size, xmm_area_size);
 4441 
 4442   if (restore_fpu && use_xmm_registers()) {
 4443     pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4444   }
 4445 #ifndef _LP64
 4446   if (restore_fpu && use_x87_registers()) {
 4447     frstor(Address(rsp, gp_area_size));
 4448   }
 4449 #endif
 4450 
 4451   pop_set(gp_registers_to_pop, 0);
 4452 
 4453   addptr(rsp, total_save_size);
 4454 
 4455   vzeroupper();
 4456 
 4457   block_comment("pop_call_clobbered_registers end");
 4458 }
 4459 
 4460 void MacroAssembler::push_set(XMMRegSet set, int offset) {
 4461   assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be");
 4462   int spill_offset = offset;
 4463 
 4464   for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) {
 4465     save_xmm_register(this, spill_offset, *it);
 4466     spill_offset += xmm_save_size();
 4467   }
 4468 }
 4469 
 4470 void MacroAssembler::pop_set(XMMRegSet set, int offset) {
 4471   int restore_size = set.size() * xmm_save_size();
 4472   assert(is_aligned(restore_size, StackAlignmentInBytes), "must be");
 4473 
 4474   int restore_offset = offset + restore_size - xmm_save_size();
 4475 
 4476   for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) {
 4477     restore_xmm_register(this, restore_offset, *it);
 4478     restore_offset -= xmm_save_size();
 4479   }
 4480 }
 4481 
 4482 void MacroAssembler::push_set(RegSet set, int offset) {
 4483   int spill_offset;
 4484   if (offset == -1) {
 4485     int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4486     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 4487     subptr(rsp, aligned_size);
 4488     spill_offset = 0;
 4489   } else {
 4490     spill_offset = offset;
 4491   }
 4492 
 4493   for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) {
 4494     movptr(Address(rsp, spill_offset), *it);
 4495     spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4496   }
 4497 }
 4498 
 4499 void MacroAssembler::pop_set(RegSet set, int offset) {
 4500 
 4501   int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4502   int restore_size = set.size() * gp_reg_size;
 4503   int aligned_size = align_up(restore_size, StackAlignmentInBytes);
 4504 
 4505   int restore_offset;
 4506   if (offset == -1) {
 4507     restore_offset = restore_size - gp_reg_size;
 4508   } else {
 4509     restore_offset = offset + restore_size - gp_reg_size;
 4510   }
 4511   for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) {
 4512     movptr(*it, Address(rsp, restore_offset));
 4513     restore_offset -= gp_reg_size;
 4514   }
 4515 
 4516   if (offset == -1) {
 4517     addptr(rsp, aligned_size);
 4518   }
 4519 }
 4520 
 4521 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
 4522 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
 4523   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
 4524   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
 4525   Label done;
 4526 
 4527   testptr(length_in_bytes, length_in_bytes);
 4528   jcc(Assembler::zero, done);
 4529 
 4530   // initialize topmost word, divide index by 2, check if odd and test if zero
 4531   // note: for the remaining code to work, index must be a multiple of BytesPerWord
 4532 #ifdef ASSERT
 4533   {
 4534     Label L;
 4535     testptr(length_in_bytes, BytesPerWord - 1);
 4536     jcc(Assembler::zero, L);
 4537     stop("length must be a multiple of BytesPerWord");
 4538     bind(L);
 4539   }
 4540 #endif
 4541   Register index = length_in_bytes;
 4542   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
 4543   if (UseIncDec) {
 4544     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
 4545   } else {
 4546     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
 4547     shrptr(index, 1);
 4548   }
 4549 #ifndef _LP64
 4550   // index could have not been a multiple of 8 (i.e., bit 2 was set)
 4551   {
 4552     Label even;
 4553     // note: if index was a multiple of 8, then it cannot
 4554     //       be 0 now otherwise it must have been 0 before
 4555     //       => if it is even, we don't need to check for 0 again
 4556     jcc(Assembler::carryClear, even);
 4557     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4558     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4559     // index could be 0 now, must check again
 4560     jcc(Assembler::zero, done);
 4561     bind(even);
 4562   }
 4563 #endif // !_LP64
 4564   // initialize remaining object fields: index is a multiple of 2 now
 4565   {
 4566     Label loop;
 4567     bind(loop);
 4568     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4569     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4570     decrement(index);
 4571     jcc(Assembler::notZero, loop);
 4572   }
 4573 
 4574   bind(done);
 4575 }
 4576 
 4577 void MacroAssembler::get_inline_type_field_klass(Register klass, Register index, Register inline_klass) {
 4578   movptr(inline_klass, Address(klass, InstanceKlass::inline_type_field_klasses_offset()));
 4579 #ifdef ASSERT
 4580   {
 4581     Label done;
 4582     cmpptr(inline_klass, 0);
 4583     jcc(Assembler::notEqual, done);
 4584     stop("get_inline_type_field_klass contains no inline klass");
 4585     bind(done);
 4586   }
 4587 #endif
 4588   movptr(inline_klass, Address(inline_klass, index, Address::times_ptr));
 4589 }
 4590 
 4591 void MacroAssembler::get_default_value_oop(Register inline_klass, Register temp_reg, Register obj) {
 4592 #ifdef ASSERT
 4593   {
 4594     Label done_check;
 4595     test_klass_is_inline_type(inline_klass, temp_reg, done_check);
 4596     stop("get_default_value_oop from non inline type klass");
 4597     bind(done_check);
 4598   }
 4599 #endif
 4600   Register offset = temp_reg;
 4601   // Getting the offset of the pre-allocated default value
 4602   movptr(offset, Address(inline_klass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())));
 4603   movl(offset, Address(offset, in_bytes(InlineKlass::default_value_offset_offset())));
 4604 
 4605   // Getting the mirror
 4606   movptr(obj, Address(inline_klass, in_bytes(Klass::java_mirror_offset())));
 4607   resolve_oop_handle(obj, inline_klass);
 4608 
 4609   // Getting the pre-allocated default value from the mirror
 4610   Address field(obj, offset, Address::times_1);
 4611   load_heap_oop(obj, field);
 4612 }
 4613 
 4614 void MacroAssembler::get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj) {
 4615 #ifdef ASSERT
 4616   {
 4617     Label done_check;
 4618     test_klass_is_empty_inline_type(inline_klass, temp_reg, done_check);
 4619     stop("get_empty_value from non-empty inline klass");
 4620     bind(done_check);
 4621   }
 4622 #endif
 4623   get_default_value_oop(inline_klass, temp_reg, obj);
 4624 }
 4625 
 4626 
 4627 // Look up the method for a megamorphic invokeinterface call.
 4628 // The target method is determined by <intf_klass, itable_index>.
 4629 // The receiver klass is in recv_klass.
 4630 // On success, the result will be in method_result, and execution falls through.
 4631 // On failure, execution transfers to the given label.
 4632 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4633                                              Register intf_klass,
 4634                                              RegisterOrConstant itable_index,
 4635                                              Register method_result,
 4636                                              Register scan_temp,
 4637                                              Label& L_no_such_interface,
 4638                                              bool return_method) {
 4639   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4640   assert_different_registers(method_result, intf_klass, scan_temp);
 4641   assert(recv_klass != method_result || !return_method,
 4642          "recv_klass can be destroyed when method isn't needed");
 4643 
 4644   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4645          "caller must use same register for non-constant itable index as for method");
 4646 
 4647   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
 4648   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4649   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4650   int scan_step   = itableOffsetEntry::size() * wordSize;
 4651   int vte_size    = vtableEntry::size_in_bytes();
 4652   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4653   assert(vte_size == wordSize, "else adjust times_vte_scale");
 4654 
 4655   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4656 
 4657   // %%% Could store the aligned, prescaled offset in the klassoop.
 4658   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
 4659 
 4660   if (return_method) {
 4661     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
 4662     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4663     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
 4664   }
 4665 
 4666   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
 4667   //   if (scan->interface() == intf) {
 4668   //     result = (klass + scan->offset() + itable_index);
 4669   //   }
 4670   // }
 4671   Label search, found_method;
 4672 
 4673   for (int peel = 1; peel >= 0; peel--) {
 4674     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset()));
 4675     cmpptr(intf_klass, method_result);
 4676 
 4677     if (peel) {
 4678       jccb(Assembler::equal, found_method);
 4679     } else {
 4680       jccb(Assembler::notEqual, search);
 4681       // (invert the test to fall through to found_method...)
 4682     }
 4683 
 4684     if (!peel)  break;
 4685 
 4686     bind(search);
 4687 
 4688     // Check that the previous entry is non-null.  A null entry means that
 4689     // the receiver class doesn't implement the interface, and wasn't the
 4690     // same as when the caller was compiled.
 4691     testptr(method_result, method_result);
 4692     jcc(Assembler::zero, L_no_such_interface);
 4693     addptr(scan_temp, scan_step);
 4694   }
 4695 
 4696   bind(found_method);
 4697 
 4698   if (return_method) {
 4699     // Got a hit.
 4700     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset()));
 4701     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
 4702   }
 4703 }
 4704 
 4705 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
 4706 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
 4707 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
 4708 // The target method is determined by <holder_klass, itable_index>.
 4709 // The receiver klass is in recv_klass.
 4710 // On success, the result will be in method_result, and execution falls through.
 4711 // On failure, execution transfers to the given label.
 4712 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
 4713                                                   Register holder_klass,
 4714                                                   Register resolved_klass,
 4715                                                   Register method_result,
 4716                                                   Register scan_temp,
 4717                                                   Register temp_reg2,
 4718                                                   Register receiver,
 4719                                                   int itable_index,
 4720                                                   Label& L_no_such_interface) {
 4721   assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver);
 4722   Register temp_itbl_klass = method_result;
 4723   Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl
 4724 
 4725   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4726   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4727   int scan_step = itableOffsetEntry::size() * wordSize;
 4728   int vte_size = vtableEntry::size_in_bytes();
 4729   int ioffset = in_bytes(itableOffsetEntry::interface_offset());
 4730   int ooffset = in_bytes(itableOffsetEntry::offset_offset());
 4731   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4732   assert(vte_size == wordSize, "adjust times_vte_scale");
 4733 
 4734   Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found;
 4735 
 4736   // temp_itbl_klass = recv_klass.itable[0]
 4737   // scan_temp = &recv_klass.itable[0] + step
 4738   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4739   movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset));
 4740   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step));
 4741   xorptr(temp_reg, temp_reg);
 4742 
 4743   // Initial checks:
 4744   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
 4745   //   - if (itable[0] == 0), no such interface
 4746   //   - if (itable[0] == holder_klass), shortcut to "holder found"
 4747   cmpptr(holder_klass, resolved_klass);
 4748   jccb(Assembler::notEqual, L_loop_scan_resolved_entry);
 4749   testptr(temp_itbl_klass, temp_itbl_klass);
 4750   jccb(Assembler::zero, L_no_such_interface);
 4751   cmpptr(holder_klass, temp_itbl_klass);
 4752   jccb(Assembler::equal, L_holder_found);
 4753 
 4754   // Loop: Look for holder_klass record in itable
 4755   //   do {
 4756   //     tmp = itable[index];
 4757   //     index += step;
 4758   //     if (tmp == holder_klass) {
 4759   //       goto L_holder_found; // Found!
 4760   //     }
 4761   //   } while (tmp != 0);
 4762   //   goto L_no_such_interface // Not found.
 4763   Label L_scan_holder;
 4764   bind(L_scan_holder);
 4765     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4766     addptr(scan_temp, scan_step);
 4767     cmpptr(holder_klass, temp_itbl_klass);
 4768     jccb(Assembler::equal, L_holder_found);
 4769     testptr(temp_itbl_klass, temp_itbl_klass);
 4770     jccb(Assembler::notZero, L_scan_holder);
 4771 
 4772   jmpb(L_no_such_interface);
 4773 
 4774   // Loop: Look for resolved_class record in itable
 4775   //   do {
 4776   //     tmp = itable[index];
 4777   //     index += step;
 4778   //     if (tmp == holder_klass) {
 4779   //        // Also check if we have met a holder klass
 4780   //        holder_tmp = itable[index-step-ioffset];
 4781   //     }
 4782   //     if (tmp == resolved_klass) {
 4783   //        goto L_resolved_found;  // Found!
 4784   //     }
 4785   //   } while (tmp != 0);
 4786   //   goto L_no_such_interface // Not found.
 4787   //
 4788   Label L_loop_scan_resolved;
 4789   bind(L_loop_scan_resolved);
 4790     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4791     addptr(scan_temp, scan_step);
 4792     bind(L_loop_scan_resolved_entry);
 4793     cmpptr(holder_klass, temp_itbl_klass);
 4794     cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4795     cmpptr(resolved_klass, temp_itbl_klass);
 4796     jccb(Assembler::equal, L_resolved_found);
 4797     testptr(temp_itbl_klass, temp_itbl_klass);
 4798     jccb(Assembler::notZero, L_loop_scan_resolved);
 4799 
 4800   jmpb(L_no_such_interface);
 4801 
 4802   Label L_ready;
 4803 
 4804   // See if we already have a holder klass. If not, go and scan for it.
 4805   bind(L_resolved_found);
 4806   testptr(temp_reg, temp_reg);
 4807   jccb(Assembler::zero, L_scan_holder);
 4808   jmpb(L_ready);
 4809 
 4810   bind(L_holder_found);
 4811   movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4812 
 4813   // Finally, temp_reg contains holder_klass vtable offset
 4814   bind(L_ready);
 4815   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4816   if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl
 4817     load_klass(scan_temp, receiver, noreg);
 4818     movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4819   } else {
 4820     movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4821   }
 4822 }
 4823 
 4824 
 4825 // virtual method calling
 4826 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 4827                                            RegisterOrConstant vtable_index,
 4828                                            Register method_result) {
 4829   const ByteSize base = Klass::vtable_start_offset();
 4830   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
 4831   Address vtable_entry_addr(recv_klass,
 4832                             vtable_index, Address::times_ptr,
 4833                             base + vtableEntry::method_offset());
 4834   movptr(method_result, vtable_entry_addr);
 4835 }
 4836 
 4837 
 4838 void MacroAssembler::check_klass_subtype(Register sub_klass,
 4839                            Register super_klass,
 4840                            Register temp_reg,
 4841                            Label& L_success) {
 4842   Label L_failure;
 4843   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, nullptr);
 4844   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr);
 4845   bind(L_failure);
 4846 }
 4847 
 4848 
 4849 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
 4850                                                    Register super_klass,
 4851                                                    Register temp_reg,
 4852                                                    Label* L_success,
 4853                                                    Label* L_failure,
 4854                                                    Label* L_slow_path,
 4855                                         RegisterOrConstant super_check_offset) {
 4856   assert_different_registers(sub_klass, super_klass, temp_reg);
 4857   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
 4858   if (super_check_offset.is_register()) {
 4859     assert_different_registers(sub_klass, super_klass,
 4860                                super_check_offset.as_register());
 4861   } else if (must_load_sco) {
 4862     assert(temp_reg != noreg, "supply either a temp or a register offset");
 4863   }
 4864 
 4865   Label L_fallthrough;
 4866   int label_nulls = 0;
 4867   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4868   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4869   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
 4870   assert(label_nulls <= 1, "at most one null in the batch");
 4871 
 4872   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4873   int sco_offset = in_bytes(Klass::super_check_offset_offset());
 4874   Address super_check_offset_addr(super_klass, sco_offset);
 4875 
 4876   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
 4877   // range of a jccb.  If this routine grows larger, reconsider at
 4878   // least some of these.
 4879 #define local_jcc(assembler_cond, label)                                \
 4880   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
 4881   else                             jcc( assembler_cond, label) /*omit semi*/
 4882 
 4883   // Hacked jmp, which may only be used just before L_fallthrough.
 4884 #define final_jmp(label)                                                \
 4885   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
 4886   else                            jmp(label)                /*omit semi*/
 4887 
 4888   // If the pointers are equal, we are done (e.g., String[] elements).
 4889   // This self-check enables sharing of secondary supertype arrays among
 4890   // non-primary types such as array-of-interface.  Otherwise, each such
 4891   // type would need its own customized SSA.
 4892   // We move this check to the front of the fast path because many
 4893   // type checks are in fact trivially successful in this manner,
 4894   // so we get a nicely predicted branch right at the start of the check.
 4895   cmpptr(sub_klass, super_klass);
 4896   local_jcc(Assembler::equal, *L_success);
 4897 
 4898   // Check the supertype display:
 4899   if (must_load_sco) {
 4900     // Positive movl does right thing on LP64.
 4901     movl(temp_reg, super_check_offset_addr);
 4902     super_check_offset = RegisterOrConstant(temp_reg);
 4903   }
 4904   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
 4905   cmpptr(super_klass, super_check_addr); // load displayed supertype
 4906 
 4907   // This check has worked decisively for primary supers.
 4908   // Secondary supers are sought in the super_cache ('super_cache_addr').
 4909   // (Secondary supers are interfaces and very deeply nested subtypes.)
 4910   // This works in the same check above because of a tricky aliasing
 4911   // between the super_cache and the primary super display elements.
 4912   // (The 'super_check_addr' can address either, as the case requires.)
 4913   // Note that the cache is updated below if it does not help us find
 4914   // what we need immediately.
 4915   // So if it was a primary super, we can just fail immediately.
 4916   // Otherwise, it's the slow path for us (no success at this point).
 4917 
 4918   if (super_check_offset.is_register()) {
 4919     local_jcc(Assembler::equal, *L_success);
 4920     cmpl(super_check_offset.as_register(), sc_offset);
 4921     if (L_failure == &L_fallthrough) {
 4922       local_jcc(Assembler::equal, *L_slow_path);
 4923     } else {
 4924       local_jcc(Assembler::notEqual, *L_failure);
 4925       final_jmp(*L_slow_path);
 4926     }
 4927   } else if (super_check_offset.as_constant() == sc_offset) {
 4928     // Need a slow path; fast failure is impossible.
 4929     if (L_slow_path == &L_fallthrough) {
 4930       local_jcc(Assembler::equal, *L_success);
 4931     } else {
 4932       local_jcc(Assembler::notEqual, *L_slow_path);
 4933       final_jmp(*L_success);
 4934     }
 4935   } else {
 4936     // No slow path; it's a fast decision.
 4937     if (L_failure == &L_fallthrough) {
 4938       local_jcc(Assembler::equal, *L_success);
 4939     } else {
 4940       local_jcc(Assembler::notEqual, *L_failure);
 4941       final_jmp(*L_success);
 4942     }
 4943   }
 4944 
 4945   bind(L_fallthrough);
 4946 
 4947 #undef local_jcc
 4948 #undef final_jmp
 4949 }
 4950 
 4951 
 4952 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4953                                                    Register super_klass,
 4954                                                    Register temp_reg,
 4955                                                    Register temp2_reg,
 4956                                                    Label* L_success,
 4957                                                    Label* L_failure,
 4958                                                    bool set_cond_codes) {
 4959   assert_different_registers(sub_klass, super_klass, temp_reg);
 4960   if (temp2_reg != noreg)
 4961     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
 4962 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
 4963 
 4964   Label L_fallthrough;
 4965   int label_nulls = 0;
 4966   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4967   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4968   assert(label_nulls <= 1, "at most one null in the batch");
 4969 
 4970   // a couple of useful fields in sub_klass:
 4971   int ss_offset = in_bytes(Klass::secondary_supers_offset());
 4972   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4973   Address secondary_supers_addr(sub_klass, ss_offset);
 4974   Address super_cache_addr(     sub_klass, sc_offset);
 4975 
 4976   // Do a linear scan of the secondary super-klass chain.
 4977   // This code is rarely used, so simplicity is a virtue here.
 4978   // The repne_scan instruction uses fixed registers, which we must spill.
 4979   // Don't worry too much about pre-existing connections with the input regs.
 4980 
 4981   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
 4982   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
 4983 
 4984   // Get super_klass value into rax (even if it was in rdi or rcx).
 4985   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
 4986   if (super_klass != rax) {
 4987     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
 4988     mov(rax, super_klass);
 4989   }
 4990   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
 4991   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
 4992 
 4993 #ifndef PRODUCT
 4994   uint* pst_counter = &SharedRuntime::_partial_subtype_ctr;
 4995   ExternalAddress pst_counter_addr((address) pst_counter);
 4996   NOT_LP64(  incrementl(pst_counter_addr) );
 4997   LP64_ONLY( lea(rcx, pst_counter_addr) );
 4998   LP64_ONLY( incrementl(Address(rcx, 0)) );
 4999 #endif //PRODUCT
 5000 
 5001   // We will consult the secondary-super array.
 5002   movptr(rdi, secondary_supers_addr);
 5003   // Load the array length.  (Positive movl does right thing on LP64.)
 5004   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
 5005   // Skip to start of data.
 5006   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
 5007 
 5008   // Scan RCX words at [RDI] for an occurrence of RAX.
 5009   // Set NZ/Z based on last compare.
 5010   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
 5011   // not change flags (only scas instruction which is repeated sets flags).
 5012   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
 5013 
 5014     testptr(rax,rax); // Set Z = 0
 5015     repne_scan();
 5016 
 5017   // Unspill the temp. registers:
 5018   if (pushed_rdi)  pop(rdi);
 5019   if (pushed_rcx)  pop(rcx);
 5020   if (pushed_rax)  pop(rax);
 5021 
 5022   if (set_cond_codes) {
 5023     // Special hack for the AD files:  rdi is guaranteed non-zero.
 5024     assert(!pushed_rdi, "rdi must be left non-null");
 5025     // Also, the condition codes are properly set Z/NZ on succeed/failure.
 5026   }
 5027 
 5028   if (L_failure == &L_fallthrough)
 5029         jccb(Assembler::notEqual, *L_failure);
 5030   else  jcc(Assembler::notEqual, *L_failure);
 5031 
 5032   // Success.  Cache the super we found and proceed in triumph.
 5033   movptr(super_cache_addr, super_klass);
 5034 
 5035   if (L_success != &L_fallthrough) {
 5036     jmp(*L_success);
 5037   }
 5038 
 5039 #undef IS_A_TEMP
 5040 
 5041   bind(L_fallthrough);
 5042 }
 5043 
 5044 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
 5045   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
 5046 
 5047   Label L_fallthrough;
 5048   if (L_fast_path == nullptr) {
 5049     L_fast_path = &L_fallthrough;
 5050   } else if (L_slow_path == nullptr) {
 5051     L_slow_path = &L_fallthrough;
 5052   }
 5053 
 5054   // Fast path check: class is fully initialized
 5055   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
 5056   jcc(Assembler::equal, *L_fast_path);
 5057 
 5058   // Fast path check: current thread is initializer thread
 5059   cmpptr(thread, Address(klass, InstanceKlass::init_thread_offset()));
 5060   if (L_slow_path == &L_fallthrough) {
 5061     jcc(Assembler::equal, *L_fast_path);
 5062     bind(*L_slow_path);
 5063   } else if (L_fast_path == &L_fallthrough) {
 5064     jcc(Assembler::notEqual, *L_slow_path);
 5065     bind(*L_fast_path);
 5066   } else {
 5067     Unimplemented();
 5068   }
 5069 }
 5070 
 5071 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
 5072   if (VM_Version::supports_cmov()) {
 5073     cmovl(cc, dst, src);
 5074   } else {
 5075     Label L;
 5076     jccb(negate_condition(cc), L);
 5077     movl(dst, src);
 5078     bind(L);
 5079   }
 5080 }
 5081 
 5082 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 5083   if (VM_Version::supports_cmov()) {
 5084     cmovl(cc, dst, src);
 5085   } else {
 5086     Label L;
 5087     jccb(negate_condition(cc), L);
 5088     movl(dst, src);
 5089     bind(L);
 5090   }
 5091 }
 5092 
 5093 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 5094   if (!VerifyOops || VerifyAdapterSharing) {
 5095     // Below address of the code string confuses VerifyAdapterSharing
 5096     // because it may differ between otherwise equivalent adapters.
 5097     return;
 5098   }
 5099 
 5100   BLOCK_COMMENT("verify_oop {");
 5101 #ifdef _LP64
 5102   push(rscratch1);
 5103 #endif
 5104   push(rax);                          // save rax
 5105   push(reg);                          // pass register argument
 5106 
 5107   // Pass register number to verify_oop_subroutine
 5108   const char* b = nullptr;
 5109   {
 5110     ResourceMark rm;
 5111     stringStream ss;
 5112     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 5113     b = code_string(ss.as_string());
 5114   }
 5115   ExternalAddress buffer((address) b);
 5116   pushptr(buffer.addr(), rscratch1);
 5117 
 5118   // call indirectly to solve generation ordering problem
 5119   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5120   call(rax);
 5121   // Caller pops the arguments (oop, message) and restores rax, r10
 5122   BLOCK_COMMENT("} verify_oop");
 5123 }
 5124 
 5125 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
 5126   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
 5127     // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without
 5128     // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog
 5129     vpternlogd(dst, 0xFF, dst, dst, vector_len);
 5130   } else if (VM_Version::supports_avx()) {
 5131     vpcmpeqd(dst, dst, dst, vector_len);
 5132   } else {
 5133     assert(VM_Version::supports_sse2(), "");
 5134     pcmpeqd(dst, dst);
 5135   }
 5136 }
 5137 
 5138 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
 5139                                          int extra_slot_offset) {
 5140   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 5141   int stackElementSize = Interpreter::stackElementSize;
 5142   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 5143 #ifdef ASSERT
 5144   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 5145   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 5146 #endif
 5147   Register             scale_reg    = noreg;
 5148   Address::ScaleFactor scale_factor = Address::no_scale;
 5149   if (arg_slot.is_constant()) {
 5150     offset += arg_slot.as_constant() * stackElementSize;
 5151   } else {
 5152     scale_reg    = arg_slot.as_register();
 5153     scale_factor = Address::times(stackElementSize);
 5154   }
 5155   offset += wordSize;           // return PC is on stack
 5156   return Address(rsp, scale_reg, scale_factor, offset);
 5157 }
 5158 
 5159 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 5160   if (!VerifyOops || VerifyAdapterSharing) {
 5161     // Below address of the code string confuses VerifyAdapterSharing
 5162     // because it may differ between otherwise equivalent adapters.
 5163     return;
 5164   }
 5165 
 5166 #ifdef _LP64
 5167   push(rscratch1);
 5168 #endif
 5169   push(rax); // save rax,
 5170   // addr may contain rsp so we will have to adjust it based on the push
 5171   // we just did (and on 64 bit we do two pushes)
 5172   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 5173   // stores rax into addr which is backwards of what was intended.
 5174   if (addr.uses(rsp)) {
 5175     lea(rax, addr);
 5176     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
 5177   } else {
 5178     pushptr(addr);
 5179   }
 5180 
 5181   // Pass register number to verify_oop_subroutine
 5182   const char* b = nullptr;
 5183   {
 5184     ResourceMark rm;
 5185     stringStream ss;
 5186     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 5187     b = code_string(ss.as_string());
 5188   }
 5189   ExternalAddress buffer((address) b);
 5190   pushptr(buffer.addr(), rscratch1);
 5191 
 5192   // call indirectly to solve generation ordering problem
 5193   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5194   call(rax);
 5195   // Caller pops the arguments (addr, message) and restores rax, r10.
 5196 }
 5197 
 5198 void MacroAssembler::verify_tlab() {
 5199 #ifdef ASSERT
 5200   if (UseTLAB && VerifyOops) {
 5201     Label next, ok;
 5202     Register t1 = rsi;
 5203     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
 5204 
 5205     push(t1);
 5206     NOT_LP64(push(thread_reg));
 5207     NOT_LP64(get_thread(thread_reg));
 5208 
 5209     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5210     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
 5211     jcc(Assembler::aboveEqual, next);
 5212     STOP("assert(top >= start)");
 5213     should_not_reach_here();
 5214 
 5215     bind(next);
 5216     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
 5217     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5218     jcc(Assembler::aboveEqual, ok);
 5219     STOP("assert(top <= end)");
 5220     should_not_reach_here();
 5221 
 5222     bind(ok);
 5223     NOT_LP64(pop(thread_reg));
 5224     pop(t1);
 5225   }
 5226 #endif
 5227 }
 5228 
 5229 class ControlWord {
 5230  public:
 5231   int32_t _value;
 5232 
 5233   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
 5234   int  precision_control() const       { return  (_value >>  8) & 3      ; }
 5235   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5236   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5237   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5238   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5239   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5240   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5241 
 5242   void print() const {
 5243     // rounding control
 5244     const char* rc;
 5245     switch (rounding_control()) {
 5246       case 0: rc = "round near"; break;
 5247       case 1: rc = "round down"; break;
 5248       case 2: rc = "round up  "; break;
 5249       case 3: rc = "chop      "; break;
 5250       default:
 5251         rc = nullptr; // silence compiler warnings
 5252         fatal("Unknown rounding control: %d", rounding_control());
 5253     };
 5254     // precision control
 5255     const char* pc;
 5256     switch (precision_control()) {
 5257       case 0: pc = "24 bits "; break;
 5258       case 1: pc = "reserved"; break;
 5259       case 2: pc = "53 bits "; break;
 5260       case 3: pc = "64 bits "; break;
 5261       default:
 5262         pc = nullptr; // silence compiler warnings
 5263         fatal("Unknown precision control: %d", precision_control());
 5264     };
 5265     // flags
 5266     char f[9];
 5267     f[0] = ' ';
 5268     f[1] = ' ';
 5269     f[2] = (precision   ()) ? 'P' : 'p';
 5270     f[3] = (underflow   ()) ? 'U' : 'u';
 5271     f[4] = (overflow    ()) ? 'O' : 'o';
 5272     f[5] = (zero_divide ()) ? 'Z' : 'z';
 5273     f[6] = (denormalized()) ? 'D' : 'd';
 5274     f[7] = (invalid     ()) ? 'I' : 'i';
 5275     f[8] = '\x0';
 5276     // output
 5277     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
 5278   }
 5279 
 5280 };
 5281 
 5282 class StatusWord {
 5283  public:
 5284   int32_t _value;
 5285 
 5286   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
 5287   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
 5288   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
 5289   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
 5290   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
 5291   int  top() const                     { return  (_value >> 11) & 7      ; }
 5292   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
 5293   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
 5294   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5295   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5296   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5297   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5298   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5299   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5300 
 5301   void print() const {
 5302     // condition codes
 5303     char c[5];
 5304     c[0] = (C3()) ? '3' : '-';
 5305     c[1] = (C2()) ? '2' : '-';
 5306     c[2] = (C1()) ? '1' : '-';
 5307     c[3] = (C0()) ? '0' : '-';
 5308     c[4] = '\x0';
 5309     // flags
 5310     char f[9];
 5311     f[0] = (error_status()) ? 'E' : '-';
 5312     f[1] = (stack_fault ()) ? 'S' : '-';
 5313     f[2] = (precision   ()) ? 'P' : '-';
 5314     f[3] = (underflow   ()) ? 'U' : '-';
 5315     f[4] = (overflow    ()) ? 'O' : '-';
 5316     f[5] = (zero_divide ()) ? 'Z' : '-';
 5317     f[6] = (denormalized()) ? 'D' : '-';
 5318     f[7] = (invalid     ()) ? 'I' : '-';
 5319     f[8] = '\x0';
 5320     // output
 5321     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
 5322   }
 5323 
 5324 };
 5325 
 5326 class TagWord {
 5327  public:
 5328   int32_t _value;
 5329 
 5330   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
 5331 
 5332   void print() const {
 5333     printf("%04x", _value & 0xFFFF);
 5334   }
 5335 
 5336 };
 5337 
 5338 class FPU_Register {
 5339  public:
 5340   int32_t _m0;
 5341   int32_t _m1;
 5342   int16_t _ex;
 5343 
 5344   bool is_indefinite() const           {
 5345     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
 5346   }
 5347 
 5348   void print() const {
 5349     char  sign = (_ex < 0) ? '-' : '+';
 5350     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
 5351     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
 5352   };
 5353 
 5354 };
 5355 
 5356 class FPU_State {
 5357  public:
 5358   enum {
 5359     register_size       = 10,
 5360     number_of_registers =  8,
 5361     register_mask       =  7
 5362   };
 5363 
 5364   ControlWord  _control_word;
 5365   StatusWord   _status_word;
 5366   TagWord      _tag_word;
 5367   int32_t      _error_offset;
 5368   int32_t      _error_selector;
 5369   int32_t      _data_offset;
 5370   int32_t      _data_selector;
 5371   int8_t       _register[register_size * number_of_registers];
 5372 
 5373   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
 5374   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
 5375 
 5376   const char* tag_as_string(int tag) const {
 5377     switch (tag) {
 5378       case 0: return "valid";
 5379       case 1: return "zero";
 5380       case 2: return "special";
 5381       case 3: return "empty";
 5382     }
 5383     ShouldNotReachHere();
 5384     return nullptr;
 5385   }
 5386 
 5387   void print() const {
 5388     // print computation registers
 5389     { int t = _status_word.top();
 5390       for (int i = 0; i < number_of_registers; i++) {
 5391         int j = (i - t) & register_mask;
 5392         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
 5393         st(j)->print();
 5394         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
 5395       }
 5396     }
 5397     printf("\n");
 5398     // print control registers
 5399     printf("ctrl = "); _control_word.print(); printf("\n");
 5400     printf("stat = "); _status_word .print(); printf("\n");
 5401     printf("tags = "); _tag_word    .print(); printf("\n");
 5402   }
 5403 
 5404 };
 5405 
 5406 class Flag_Register {
 5407  public:
 5408   int32_t _value;
 5409 
 5410   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
 5411   bool direction() const               { return ((_value >> 10) & 1) != 0; }
 5412   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
 5413   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
 5414   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
 5415   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
 5416   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
 5417 
 5418   void print() const {
 5419     // flags
 5420     char f[8];
 5421     f[0] = (overflow       ()) ? 'O' : '-';
 5422     f[1] = (direction      ()) ? 'D' : '-';
 5423     f[2] = (sign           ()) ? 'S' : '-';
 5424     f[3] = (zero           ()) ? 'Z' : '-';
 5425     f[4] = (auxiliary_carry()) ? 'A' : '-';
 5426     f[5] = (parity         ()) ? 'P' : '-';
 5427     f[6] = (carry          ()) ? 'C' : '-';
 5428     f[7] = '\x0';
 5429     // output
 5430     printf("%08x  flags = %s", _value, f);
 5431   }
 5432 
 5433 };
 5434 
 5435 class IU_Register {
 5436  public:
 5437   int32_t _value;
 5438 
 5439   void print() const {
 5440     printf("%08x  %11d", _value, _value);
 5441   }
 5442 
 5443 };
 5444 
 5445 class IU_State {
 5446  public:
 5447   Flag_Register _eflags;
 5448   IU_Register   _rdi;
 5449   IU_Register   _rsi;
 5450   IU_Register   _rbp;
 5451   IU_Register   _rsp;
 5452   IU_Register   _rbx;
 5453   IU_Register   _rdx;
 5454   IU_Register   _rcx;
 5455   IU_Register   _rax;
 5456 
 5457   void print() const {
 5458     // computation registers
 5459     printf("rax,  = "); _rax.print(); printf("\n");
 5460     printf("rbx,  = "); _rbx.print(); printf("\n");
 5461     printf("rcx  = "); _rcx.print(); printf("\n");
 5462     printf("rdx  = "); _rdx.print(); printf("\n");
 5463     printf("rdi  = "); _rdi.print(); printf("\n");
 5464     printf("rsi  = "); _rsi.print(); printf("\n");
 5465     printf("rbp,  = "); _rbp.print(); printf("\n");
 5466     printf("rsp  = "); _rsp.print(); printf("\n");
 5467     printf("\n");
 5468     // control registers
 5469     printf("flgs = "); _eflags.print(); printf("\n");
 5470   }
 5471 };
 5472 
 5473 
 5474 class CPU_State {
 5475  public:
 5476   FPU_State _fpu_state;
 5477   IU_State  _iu_state;
 5478 
 5479   void print() const {
 5480     printf("--------------------------------------------------\n");
 5481     _iu_state .print();
 5482     printf("\n");
 5483     _fpu_state.print();
 5484     printf("--------------------------------------------------\n");
 5485   }
 5486 
 5487 };
 5488 
 5489 
 5490 static void _print_CPU_state(CPU_State* state) {
 5491   state->print();
 5492 };
 5493 
 5494 
 5495 void MacroAssembler::print_CPU_state() {
 5496   push_CPU_state();
 5497   push(rsp);                // pass CPU state
 5498   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
 5499   addptr(rsp, wordSize);       // discard argument
 5500   pop_CPU_state();
 5501 }
 5502 
 5503 
 5504 #ifndef _LP64
 5505 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
 5506   static int counter = 0;
 5507   FPU_State* fs = &state->_fpu_state;
 5508   counter++;
 5509   // For leaf calls, only verify that the top few elements remain empty.
 5510   // We only need 1 empty at the top for C2 code.
 5511   if( stack_depth < 0 ) {
 5512     if( fs->tag_for_st(7) != 3 ) {
 5513       printf("FPR7 not empty\n");
 5514       state->print();
 5515       assert(false, "error");
 5516       return false;
 5517     }
 5518     return true;                // All other stack states do not matter
 5519   }
 5520 
 5521   assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(),
 5522          "bad FPU control word");
 5523 
 5524   // compute stack depth
 5525   int i = 0;
 5526   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
 5527   int d = i;
 5528   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
 5529   // verify findings
 5530   if (i != FPU_State::number_of_registers) {
 5531     // stack not contiguous
 5532     printf("%s: stack not contiguous at ST%d\n", s, i);
 5533     state->print();
 5534     assert(false, "error");
 5535     return false;
 5536   }
 5537   // check if computed stack depth corresponds to expected stack depth
 5538   if (stack_depth < 0) {
 5539     // expected stack depth is -stack_depth or less
 5540     if (d > -stack_depth) {
 5541       // too many elements on the stack
 5542       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
 5543       state->print();
 5544       assert(false, "error");
 5545       return false;
 5546     }
 5547   } else {
 5548     // expected stack depth is stack_depth
 5549     if (d != stack_depth) {
 5550       // wrong stack depth
 5551       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
 5552       state->print();
 5553       assert(false, "error");
 5554       return false;
 5555     }
 5556   }
 5557   // everything is cool
 5558   return true;
 5559 }
 5560 
 5561 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
 5562   if (!VerifyFPU) return;
 5563   push_CPU_state();
 5564   push(rsp);                // pass CPU state
 5565   ExternalAddress msg((address) s);
 5566   // pass message string s
 5567   pushptr(msg.addr(), noreg);
 5568   push(stack_depth);        // pass stack depth
 5569   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
 5570   addptr(rsp, 3 * wordSize);   // discard arguments
 5571   // check for error
 5572   { Label L;
 5573     testl(rax, rax);
 5574     jcc(Assembler::notZero, L);
 5575     int3();                  // break if error condition
 5576     bind(L);
 5577   }
 5578   pop_CPU_state();
 5579 }
 5580 #endif // _LP64
 5581 
 5582 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) {
 5583   // Either restore the MXCSR register after returning from the JNI Call
 5584   // or verify that it wasn't changed (with -Xcheck:jni flag).
 5585   if (VM_Version::supports_sse()) {
 5586     if (RestoreMXCSROnJNICalls) {
 5587       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch);
 5588     } else if (CheckJNICalls) {
 5589       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
 5590     }
 5591   }
 5592   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
 5593   vzeroupper();
 5594 
 5595 #ifndef _LP64
 5596   // Either restore the x87 floating pointer control word after returning
 5597   // from the JNI call or verify that it wasn't changed.
 5598   if (CheckJNICalls) {
 5599     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
 5600   }
 5601 #endif // _LP64
 5602 }
 5603 
 5604 // ((OopHandle)result).resolve();
 5605 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
 5606   assert_different_registers(result, tmp);
 5607 
 5608   // Only 64 bit platforms support GCs that require a tmp register
 5609   // Only IN_HEAP loads require a thread_tmp register
 5610   // OopHandle::resolve is an indirection like jobject.
 5611   access_load_at(T_OBJECT, IN_NATIVE,
 5612                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
 5613 }
 5614 
 5615 // ((WeakHandle)result).resolve();
 5616 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
 5617   assert_different_registers(rresult, rtmp);
 5618   Label resolved;
 5619 
 5620   // A null weak handle resolves to null.
 5621   cmpptr(rresult, 0);
 5622   jcc(Assembler::equal, resolved);
 5623 
 5624   // Only 64 bit platforms support GCs that require a tmp register
 5625   // Only IN_HEAP loads require a thread_tmp register
 5626   // WeakHandle::resolve is an indirection like jweak.
 5627   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 5628                  rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg);
 5629   bind(resolved);
 5630 }
 5631 
 5632 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 5633   // get mirror
 5634   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 5635   load_method_holder(mirror, method);
 5636   movptr(mirror, Address(mirror, mirror_offset));
 5637   resolve_oop_handle(mirror, tmp);
 5638 }
 5639 
 5640 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 5641   load_method_holder(rresult, rmethod);
 5642   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 5643 }
 5644 
 5645 void MacroAssembler::load_method_holder(Register holder, Register method) {
 5646   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 5647   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 5648   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
 5649 }
 5650 
 5651 void MacroAssembler::load_metadata(Register dst, Register src) {
 5652   if (UseCompressedClassPointers) {
 5653     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5654   } else {
 5655     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5656   }
 5657 }
 5658 
 5659 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 5660   assert_different_registers(src, tmp);
 5661   assert_different_registers(dst, tmp);
 5662 #ifdef _LP64
 5663   if (UseCompressedClassPointers) {
 5664     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5665     decode_klass_not_null(dst, tmp);
 5666   } else
 5667 #endif
 5668   movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5669 }
 5670 
 5671 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
 5672   load_klass(dst, src, tmp);
 5673   movptr(dst, Address(dst, Klass::prototype_header_offset()));
 5674 }
 5675 
 5676 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 5677   assert_different_registers(src, tmp);
 5678   assert_different_registers(dst, tmp);
 5679 #ifdef _LP64
 5680   if (UseCompressedClassPointers) {
 5681     encode_klass_not_null(src, tmp);
 5682     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5683   } else
 5684 #endif
 5685     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5686 }
 5687 
 5688 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 5689                                     Register tmp1, Register thread_tmp) {
 5690   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5691   decorators = AccessInternal::decorator_fixup(decorators, type);
 5692   bool as_raw = (decorators & AS_RAW) != 0;
 5693   if (as_raw) {
 5694     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5695   } else {
 5696     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5697   }
 5698 }
 5699 
 5700 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 5701                                      Register tmp1, Register tmp2, Register tmp3) {
 5702   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5703   decorators = AccessInternal::decorator_fixup(decorators, type);
 5704   bool as_raw = (decorators & AS_RAW) != 0;
 5705   if (as_raw) {
 5706     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5707   } else {
 5708     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5709   }
 5710 }
 5711 
 5712 void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst,
 5713                                        Register inline_klass) {
 5714   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5715   bs->value_copy(this, decorators, src, dst, inline_klass);
 5716 }
 5717 
 5718 void MacroAssembler::first_field_offset(Register inline_klass, Register offset) {
 5719   movptr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 5720   movl(offset, Address(offset, InlineKlass::first_field_offset_offset()));
 5721 }
 5722 
 5723 void MacroAssembler::data_for_oop(Register oop, Register data, Register inline_klass) {
 5724   // ((address) (void*) o) + vk->first_field_offset();
 5725   Register offset = (data == oop) ? rscratch1 : data;
 5726   first_field_offset(inline_klass, offset);
 5727   if (data == oop) {
 5728     addptr(data, offset);
 5729   } else {
 5730     lea(data, Address(oop, offset));
 5731   }
 5732 }
 5733 
 5734 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass,
 5735                                                 Register index, Register data) {
 5736   assert(index != rcx, "index needs to shift by rcx");
 5737   assert_different_registers(array, array_klass, index);
 5738   assert_different_registers(rcx, array, index);
 5739 
 5740   // array->base() + (index << Klass::layout_helper_log2_element_size(lh));
 5741   movl(rcx, Address(array_klass, Klass::layout_helper_offset()));
 5742 
 5743   // Klass::layout_helper_log2_element_size(lh)
 5744   // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask;
 5745   shrl(rcx, Klass::_lh_log2_element_size_shift);
 5746   andl(rcx, Klass::_lh_log2_element_size_mask);
 5747   shlptr(index); // index << rcx
 5748 
 5749   lea(data, Address(array, index, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_PRIMITIVE_OBJECT)));
 5750 }
 5751 
 5752 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 5753                                    Register thread_tmp, DecoratorSet decorators) {
 5754   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 5755 }
 5756 
 5757 // Doesn't do verification, generates fixed size code
 5758 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 5759                                             Register thread_tmp, DecoratorSet decorators) {
 5760   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 5761 }
 5762 
 5763 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
 5764                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 5765   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
 5766 }
 5767 
 5768 // Used for storing nulls.
 5769 void MacroAssembler::store_heap_oop_null(Address dst) {
 5770   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 5771 }
 5772 
 5773 #ifdef _LP64
 5774 void MacroAssembler::store_klass_gap(Register dst, Register src) {
 5775   if (UseCompressedClassPointers) {
 5776     // Store to klass gap in destination
 5777     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
 5778   }
 5779 }
 5780 
 5781 #ifdef ASSERT
 5782 void MacroAssembler::verify_heapbase(const char* msg) {
 5783   assert (UseCompressedOops, "should be compressed");
 5784   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5785   if (CheckCompressedOops) {
 5786     Label ok;
 5787     ExternalAddress src2(CompressedOops::ptrs_base_addr());
 5788     const bool is_src2_reachable = reachable(src2);
 5789     if (!is_src2_reachable) {
 5790       push(rscratch1);  // cmpptr trashes rscratch1
 5791     }
 5792     cmpptr(r12_heapbase, src2, rscratch1);
 5793     jcc(Assembler::equal, ok);
 5794     STOP(msg);
 5795     bind(ok);
 5796     if (!is_src2_reachable) {
 5797       pop(rscratch1);
 5798     }
 5799   }
 5800 }
 5801 #endif
 5802 
 5803 // Algorithm must match oop.inline.hpp encode_heap_oop.
 5804 void MacroAssembler::encode_heap_oop(Register r) {
 5805 #ifdef ASSERT
 5806   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
 5807 #endif
 5808   verify_oop_msg(r, "broken oop in encode_heap_oop");
 5809   if (CompressedOops::base() == nullptr) {
 5810     if (CompressedOops::shift() != 0) {
 5811       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5812       shrq(r, LogMinObjAlignmentInBytes);
 5813     }
 5814     return;
 5815   }
 5816   testq(r, r);
 5817   cmovq(Assembler::equal, r, r12_heapbase);
 5818   subq(r, r12_heapbase);
 5819   shrq(r, LogMinObjAlignmentInBytes);
 5820 }
 5821 
 5822 void MacroAssembler::encode_heap_oop_not_null(Register r) {
 5823 #ifdef ASSERT
 5824   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
 5825   if (CheckCompressedOops) {
 5826     Label ok;
 5827     testq(r, r);
 5828     jcc(Assembler::notEqual, ok);
 5829     STOP("null oop passed to encode_heap_oop_not_null");
 5830     bind(ok);
 5831   }
 5832 #endif
 5833   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
 5834   if (CompressedOops::base() != nullptr) {
 5835     subq(r, r12_heapbase);
 5836   }
 5837   if (CompressedOops::shift() != 0) {
 5838     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5839     shrq(r, LogMinObjAlignmentInBytes);
 5840   }
 5841 }
 5842 
 5843 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
 5844 #ifdef ASSERT
 5845   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
 5846   if (CheckCompressedOops) {
 5847     Label ok;
 5848     testq(src, src);
 5849     jcc(Assembler::notEqual, ok);
 5850     STOP("null oop passed to encode_heap_oop_not_null2");
 5851     bind(ok);
 5852   }
 5853 #endif
 5854   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
 5855   if (dst != src) {
 5856     movq(dst, src);
 5857   }
 5858   if (CompressedOops::base() != nullptr) {
 5859     subq(dst, r12_heapbase);
 5860   }
 5861   if (CompressedOops::shift() != 0) {
 5862     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5863     shrq(dst, LogMinObjAlignmentInBytes);
 5864   }
 5865 }
 5866 
 5867 void  MacroAssembler::decode_heap_oop(Register r) {
 5868 #ifdef ASSERT
 5869   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
 5870 #endif
 5871   if (CompressedOops::base() == nullptr) {
 5872     if (CompressedOops::shift() != 0) {
 5873       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5874       shlq(r, LogMinObjAlignmentInBytes);
 5875     }
 5876   } else {
 5877     Label done;
 5878     shlq(r, LogMinObjAlignmentInBytes);
 5879     jccb(Assembler::equal, done);
 5880     addq(r, r12_heapbase);
 5881     bind(done);
 5882   }
 5883   verify_oop_msg(r, "broken oop in decode_heap_oop");
 5884 }
 5885 
 5886 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
 5887   // Note: it will change flags
 5888   assert (UseCompressedOops, "should only be used for compressed headers");
 5889   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5890   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5891   // vtableStubs also counts instructions in pd_code_size_limit.
 5892   // Also do not verify_oop as this is called by verify_oop.
 5893   if (CompressedOops::shift() != 0) {
 5894     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5895     shlq(r, LogMinObjAlignmentInBytes);
 5896     if (CompressedOops::base() != nullptr) {
 5897       addq(r, r12_heapbase);
 5898     }
 5899   } else {
 5900     assert (CompressedOops::base() == nullptr, "sanity");
 5901   }
 5902 }
 5903 
 5904 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
 5905   // Note: it will change flags
 5906   assert (UseCompressedOops, "should only be used for compressed headers");
 5907   assert (Universe::heap() != nullptr, "java heap should be initialized");
 5908   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5909   // vtableStubs also counts instructions in pd_code_size_limit.
 5910   // Also do not verify_oop as this is called by verify_oop.
 5911   if (CompressedOops::shift() != 0) {
 5912     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 5913     if (LogMinObjAlignmentInBytes == Address::times_8) {
 5914       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
 5915     } else {
 5916       if (dst != src) {
 5917         movq(dst, src);
 5918       }
 5919       shlq(dst, LogMinObjAlignmentInBytes);
 5920       if (CompressedOops::base() != nullptr) {
 5921         addq(dst, r12_heapbase);
 5922       }
 5923     }
 5924   } else {
 5925     assert (CompressedOops::base() == nullptr, "sanity");
 5926     if (dst != src) {
 5927       movq(dst, src);
 5928     }
 5929   }
 5930 }
 5931 
 5932 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
 5933   assert_different_registers(r, tmp);
 5934   if (CompressedKlassPointers::base() != nullptr) {
 5935     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 5936     subq(r, tmp);
 5937   }
 5938   if (CompressedKlassPointers::shift() != 0) {
 5939     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5940     shrq(r, LogKlassAlignmentInBytes);
 5941   }
 5942 }
 5943 
 5944 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
 5945   assert_different_registers(src, dst);
 5946   if (CompressedKlassPointers::base() != nullptr) {
 5947     mov64(dst, -(int64_t)CompressedKlassPointers::base());
 5948     addq(dst, src);
 5949   } else {
 5950     movptr(dst, src);
 5951   }
 5952   if (CompressedKlassPointers::shift() != 0) {
 5953     assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5954     shrq(dst, LogKlassAlignmentInBytes);
 5955   }
 5956 }
 5957 
 5958 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
 5959   assert_different_registers(r, tmp);
 5960   // Note: it will change flags
 5961   assert(UseCompressedClassPointers, "should only be used for compressed headers");
 5962   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5963   // vtableStubs also counts instructions in pd_code_size_limit.
 5964   // Also do not verify_oop as this is called by verify_oop.
 5965   if (CompressedKlassPointers::shift() != 0) {
 5966     assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5967     shlq(r, LogKlassAlignmentInBytes);
 5968   }
 5969   if (CompressedKlassPointers::base() != nullptr) {
 5970     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 5971     addq(r, tmp);
 5972   }
 5973 }
 5974 
 5975 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
 5976   assert_different_registers(src, dst);
 5977   // Note: it will change flags
 5978   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 5979   // Cannot assert, unverified entry point counts instructions (see .ad file)
 5980   // vtableStubs also counts instructions in pd_code_size_limit.
 5981   // Also do not verify_oop as this is called by verify_oop.
 5982 
 5983   if (CompressedKlassPointers::base() == nullptr &&
 5984       CompressedKlassPointers::shift() == 0) {
 5985     // The best case scenario is that there is no base or shift. Then it is already
 5986     // a pointer that needs nothing but a register rename.
 5987     movl(dst, src);
 5988   } else {
 5989     if (CompressedKlassPointers::base() != nullptr) {
 5990       mov64(dst, (int64_t)CompressedKlassPointers::base());
 5991     } else {
 5992       xorq(dst, dst);
 5993     }
 5994     if (CompressedKlassPointers::shift() != 0) {
 5995       assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
 5996       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
 5997       leaq(dst, Address(dst, src, Address::times_8, 0));
 5998     } else {
 5999       addq(dst, src);
 6000     }
 6001   }
 6002 }
 6003 
 6004 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
 6005   assert (UseCompressedOops, "should only be used for compressed headers");
 6006   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6007   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6008   int oop_index = oop_recorder()->find_index(obj);
 6009   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6010   mov_narrow_oop(dst, oop_index, rspec);
 6011 }
 6012 
 6013 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
 6014   assert (UseCompressedOops, "should only be used for compressed headers");
 6015   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6016   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6017   int oop_index = oop_recorder()->find_index(obj);
 6018   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6019   mov_narrow_oop(dst, oop_index, rspec);
 6020 }
 6021 
 6022 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
 6023   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6024   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6025   int klass_index = oop_recorder()->find_index(k);
 6026   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6027   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6028 }
 6029 
 6030 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
 6031   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6032   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6033   int klass_index = oop_recorder()->find_index(k);
 6034   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6035   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6036 }
 6037 
 6038 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
 6039   assert (UseCompressedOops, "should only be used for compressed headers");
 6040   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6041   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6042   int oop_index = oop_recorder()->find_index(obj);
 6043   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6044   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6045 }
 6046 
 6047 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
 6048   assert (UseCompressedOops, "should only be used for compressed headers");
 6049   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6050   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6051   int oop_index = oop_recorder()->find_index(obj);
 6052   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6053   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6054 }
 6055 
 6056 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
 6057   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6058   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6059   int klass_index = oop_recorder()->find_index(k);
 6060   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6061   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6062 }
 6063 
 6064 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
 6065   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6066   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6067   int klass_index = oop_recorder()->find_index(k);
 6068   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6069   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6070 }
 6071 
 6072 void MacroAssembler::reinit_heapbase() {
 6073   if (UseCompressedOops) {
 6074     if (Universe::heap() != nullptr) {
 6075       if (CompressedOops::base() == nullptr) {
 6076         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 6077       } else {
 6078         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
 6079       }
 6080     } else {
 6081       movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr()));
 6082     }
 6083   }
 6084 }
 6085 
 6086 #endif // _LP64
 6087 
 6088 #if COMPILER2_OR_JVMCI
 6089 
 6090 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 6091 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask) {
 6092   // cnt - number of qwords (8-byte words).
 6093   // base - start address, qword aligned.
 6094   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 6095   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 6096   if (use64byteVector) {
 6097     evpbroadcastq(xtmp, val, AVX_512bit);
 6098   } else if (MaxVectorSize >= 32) {
 6099     movdq(xtmp, val);
 6100     punpcklqdq(xtmp, xtmp);
 6101     vinserti128_high(xtmp, xtmp);
 6102   } else {
 6103     movdq(xtmp, val);
 6104     punpcklqdq(xtmp, xtmp);
 6105   }
 6106   jmp(L_zero_64_bytes);
 6107 
 6108   BIND(L_loop);
 6109   if (MaxVectorSize >= 32) {
 6110     fill64(base, 0, xtmp, use64byteVector);
 6111   } else {
 6112     movdqu(Address(base,  0), xtmp);
 6113     movdqu(Address(base, 16), xtmp);
 6114     movdqu(Address(base, 32), xtmp);
 6115     movdqu(Address(base, 48), xtmp);
 6116   }
 6117   addptr(base, 64);
 6118 
 6119   BIND(L_zero_64_bytes);
 6120   subptr(cnt, 8);
 6121   jccb(Assembler::greaterEqual, L_loop);
 6122 
 6123   // Copy trailing 64 bytes
 6124   if (use64byteVector) {
 6125     addptr(cnt, 8);
 6126     jccb(Assembler::equal, L_end);
 6127     fill64_masked(3, base, 0, xtmp, mask, cnt, val, true);
 6128     jmp(L_end);
 6129   } else {
 6130     addptr(cnt, 4);
 6131     jccb(Assembler::less, L_tail);
 6132     if (MaxVectorSize >= 32) {
 6133       vmovdqu(Address(base, 0), xtmp);
 6134     } else {
 6135       movdqu(Address(base,  0), xtmp);
 6136       movdqu(Address(base, 16), xtmp);
 6137     }
 6138   }
 6139   addptr(base, 32);
 6140   subptr(cnt, 4);
 6141 
 6142   BIND(L_tail);
 6143   addptr(cnt, 4);
 6144   jccb(Assembler::lessEqual, L_end);
 6145   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 6146     fill32_masked(3, base, 0, xtmp, mask, cnt, val);
 6147   } else {
 6148     decrement(cnt);
 6149 
 6150     BIND(L_sloop);
 6151     movq(Address(base, 0), xtmp);
 6152     addptr(base, 8);
 6153     decrement(cnt);
 6154     jccb(Assembler::greaterEqual, L_sloop);
 6155   }
 6156   BIND(L_end);
 6157 }
 6158 
 6159 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
 6160   assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields");
 6161   // An inline type might be returned. If fields are in registers we
 6162   // need to allocate an inline type instance and initialize it with
 6163   // the value of the fields.
 6164   Label skip;
 6165   // We only need a new buffered inline type if a new one is not returned
 6166   testptr(rax, 1);
 6167   jcc(Assembler::zero, skip);
 6168   int call_offset = -1;
 6169 
 6170 #ifdef _LP64
 6171   // The following code is similar to allocate_instance but has some slight differences,
 6172   // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after
 6173   // allocating is not necessary if vk != nullptr, etc. allocate_instance is not aware of these.
 6174   Label slow_case;
 6175   // 1. Try to allocate a new buffered inline instance either from TLAB or eden space
 6176   mov(rscratch1, rax); // save rax for slow_case since *_allocate may corrupt it when allocation failed
 6177   if (vk != nullptr) {
 6178     // Called from C1, where the return type is statically known.
 6179     movptr(rbx, (intptr_t)vk->get_InlineKlass());
 6180     jint obj_size = vk->layout_helper();
 6181     assert(obj_size != Klass::_lh_neutral_value, "inline class in return type must have been resolved");
 6182     if (UseTLAB) {
 6183       tlab_allocate(r15_thread, rax, noreg, obj_size, r13, r14, slow_case);
 6184     } else {
 6185       jmp(slow_case);
 6186     }
 6187   } else {
 6188     // Call from interpreter. RAX contains ((the InlineKlass* of the return type) | 0x01)
 6189     mov(rbx, rax);
 6190     andptr(rbx, -2);
 6191     movl(r14, Address(rbx, Klass::layout_helper_offset()));
 6192     if (UseTLAB) {
 6193       tlab_allocate(r15_thread, rax, r14, 0, r13, r14, slow_case);
 6194     } else {
 6195       jmp(slow_case);
 6196     }
 6197   }
 6198   if (UseTLAB) {
 6199     // 2. Initialize buffered inline instance header
 6200     Register buffer_obj = rax;
 6201     movptr(Address(buffer_obj, oopDesc::mark_offset_in_bytes()), (intptr_t)markWord::inline_type_prototype().value());
 6202     xorl(r13, r13);
 6203     store_klass_gap(buffer_obj, r13);
 6204     if (vk == nullptr) {
 6205       // store_klass corrupts rbx(klass), so save it in r13 for later use (interpreter case only).
 6206       mov(r13, rbx);
 6207     }
 6208     store_klass(buffer_obj, rbx, rscratch1);
 6209     // 3. Initialize its fields with an inline class specific handler
 6210     if (vk != nullptr) {
 6211       call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint.
 6212     } else {
 6213       movptr(rbx, Address(r13, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 6214       movptr(rbx, Address(rbx, InlineKlass::pack_handler_offset()));
 6215       call(rbx);
 6216     }
 6217     jmp(skip);
 6218   }
 6219   bind(slow_case);
 6220   // We failed to allocate a new inline type, fall back to a runtime
 6221   // call. Some oop field may be live in some registers but we can't
 6222   // tell. That runtime call will take care of preserving them
 6223   // across a GC if there's one.
 6224   mov(rax, rscratch1);
 6225 #endif
 6226 
 6227   if (from_interpreter) {
 6228     super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf());
 6229   } else {
 6230     call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf()));
 6231     call_offset = offset();
 6232   }
 6233 
 6234   bind(skip);
 6235   return call_offset;
 6236 }
 6237 
 6238 // Move a value between registers/stack slots and update the reg_state
 6239 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
 6240   assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
 6241   if (reg_state[to->value()] == reg_written) {
 6242     return true; // Already written
 6243   }
 6244   if (from != to && bt != T_VOID) {
 6245     if (reg_state[to->value()] == reg_readonly) {
 6246       return false; // Not yet writable
 6247     }
 6248     if (from->is_reg()) {
 6249       if (to->is_reg()) {
 6250         if (from->is_XMMRegister()) {
 6251           if (bt == T_DOUBLE) {
 6252             movdbl(to->as_XMMRegister(), from->as_XMMRegister());
 6253           } else {
 6254             assert(bt == T_FLOAT, "must be float");
 6255             movflt(to->as_XMMRegister(), from->as_XMMRegister());
 6256           }
 6257         } else {
 6258           movq(to->as_Register(), from->as_Register());
 6259         }
 6260       } else {
 6261         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6262         Address to_addr = Address(rsp, st_off);
 6263         if (from->is_XMMRegister()) {
 6264           if (bt == T_DOUBLE) {
 6265             movdbl(to_addr, from->as_XMMRegister());
 6266           } else {
 6267             assert(bt == T_FLOAT, "must be float");
 6268             movflt(to_addr, from->as_XMMRegister());
 6269           }
 6270         } else {
 6271           movq(to_addr, from->as_Register());
 6272         }
 6273       }
 6274     } else {
 6275       Address from_addr = Address(rsp, from->reg2stack() * VMRegImpl::stack_slot_size + wordSize);
 6276       if (to->is_reg()) {
 6277         if (to->is_XMMRegister()) {
 6278           if (bt == T_DOUBLE) {
 6279             movdbl(to->as_XMMRegister(), from_addr);
 6280           } else {
 6281             assert(bt == T_FLOAT, "must be float");
 6282             movflt(to->as_XMMRegister(), from_addr);
 6283           }
 6284         } else {
 6285           movq(to->as_Register(), from_addr);
 6286         }
 6287       } else {
 6288         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6289         movq(r13, from_addr);
 6290         movq(Address(rsp, st_off), r13);
 6291       }
 6292     }
 6293   }
 6294   // Update register states
 6295   reg_state[from->value()] = reg_writable;
 6296   reg_state[to->value()] = reg_written;
 6297   return true;
 6298 }
 6299 
 6300 // Calculate the extra stack space required for packing or unpacking inline
 6301 // args and adjust the stack pointer
 6302 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
 6303   // Two additional slots to account for return address
 6304   int sp_inc = (args_on_stack + 2) * VMRegImpl::stack_slot_size;
 6305   sp_inc = align_up(sp_inc, StackAlignmentInBytes);
 6306   // Save the return address, adjust the stack (make sure it is properly
 6307   // 16-byte aligned) and copy the return address to the new top of the stack.
 6308   // The stack will be repaired on return (see MacroAssembler::remove_frame).
 6309   assert(sp_inc > 0, "sanity");
 6310   pop(r13);
 6311   subptr(rsp, sp_inc);
 6312   push(r13);
 6313   return sp_inc;
 6314 }
 6315 
 6316 // Read all fields from an inline type buffer and store the field values in registers/stack slots.
 6317 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
 6318                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
 6319                                           RegState reg_state[]) {
 6320   assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
 6321   assert(from->is_valid(), "source must be valid");
 6322   bool progress = false;
 6323 #ifdef ASSERT
 6324   const int start_offset = offset();
 6325 #endif
 6326 
 6327   Label L_null, L_notNull;
 6328   // Don't use r14 as tmp because it's used for spilling (see MacroAssembler::spill_reg_for)
 6329   Register tmp1 = r10;
 6330   Register tmp2 = r13;
 6331   Register fromReg = noreg;
 6332   ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1);
 6333   bool done = true;
 6334   bool mark_done = true;
 6335   VMReg toReg;
 6336   BasicType bt;
 6337   // Check if argument requires a null check
 6338   bool null_check = false;
 6339   VMReg nullCheckReg;
 6340   while (stream.next(nullCheckReg, bt)) {
 6341     if (sig->at(stream.sig_index())._offset == -1) {
 6342       null_check = true;
 6343       break;
 6344     }
 6345   }
 6346   stream.reset(sig_index, to_index);
 6347   while (stream.next(toReg, bt)) {
 6348     assert(toReg->is_valid(), "destination must be valid");
 6349     int idx = (int)toReg->value();
 6350     if (reg_state[idx] == reg_readonly) {
 6351       if (idx != from->value()) {
 6352         mark_done = false;
 6353       }
 6354       done = false;
 6355       continue;
 6356     } else if (reg_state[idx] == reg_written) {
 6357       continue;
 6358     }
 6359     assert(reg_state[idx] == reg_writable, "must be writable");
 6360     reg_state[idx] = reg_written;
 6361     progress = true;
 6362 
 6363     if (fromReg == noreg) {
 6364       if (from->is_reg()) {
 6365         fromReg = from->as_Register();
 6366       } else {
 6367         int st_off = from->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6368         movq(tmp1, Address(rsp, st_off));
 6369         fromReg = tmp1;
 6370       }
 6371       if (null_check) {
 6372         // Nullable inline type argument, emit null check
 6373         testptr(fromReg, fromReg);
 6374         jcc(Assembler::zero, L_null);
 6375       }
 6376     }
 6377     int off = sig->at(stream.sig_index())._offset;
 6378     if (off == -1) {
 6379       assert(null_check, "Missing null check at");
 6380       if (toReg->is_stack()) {
 6381         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6382         movq(Address(rsp, st_off), 1);
 6383       } else {
 6384         movq(toReg->as_Register(), 1);
 6385       }
 6386       continue;
 6387     }
 6388     assert(off > 0, "offset in object should be positive");
 6389     Address fromAddr = Address(fromReg, off);
 6390     if (!toReg->is_XMMRegister()) {
 6391       Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register();
 6392       if (is_reference_type(bt)) {
 6393         load_heap_oop(dst, fromAddr);
 6394       } else {
 6395         bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN);
 6396         load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed);
 6397       }
 6398       if (toReg->is_stack()) {
 6399         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6400         movq(Address(rsp, st_off), dst);
 6401       }
 6402     } else if (bt == T_DOUBLE) {
 6403       movdbl(toReg->as_XMMRegister(), fromAddr);
 6404     } else {
 6405       assert(bt == T_FLOAT, "must be float");
 6406       movflt(toReg->as_XMMRegister(), fromAddr);
 6407     }
 6408   }
 6409   if (progress && null_check) {
 6410     if (done) {
 6411       jmp(L_notNull);
 6412       bind(L_null);
 6413       // Set IsInit field to zero to signal that the argument is null.
 6414       // Also set all oop fields to zero to make the GC happy.
 6415       stream.reset(sig_index, to_index);
 6416       while (stream.next(toReg, bt)) {
 6417         if (sig->at(stream.sig_index())._offset == -1 ||
 6418             bt == T_OBJECT || bt == T_ARRAY) {
 6419           if (toReg->is_stack()) {
 6420             int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6421             movq(Address(rsp, st_off), 0);
 6422           } else {
 6423             xorq(toReg->as_Register(), toReg->as_Register());
 6424           }
 6425         }
 6426       }
 6427       bind(L_notNull);
 6428     } else {
 6429       bind(L_null);
 6430     }
 6431   }
 6432 
 6433   sig_index = stream.sig_index();
 6434   to_index = stream.regs_index();
 6435 
 6436   if (mark_done && reg_state[from->value()] != reg_written) {
 6437     // This is okay because no one else will write to that slot
 6438     reg_state[from->value()] = reg_writable;
 6439   }
 6440   from_index--;
 6441   assert(progress || (start_offset == offset()), "should not emit code");
 6442   return done;
 6443 }
 6444 
 6445 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
 6446                                         VMRegPair* from, int from_count, int& from_index, VMReg to,
 6447                                         RegState reg_state[], Register val_array) {
 6448   assert(sig->at(sig_index)._bt == T_METADATA, "should be at delimiter");
 6449   assert(to->is_valid(), "destination must be valid");
 6450 
 6451   if (reg_state[to->value()] == reg_written) {
 6452     skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
 6453     return true; // Already written
 6454   }
 6455 
 6456   // TODO 8284443 Isn't it an issue if below code uses r14 as tmp when it contains a spilled value?
 6457   // Be careful with r14 because it's used for spilling (see MacroAssembler::spill_reg_for).
 6458   Register val_obj_tmp = r11;
 6459   Register from_reg_tmp = r14;
 6460   Register tmp1 = r10;
 6461   Register tmp2 = r13;
 6462   Register tmp3 = rbx;
 6463   Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register();
 6464 
 6465   assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array);
 6466 
 6467   if (reg_state[to->value()] == reg_readonly) {
 6468     if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) {
 6469       skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
 6470       return false; // Not yet writable
 6471     }
 6472     val_obj = val_obj_tmp;
 6473   }
 6474 
 6475   int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_OBJECT);
 6476   load_heap_oop(val_obj, Address(val_array, index));
 6477 
 6478   ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index);
 6479   VMReg fromReg;
 6480   BasicType bt;
 6481   Label L_null;
 6482   while (stream.next(fromReg, bt)) {
 6483     assert(fromReg->is_valid(), "source must be valid");
 6484     reg_state[fromReg->value()] = reg_writable;
 6485 
 6486     int off = sig->at(stream.sig_index())._offset;
 6487     if (off == -1) {
 6488       // Nullable inline type argument, emit null check
 6489       Label L_notNull;
 6490       if (fromReg->is_stack()) {
 6491         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6492         testb(Address(rsp, ld_off), 1);
 6493       } else {
 6494         testb(fromReg->as_Register(), 1);
 6495       }
 6496       jcc(Assembler::notZero, L_notNull);
 6497       movptr(val_obj, 0);
 6498       jmp(L_null);
 6499       bind(L_notNull);
 6500       continue;
 6501     }
 6502 
 6503     assert(off > 0, "offset in object should be positive");
 6504     size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
 6505 
 6506     Address dst(val_obj, off);
 6507     if (!fromReg->is_XMMRegister()) {
 6508       Register src;
 6509       if (fromReg->is_stack()) {
 6510         src = from_reg_tmp;
 6511         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6512         load_sized_value(src, Address(rsp, ld_off), size_in_bytes, /* is_signed */ false);
 6513       } else {
 6514         src = fromReg->as_Register();
 6515       }
 6516       assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array);
 6517       if (is_reference_type(bt)) {
 6518         store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 6519       } else {
 6520         store_sized_value(dst, src, size_in_bytes);
 6521       }
 6522     } else if (bt == T_DOUBLE) {
 6523       movdbl(dst, fromReg->as_XMMRegister());
 6524     } else {
 6525       assert(bt == T_FLOAT, "must be float");
 6526       movflt(dst, fromReg->as_XMMRegister());
 6527     }
 6528   }
 6529   bind(L_null);
 6530   sig_index = stream.sig_index();
 6531   from_index = stream.regs_index();
 6532 
 6533   assert(reg_state[to->value()] == reg_writable, "must have already been read");
 6534   bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state);
 6535   assert(success, "to register must be writeable");
 6536   return true;
 6537 }
 6538 
 6539 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
 6540   return reg->is_XMMRegister() ? xmm8->as_VMReg() : r14->as_VMReg();
 6541 }
 6542 
 6543 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
 6544   assert((initial_framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 6545   if (needs_stack_repair) {
 6546     movq(rbp, Address(rsp, initial_framesize));
 6547     // The stack increment resides just below the saved rbp
 6548     addq(rsp, Address(rsp, initial_framesize - wordSize));
 6549   } else {
 6550     if (initial_framesize > 0) {
 6551       addq(rsp, initial_framesize);
 6552     }
 6553     pop(rbp);
 6554   }
 6555 }
 6556 
 6557 // Clearing constant sized memory using YMM/ZMM registers.
 6558 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6559   assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), "");
 6560   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 6561 
 6562   int vector64_count = (cnt & (~0x7)) >> 3;
 6563   cnt = cnt & 0x7;
 6564   const int fill64_per_loop = 4;
 6565   const int max_unrolled_fill64 = 8;
 6566 
 6567   // 64 byte initialization loop.
 6568   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 6569   int start64 = 0;
 6570   if (vector64_count > max_unrolled_fill64) {
 6571     Label LOOP;
 6572     Register index = rtmp;
 6573 
 6574     start64 = vector64_count - (vector64_count % fill64_per_loop);
 6575 
 6576     movl(index, 0);
 6577     BIND(LOOP);
 6578     for (int i = 0; i < fill64_per_loop; i++) {
 6579       fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector);
 6580     }
 6581     addl(index, fill64_per_loop * 64);
 6582     cmpl(index, start64 * 64);
 6583     jccb(Assembler::less, LOOP);
 6584   }
 6585   for (int i = start64; i < vector64_count; i++) {
 6586     fill64(base, i * 64, xtmp, use64byteVector);
 6587   }
 6588 
 6589   // Clear remaining 64 byte tail.
 6590   int disp = vector64_count * 64;
 6591   if (cnt) {
 6592     switch (cnt) {
 6593       case 1:
 6594         movq(Address(base, disp), xtmp);
 6595         break;
 6596       case 2:
 6597         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit);
 6598         break;
 6599       case 3:
 6600         movl(rtmp, 0x7);
 6601         kmovwl(mask, rtmp);
 6602         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit);
 6603         break;
 6604       case 4:
 6605         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6606         break;
 6607       case 5:
 6608         if (use64byteVector) {
 6609           movl(rtmp, 0x1F);
 6610           kmovwl(mask, rtmp);
 6611           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6612         } else {
 6613           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6614           movq(Address(base, disp + 32), xtmp);
 6615         }
 6616         break;
 6617       case 6:
 6618         if (use64byteVector) {
 6619           movl(rtmp, 0x3F);
 6620           kmovwl(mask, rtmp);
 6621           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6622         } else {
 6623           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6624           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit);
 6625         }
 6626         break;
 6627       case 7:
 6628         if (use64byteVector) {
 6629           movl(rtmp, 0x7F);
 6630           kmovwl(mask, rtmp);
 6631           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6632         } else {
 6633           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6634           movl(rtmp, 0x7);
 6635           kmovwl(mask, rtmp);
 6636           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
 6637         }
 6638         break;
 6639       default:
 6640         fatal("Unexpected length : %d\n",cnt);
 6641         break;
 6642     }
 6643   }
 6644 }
 6645 
 6646 void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp,
 6647                                bool is_large, bool word_copy_only, KRegister mask) {
 6648   // cnt      - number of qwords (8-byte words).
 6649   // base     - start address, qword aligned.
 6650   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 6651   assert(base==rdi, "base register must be edi for rep stos");
 6652   assert(val==rax,   "val register must be eax for rep stos");
 6653   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 6654   assert(InitArrayShortSize % BytesPerLong == 0,
 6655     "InitArrayShortSize should be the multiple of BytesPerLong");
 6656 
 6657   Label DONE;
 6658 
 6659   if (!is_large) {
 6660     Label LOOP, LONG;
 6661     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 6662     jccb(Assembler::greater, LONG);
 6663 
 6664     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6665 
 6666     decrement(cnt);
 6667     jccb(Assembler::negative, DONE); // Zero length
 6668 
 6669     // Use individual pointer-sized stores for small counts:
 6670     BIND(LOOP);
 6671     movptr(Address(base, cnt, Address::times_ptr), val);
 6672     decrement(cnt);
 6673     jccb(Assembler::greaterEqual, LOOP);
 6674     jmpb(DONE);
 6675 
 6676     BIND(LONG);
 6677   }
 6678 
 6679   // Use longer rep-prefixed ops for non-small counts:
 6680   if (UseFastStosb && !word_copy_only) {
 6681     shlptr(cnt, 3); // convert to number of bytes
 6682     rep_stosb();
 6683   } else if (UseXMMForObjInit) {
 6684     xmm_clear_mem(base, cnt, val, xtmp, mask);
 6685   } else {
 6686     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6687     rep_stos();
 6688   }
 6689 
 6690   BIND(DONE);
 6691 }
 6692 
 6693 #endif //COMPILER2_OR_JVMCI
 6694 
 6695 
 6696 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 6697                                    Register to, Register value, Register count,
 6698                                    Register rtmp, XMMRegister xtmp) {
 6699   ShortBranchVerifier sbv(this);
 6700   assert_different_registers(to, value, count, rtmp);
 6701   Label L_exit;
 6702   Label L_fill_2_bytes, L_fill_4_bytes;
 6703 
 6704 #if defined(COMPILER2) && defined(_LP64)
 6705   if(MaxVectorSize >=32 &&
 6706      VM_Version::supports_avx512vlbw() &&
 6707      VM_Version::supports_bmi2()) {
 6708     generate_fill_avx3(t, to, value, count, rtmp, xtmp);
 6709     return;
 6710   }
 6711 #endif
 6712 
 6713   int shift = -1;
 6714   switch (t) {
 6715     case T_BYTE:
 6716       shift = 2;
 6717       break;
 6718     case T_SHORT:
 6719       shift = 1;
 6720       break;
 6721     case T_INT:
 6722       shift = 0;
 6723       break;
 6724     default: ShouldNotReachHere();
 6725   }
 6726 
 6727   if (t == T_BYTE) {
 6728     andl(value, 0xff);
 6729     movl(rtmp, value);
 6730     shll(rtmp, 8);
 6731     orl(value, rtmp);
 6732   }
 6733   if (t == T_SHORT) {
 6734     andl(value, 0xffff);
 6735   }
 6736   if (t == T_BYTE || t == T_SHORT) {
 6737     movl(rtmp, value);
 6738     shll(rtmp, 16);
 6739     orl(value, rtmp);
 6740   }
 6741 
 6742   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
 6743   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
 6744   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 6745     Label L_skip_align2;
 6746     // align source address at 4 bytes address boundary
 6747     if (t == T_BYTE) {
 6748       Label L_skip_align1;
 6749       // One byte misalignment happens only for byte arrays
 6750       testptr(to, 1);
 6751       jccb(Assembler::zero, L_skip_align1);
 6752       movb(Address(to, 0), value);
 6753       increment(to);
 6754       decrement(count);
 6755       BIND(L_skip_align1);
 6756     }
 6757     // Two bytes misalignment happens only for byte and short (char) arrays
 6758     testptr(to, 2);
 6759     jccb(Assembler::zero, L_skip_align2);
 6760     movw(Address(to, 0), value);
 6761     addptr(to, 2);
 6762     subl(count, 1<<(shift-1));
 6763     BIND(L_skip_align2);
 6764   }
 6765   if (UseSSE < 2) {
 6766     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6767     // Fill 32-byte chunks
 6768     subl(count, 8 << shift);
 6769     jcc(Assembler::less, L_check_fill_8_bytes);
 6770     align(16);
 6771 
 6772     BIND(L_fill_32_bytes_loop);
 6773 
 6774     for (int i = 0; i < 32; i += 4) {
 6775       movl(Address(to, i), value);
 6776     }
 6777 
 6778     addptr(to, 32);
 6779     subl(count, 8 << shift);
 6780     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6781     BIND(L_check_fill_8_bytes);
 6782     addl(count, 8 << shift);
 6783     jccb(Assembler::zero, L_exit);
 6784     jmpb(L_fill_8_bytes);
 6785 
 6786     //
 6787     // length is too short, just fill qwords
 6788     //
 6789     BIND(L_fill_8_bytes_loop);
 6790     movl(Address(to, 0), value);
 6791     movl(Address(to, 4), value);
 6792     addptr(to, 8);
 6793     BIND(L_fill_8_bytes);
 6794     subl(count, 1 << (shift + 1));
 6795     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6796     // fall through to fill 4 bytes
 6797   } else {
 6798     Label L_fill_32_bytes;
 6799     if (!UseUnalignedLoadStores) {
 6800       // align to 8 bytes, we know we are 4 byte aligned to start
 6801       testptr(to, 4);
 6802       jccb(Assembler::zero, L_fill_32_bytes);
 6803       movl(Address(to, 0), value);
 6804       addptr(to, 4);
 6805       subl(count, 1<<shift);
 6806     }
 6807     BIND(L_fill_32_bytes);
 6808     {
 6809       assert( UseSSE >= 2, "supported cpu only" );
 6810       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6811       movdl(xtmp, value);
 6812       if (UseAVX >= 2 && UseUnalignedLoadStores) {
 6813         Label L_check_fill_32_bytes;
 6814         if (UseAVX > 2) {
 6815           // Fill 64-byte chunks
 6816           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
 6817 
 6818           // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
 6819           cmpl(count, VM_Version::avx3_threshold());
 6820           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
 6821 
 6822           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
 6823 
 6824           subl(count, 16 << shift);
 6825           jccb(Assembler::less, L_check_fill_32_bytes);
 6826           align(16);
 6827 
 6828           BIND(L_fill_64_bytes_loop_avx3);
 6829           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
 6830           addptr(to, 64);
 6831           subl(count, 16 << shift);
 6832           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
 6833           jmpb(L_check_fill_32_bytes);
 6834 
 6835           BIND(L_check_fill_64_bytes_avx2);
 6836         }
 6837         // Fill 64-byte chunks
 6838         Label L_fill_64_bytes_loop;
 6839         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
 6840 
 6841         subl(count, 16 << shift);
 6842         jcc(Assembler::less, L_check_fill_32_bytes);
 6843         align(16);
 6844 
 6845         BIND(L_fill_64_bytes_loop);
 6846         vmovdqu(Address(to, 0), xtmp);
 6847         vmovdqu(Address(to, 32), xtmp);
 6848         addptr(to, 64);
 6849         subl(count, 16 << shift);
 6850         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
 6851 
 6852         BIND(L_check_fill_32_bytes);
 6853         addl(count, 8 << shift);
 6854         jccb(Assembler::less, L_check_fill_8_bytes);
 6855         vmovdqu(Address(to, 0), xtmp);
 6856         addptr(to, 32);
 6857         subl(count, 8 << shift);
 6858 
 6859         BIND(L_check_fill_8_bytes);
 6860         // clean upper bits of YMM registers
 6861         movdl(xtmp, value);
 6862         pshufd(xtmp, xtmp, 0);
 6863       } else {
 6864         // Fill 32-byte chunks
 6865         pshufd(xtmp, xtmp, 0);
 6866 
 6867         subl(count, 8 << shift);
 6868         jcc(Assembler::less, L_check_fill_8_bytes);
 6869         align(16);
 6870 
 6871         BIND(L_fill_32_bytes_loop);
 6872 
 6873         if (UseUnalignedLoadStores) {
 6874           movdqu(Address(to, 0), xtmp);
 6875           movdqu(Address(to, 16), xtmp);
 6876         } else {
 6877           movq(Address(to, 0), xtmp);
 6878           movq(Address(to, 8), xtmp);
 6879           movq(Address(to, 16), xtmp);
 6880           movq(Address(to, 24), xtmp);
 6881         }
 6882 
 6883         addptr(to, 32);
 6884         subl(count, 8 << shift);
 6885         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6886 
 6887         BIND(L_check_fill_8_bytes);
 6888       }
 6889       addl(count, 8 << shift);
 6890       jccb(Assembler::zero, L_exit);
 6891       jmpb(L_fill_8_bytes);
 6892 
 6893       //
 6894       // length is too short, just fill qwords
 6895       //
 6896       BIND(L_fill_8_bytes_loop);
 6897       movq(Address(to, 0), xtmp);
 6898       addptr(to, 8);
 6899       BIND(L_fill_8_bytes);
 6900       subl(count, 1 << (shift + 1));
 6901       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6902     }
 6903   }
 6904   // fill trailing 4 bytes
 6905   BIND(L_fill_4_bytes);
 6906   testl(count, 1<<shift);
 6907   jccb(Assembler::zero, L_fill_2_bytes);
 6908   movl(Address(to, 0), value);
 6909   if (t == T_BYTE || t == T_SHORT) {
 6910     Label L_fill_byte;
 6911     addptr(to, 4);
 6912     BIND(L_fill_2_bytes);
 6913     // fill trailing 2 bytes
 6914     testl(count, 1<<(shift-1));
 6915     jccb(Assembler::zero, L_fill_byte);
 6916     movw(Address(to, 0), value);
 6917     if (t == T_BYTE) {
 6918       addptr(to, 2);
 6919       BIND(L_fill_byte);
 6920       // fill trailing byte
 6921       testl(count, 1);
 6922       jccb(Assembler::zero, L_exit);
 6923       movb(Address(to, 0), value);
 6924     } else {
 6925       BIND(L_fill_byte);
 6926     }
 6927   } else {
 6928     BIND(L_fill_2_bytes);
 6929   }
 6930   BIND(L_exit);
 6931 }
 6932 
 6933 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) {
 6934   switch(type) {
 6935     case T_BYTE:
 6936     case T_BOOLEAN:
 6937       evpbroadcastb(dst, src, vector_len);
 6938       break;
 6939     case T_SHORT:
 6940     case T_CHAR:
 6941       evpbroadcastw(dst, src, vector_len);
 6942       break;
 6943     case T_INT:
 6944     case T_FLOAT:
 6945       evpbroadcastd(dst, src, vector_len);
 6946       break;
 6947     case T_LONG:
 6948     case T_DOUBLE:
 6949       evpbroadcastq(dst, src, vector_len);
 6950       break;
 6951     default:
 6952       fatal("Unhandled type : %s", type2name(type));
 6953       break;
 6954   }
 6955 }
 6956 
 6957 // encode char[] to byte[] in ISO_8859_1 or ASCII
 6958    //@IntrinsicCandidate
 6959    //private static int implEncodeISOArray(byte[] sa, int sp,
 6960    //byte[] da, int dp, int len) {
 6961    //  int i = 0;
 6962    //  for (; i < len; i++) {
 6963    //    char c = StringUTF16.getChar(sa, sp++);
 6964    //    if (c > '\u00FF')
 6965    //      break;
 6966    //    da[dp++] = (byte)c;
 6967    //  }
 6968    //  return i;
 6969    //}
 6970    //
 6971    //@IntrinsicCandidate
 6972    //private static int implEncodeAsciiArray(char[] sa, int sp,
 6973    //    byte[] da, int dp, int len) {
 6974    //  int i = 0;
 6975    //  for (; i < len; i++) {
 6976    //    char c = sa[sp++];
 6977    //    if (c >= '\u0080')
 6978    //      break;
 6979    //    da[dp++] = (byte)c;
 6980    //  }
 6981    //  return i;
 6982    //}
 6983 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
 6984   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 6985   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 6986   Register tmp5, Register result, bool ascii) {
 6987 
 6988   // rsi: src
 6989   // rdi: dst
 6990   // rdx: len
 6991   // rcx: tmp5
 6992   // rax: result
 6993   ShortBranchVerifier sbv(this);
 6994   assert_different_registers(src, dst, len, tmp5, result);
 6995   Label L_done, L_copy_1_char, L_copy_1_char_exit;
 6996 
 6997   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
 6998   int short_mask = ascii ? 0xff80 : 0xff00;
 6999 
 7000   // set result
 7001   xorl(result, result);
 7002   // check for zero length
 7003   testl(len, len);
 7004   jcc(Assembler::zero, L_done);
 7005 
 7006   movl(result, len);
 7007 
 7008   // Setup pointers
 7009   lea(src, Address(src, len, Address::times_2)); // char[]
 7010   lea(dst, Address(dst, len, Address::times_1)); // byte[]
 7011   negptr(len);
 7012 
 7013   if (UseSSE42Intrinsics || UseAVX >= 2) {
 7014     Label L_copy_8_chars, L_copy_8_chars_exit;
 7015     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
 7016 
 7017     if (UseAVX >= 2) {
 7018       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
 7019       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 7020       movdl(tmp1Reg, tmp5);
 7021       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
 7022       jmp(L_chars_32_check);
 7023 
 7024       bind(L_copy_32_chars);
 7025       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
 7026       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
 7027       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 7028       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 7029       jccb(Assembler::notZero, L_copy_32_chars_exit);
 7030       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 7031       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
 7032       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
 7033 
 7034       bind(L_chars_32_check);
 7035       addptr(len, 32);
 7036       jcc(Assembler::lessEqual, L_copy_32_chars);
 7037 
 7038       bind(L_copy_32_chars_exit);
 7039       subptr(len, 16);
 7040       jccb(Assembler::greater, L_copy_16_chars_exit);
 7041 
 7042     } else if (UseSSE42Intrinsics) {
 7043       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 7044       movdl(tmp1Reg, tmp5);
 7045       pshufd(tmp1Reg, tmp1Reg, 0);
 7046       jmpb(L_chars_16_check);
 7047     }
 7048 
 7049     bind(L_copy_16_chars);
 7050     if (UseAVX >= 2) {
 7051       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
 7052       vptest(tmp2Reg, tmp1Reg);
 7053       jcc(Assembler::notZero, L_copy_16_chars_exit);
 7054       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
 7055       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
 7056     } else {
 7057       if (UseAVX > 0) {
 7058         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 7059         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 7060         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
 7061       } else {
 7062         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 7063         por(tmp2Reg, tmp3Reg);
 7064         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 7065         por(tmp2Reg, tmp4Reg);
 7066       }
 7067       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 7068       jccb(Assembler::notZero, L_copy_16_chars_exit);
 7069       packuswb(tmp3Reg, tmp4Reg);
 7070     }
 7071     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
 7072 
 7073     bind(L_chars_16_check);
 7074     addptr(len, 16);
 7075     jcc(Assembler::lessEqual, L_copy_16_chars);
 7076 
 7077     bind(L_copy_16_chars_exit);
 7078     if (UseAVX >= 2) {
 7079       // clean upper bits of YMM registers
 7080       vpxor(tmp2Reg, tmp2Reg);
 7081       vpxor(tmp3Reg, tmp3Reg);
 7082       vpxor(tmp4Reg, tmp4Reg);
 7083       movdl(tmp1Reg, tmp5);
 7084       pshufd(tmp1Reg, tmp1Reg, 0);
 7085     }
 7086     subptr(len, 8);
 7087     jccb(Assembler::greater, L_copy_8_chars_exit);
 7088 
 7089     bind(L_copy_8_chars);
 7090     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
 7091     ptest(tmp3Reg, tmp1Reg);
 7092     jccb(Assembler::notZero, L_copy_8_chars_exit);
 7093     packuswb(tmp3Reg, tmp1Reg);
 7094     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
 7095     addptr(len, 8);
 7096     jccb(Assembler::lessEqual, L_copy_8_chars);
 7097 
 7098     bind(L_copy_8_chars_exit);
 7099     subptr(len, 8);
 7100     jccb(Assembler::zero, L_done);
 7101   }
 7102 
 7103   bind(L_copy_1_char);
 7104   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
 7105   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
 7106   jccb(Assembler::notZero, L_copy_1_char_exit);
 7107   movb(Address(dst, len, Address::times_1, 0), tmp5);
 7108   addptr(len, 1);
 7109   jccb(Assembler::less, L_copy_1_char);
 7110 
 7111   bind(L_copy_1_char_exit);
 7112   addptr(result, len); // len is negative count of not processed elements
 7113 
 7114   bind(L_done);
 7115 }
 7116 
 7117 #ifdef _LP64
 7118 /**
 7119  * Helper for multiply_to_len().
 7120  */
 7121 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
 7122   addq(dest_lo, src1);
 7123   adcq(dest_hi, 0);
 7124   addq(dest_lo, src2);
 7125   adcq(dest_hi, 0);
 7126 }
 7127 
 7128 /**
 7129  * Multiply 64 bit by 64 bit first loop.
 7130  */
 7131 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
 7132                                            Register y, Register y_idx, Register z,
 7133                                            Register carry, Register product,
 7134                                            Register idx, Register kdx) {
 7135   //
 7136   //  jlong carry, x[], y[], z[];
 7137   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7138   //    huge_128 product = y[idx] * x[xstart] + carry;
 7139   //    z[kdx] = (jlong)product;
 7140   //    carry  = (jlong)(product >>> 64);
 7141   //  }
 7142   //  z[xstart] = carry;
 7143   //
 7144 
 7145   Label L_first_loop, L_first_loop_exit;
 7146   Label L_one_x, L_one_y, L_multiply;
 7147 
 7148   decrementl(xstart);
 7149   jcc(Assembler::negative, L_one_x);
 7150 
 7151   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7152   rorq(x_xstart, 32); // convert big-endian to little-endian
 7153 
 7154   bind(L_first_loop);
 7155   decrementl(idx);
 7156   jcc(Assembler::negative, L_first_loop_exit);
 7157   decrementl(idx);
 7158   jcc(Assembler::negative, L_one_y);
 7159   movq(y_idx, Address(y, idx, Address::times_4,  0));
 7160   rorq(y_idx, 32); // convert big-endian to little-endian
 7161   bind(L_multiply);
 7162   movq(product, x_xstart);
 7163   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
 7164   addq(product, carry);
 7165   adcq(rdx, 0);
 7166   subl(kdx, 2);
 7167   movl(Address(z, kdx, Address::times_4,  4), product);
 7168   shrq(product, 32);
 7169   movl(Address(z, kdx, Address::times_4,  0), product);
 7170   movq(carry, rdx);
 7171   jmp(L_first_loop);
 7172 
 7173   bind(L_one_y);
 7174   movl(y_idx, Address(y,  0));
 7175   jmp(L_multiply);
 7176 
 7177   bind(L_one_x);
 7178   movl(x_xstart, Address(x,  0));
 7179   jmp(L_first_loop);
 7180 
 7181   bind(L_first_loop_exit);
 7182 }
 7183 
 7184 /**
 7185  * Multiply 64 bit by 64 bit and add 128 bit.
 7186  */
 7187 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
 7188                                             Register yz_idx, Register idx,
 7189                                             Register carry, Register product, int offset) {
 7190   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
 7191   //     z[kdx] = (jlong)product;
 7192 
 7193   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
 7194   rorq(yz_idx, 32); // convert big-endian to little-endian
 7195   movq(product, x_xstart);
 7196   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
 7197   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
 7198   rorq(yz_idx, 32); // convert big-endian to little-endian
 7199 
 7200   add2_with_carry(rdx, product, carry, yz_idx);
 7201 
 7202   movl(Address(z, idx, Address::times_4,  offset+4), product);
 7203   shrq(product, 32);
 7204   movl(Address(z, idx, Address::times_4,  offset), product);
 7205 
 7206 }
 7207 
 7208 /**
 7209  * Multiply 128 bit by 128 bit. Unrolled inner loop.
 7210  */
 7211 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
 7212                                              Register yz_idx, Register idx, Register jdx,
 7213                                              Register carry, Register product,
 7214                                              Register carry2) {
 7215   //   jlong carry, x[], y[], z[];
 7216   //   int kdx = ystart+1;
 7217   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 7218   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
 7219   //     z[kdx+idx+1] = (jlong)product;
 7220   //     jlong carry2  = (jlong)(product >>> 64);
 7221   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
 7222   //     z[kdx+idx] = (jlong)product;
 7223   //     carry  = (jlong)(product >>> 64);
 7224   //   }
 7225   //   idx += 2;
 7226   //   if (idx > 0) {
 7227   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
 7228   //     z[kdx+idx] = (jlong)product;
 7229   //     carry  = (jlong)(product >>> 64);
 7230   //   }
 7231   //
 7232 
 7233   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 7234 
 7235   movl(jdx, idx);
 7236   andl(jdx, 0xFFFFFFFC);
 7237   shrl(jdx, 2);
 7238 
 7239   bind(L_third_loop);
 7240   subl(jdx, 1);
 7241   jcc(Assembler::negative, L_third_loop_exit);
 7242   subl(idx, 4);
 7243 
 7244   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
 7245   movq(carry2, rdx);
 7246 
 7247   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
 7248   movq(carry, rdx);
 7249   jmp(L_third_loop);
 7250 
 7251   bind (L_third_loop_exit);
 7252 
 7253   andl (idx, 0x3);
 7254   jcc(Assembler::zero, L_post_third_loop_done);
 7255 
 7256   Label L_check_1;
 7257   subl(idx, 2);
 7258   jcc(Assembler::negative, L_check_1);
 7259 
 7260   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
 7261   movq(carry, rdx);
 7262 
 7263   bind (L_check_1);
 7264   addl (idx, 0x2);
 7265   andl (idx, 0x1);
 7266   subl(idx, 1);
 7267   jcc(Assembler::negative, L_post_third_loop_done);
 7268 
 7269   movl(yz_idx, Address(y, idx, Address::times_4,  0));
 7270   movq(product, x_xstart);
 7271   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
 7272   movl(yz_idx, Address(z, idx, Address::times_4,  0));
 7273 
 7274   add2_with_carry(rdx, product, yz_idx, carry);
 7275 
 7276   movl(Address(z, idx, Address::times_4,  0), product);
 7277   shrq(product, 32);
 7278 
 7279   shlq(rdx, 32);
 7280   orq(product, rdx);
 7281   movq(carry, product);
 7282 
 7283   bind(L_post_third_loop_done);
 7284 }
 7285 
 7286 /**
 7287  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
 7288  *
 7289  */
 7290 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
 7291                                                   Register carry, Register carry2,
 7292                                                   Register idx, Register jdx,
 7293                                                   Register yz_idx1, Register yz_idx2,
 7294                                                   Register tmp, Register tmp3, Register tmp4) {
 7295   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
 7296 
 7297   //   jlong carry, x[], y[], z[];
 7298   //   int kdx = ystart+1;
 7299   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 7300   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
 7301   //     jlong carry2  = (jlong)(tmp3 >>> 64);
 7302   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
 7303   //     carry  = (jlong)(tmp4 >>> 64);
 7304   //     z[kdx+idx+1] = (jlong)tmp3;
 7305   //     z[kdx+idx] = (jlong)tmp4;
 7306   //   }
 7307   //   idx += 2;
 7308   //   if (idx > 0) {
 7309   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
 7310   //     z[kdx+idx] = (jlong)yz_idx1;
 7311   //     carry  = (jlong)(yz_idx1 >>> 64);
 7312   //   }
 7313   //
 7314 
 7315   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 7316 
 7317   movl(jdx, idx);
 7318   andl(jdx, 0xFFFFFFFC);
 7319   shrl(jdx, 2);
 7320 
 7321   bind(L_third_loop);
 7322   subl(jdx, 1);
 7323   jcc(Assembler::negative, L_third_loop_exit);
 7324   subl(idx, 4);
 7325 
 7326   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
 7327   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
 7328   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
 7329   rorxq(yz_idx2, yz_idx2, 32);
 7330 
 7331   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
 7332   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
 7333 
 7334   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
 7335   rorxq(yz_idx1, yz_idx1, 32);
 7336   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 7337   rorxq(yz_idx2, yz_idx2, 32);
 7338 
 7339   if (VM_Version::supports_adx()) {
 7340     adcxq(tmp3, carry);
 7341     adoxq(tmp3, yz_idx1);
 7342 
 7343     adcxq(tmp4, tmp);
 7344     adoxq(tmp4, yz_idx2);
 7345 
 7346     movl(carry, 0); // does not affect flags
 7347     adcxq(carry2, carry);
 7348     adoxq(carry2, carry);
 7349   } else {
 7350     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
 7351     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
 7352   }
 7353   movq(carry, carry2);
 7354 
 7355   movl(Address(z, idx, Address::times_4, 12), tmp3);
 7356   shrq(tmp3, 32);
 7357   movl(Address(z, idx, Address::times_4,  8), tmp3);
 7358 
 7359   movl(Address(z, idx, Address::times_4,  4), tmp4);
 7360   shrq(tmp4, 32);
 7361   movl(Address(z, idx, Address::times_4,  0), tmp4);
 7362 
 7363   jmp(L_third_loop);
 7364 
 7365   bind (L_third_loop_exit);
 7366 
 7367   andl (idx, 0x3);
 7368   jcc(Assembler::zero, L_post_third_loop_done);
 7369 
 7370   Label L_check_1;
 7371   subl(idx, 2);
 7372   jcc(Assembler::negative, L_check_1);
 7373 
 7374   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
 7375   rorxq(yz_idx1, yz_idx1, 32);
 7376   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
 7377   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 7378   rorxq(yz_idx2, yz_idx2, 32);
 7379 
 7380   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
 7381 
 7382   movl(Address(z, idx, Address::times_4,  4), tmp3);
 7383   shrq(tmp3, 32);
 7384   movl(Address(z, idx, Address::times_4,  0), tmp3);
 7385   movq(carry, tmp4);
 7386 
 7387   bind (L_check_1);
 7388   addl (idx, 0x2);
 7389   andl (idx, 0x1);
 7390   subl(idx, 1);
 7391   jcc(Assembler::negative, L_post_third_loop_done);
 7392   movl(tmp4, Address(y, idx, Address::times_4,  0));
 7393   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
 7394   movl(tmp4, Address(z, idx, Address::times_4,  0));
 7395 
 7396   add2_with_carry(carry2, tmp3, tmp4, carry);
 7397 
 7398   movl(Address(z, idx, Address::times_4,  0), tmp3);
 7399   shrq(tmp3, 32);
 7400 
 7401   shlq(carry2, 32);
 7402   orq(tmp3, carry2);
 7403   movq(carry, tmp3);
 7404 
 7405   bind(L_post_third_loop_done);
 7406 }
 7407 
 7408 /**
 7409  * Code for BigInteger::multiplyToLen() intrinsic.
 7410  *
 7411  * rdi: x
 7412  * rax: xlen
 7413  * rsi: y
 7414  * rcx: ylen
 7415  * r8:  z
 7416  * r11: zlen
 7417  * r12: tmp1
 7418  * r13: tmp2
 7419  * r14: tmp3
 7420  * r15: tmp4
 7421  * rbx: tmp5
 7422  *
 7423  */
 7424 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
 7425                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
 7426   ShortBranchVerifier sbv(this);
 7427   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
 7428 
 7429   push(tmp1);
 7430   push(tmp2);
 7431   push(tmp3);
 7432   push(tmp4);
 7433   push(tmp5);
 7434 
 7435   push(xlen);
 7436   push(zlen);
 7437 
 7438   const Register idx = tmp1;
 7439   const Register kdx = tmp2;
 7440   const Register xstart = tmp3;
 7441 
 7442   const Register y_idx = tmp4;
 7443   const Register carry = tmp5;
 7444   const Register product  = xlen;
 7445   const Register x_xstart = zlen;  // reuse register
 7446 
 7447   // First Loop.
 7448   //
 7449   //  final static long LONG_MASK = 0xffffffffL;
 7450   //  int xstart = xlen - 1;
 7451   //  int ystart = ylen - 1;
 7452   //  long carry = 0;
 7453   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7454   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
 7455   //    z[kdx] = (int)product;
 7456   //    carry = product >>> 32;
 7457   //  }
 7458   //  z[xstart] = (int)carry;
 7459   //
 7460 
 7461   movl(idx, ylen);      // idx = ylen;
 7462   movl(kdx, zlen);      // kdx = xlen+ylen;
 7463   xorq(carry, carry);   // carry = 0;
 7464 
 7465   Label L_done;
 7466 
 7467   movl(xstart, xlen);
 7468   decrementl(xstart);
 7469   jcc(Assembler::negative, L_done);
 7470 
 7471   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
 7472 
 7473   Label L_second_loop;
 7474   testl(kdx, kdx);
 7475   jcc(Assembler::zero, L_second_loop);
 7476 
 7477   Label L_carry;
 7478   subl(kdx, 1);
 7479   jcc(Assembler::zero, L_carry);
 7480 
 7481   movl(Address(z, kdx, Address::times_4,  0), carry);
 7482   shrq(carry, 32);
 7483   subl(kdx, 1);
 7484 
 7485   bind(L_carry);
 7486   movl(Address(z, kdx, Address::times_4,  0), carry);
 7487 
 7488   // Second and third (nested) loops.
 7489   //
 7490   // for (int i = xstart-1; i >= 0; i--) { // Second loop
 7491   //   carry = 0;
 7492   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
 7493   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
 7494   //                    (z[k] & LONG_MASK) + carry;
 7495   //     z[k] = (int)product;
 7496   //     carry = product >>> 32;
 7497   //   }
 7498   //   z[i] = (int)carry;
 7499   // }
 7500   //
 7501   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
 7502 
 7503   const Register jdx = tmp1;
 7504 
 7505   bind(L_second_loop);
 7506   xorl(carry, carry);    // carry = 0;
 7507   movl(jdx, ylen);       // j = ystart+1
 7508 
 7509   subl(xstart, 1);       // i = xstart-1;
 7510   jcc(Assembler::negative, L_done);
 7511 
 7512   push (z);
 7513 
 7514   Label L_last_x;
 7515   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
 7516   subl(xstart, 1);       // i = xstart-1;
 7517   jcc(Assembler::negative, L_last_x);
 7518 
 7519   if (UseBMI2Instructions) {
 7520     movq(rdx,  Address(x, xstart, Address::times_4,  0));
 7521     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
 7522   } else {
 7523     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7524     rorq(x_xstart, 32);  // convert big-endian to little-endian
 7525   }
 7526 
 7527   Label L_third_loop_prologue;
 7528   bind(L_third_loop_prologue);
 7529 
 7530   push (x);
 7531   push (xstart);
 7532   push (ylen);
 7533 
 7534 
 7535   if (UseBMI2Instructions) {
 7536     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
 7537   } else { // !UseBMI2Instructions
 7538     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
 7539   }
 7540 
 7541   pop(ylen);
 7542   pop(xlen);
 7543   pop(x);
 7544   pop(z);
 7545 
 7546   movl(tmp3, xlen);
 7547   addl(tmp3, 1);
 7548   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7549   subl(tmp3, 1);
 7550   jccb(Assembler::negative, L_done);
 7551 
 7552   shrq(carry, 32);
 7553   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7554   jmp(L_second_loop);
 7555 
 7556   // Next infrequent code is moved outside loops.
 7557   bind(L_last_x);
 7558   if (UseBMI2Instructions) {
 7559     movl(rdx, Address(x,  0));
 7560   } else {
 7561     movl(x_xstart, Address(x,  0));
 7562   }
 7563   jmp(L_third_loop_prologue);
 7564 
 7565   bind(L_done);
 7566 
 7567   pop(zlen);
 7568   pop(xlen);
 7569 
 7570   pop(tmp5);
 7571   pop(tmp4);
 7572   pop(tmp3);
 7573   pop(tmp2);
 7574   pop(tmp1);
 7575 }
 7576 
 7577 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
 7578   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
 7579   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
 7580   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
 7581   Label VECTOR8_TAIL, VECTOR4_TAIL;
 7582   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
 7583   Label SAME_TILL_END, DONE;
 7584   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
 7585 
 7586   //scale is in rcx in both Win64 and Unix
 7587   ShortBranchVerifier sbv(this);
 7588 
 7589   shlq(length);
 7590   xorq(result, result);
 7591 
 7592   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
 7593       VM_Version::supports_avx512vlbw()) {
 7594     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
 7595 
 7596     cmpq(length, 64);
 7597     jcc(Assembler::less, VECTOR32_TAIL);
 7598 
 7599     movq(tmp1, length);
 7600     andq(tmp1, 0x3F);      // tail count
 7601     andq(length, ~(0x3F)); //vector count
 7602 
 7603     bind(VECTOR64_LOOP);
 7604     // AVX512 code to compare 64 byte vectors.
 7605     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
 7606     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7607     kortestql(k7, k7);
 7608     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
 7609     addq(result, 64);
 7610     subq(length, 64);
 7611     jccb(Assembler::notZero, VECTOR64_LOOP);
 7612 
 7613     //bind(VECTOR64_TAIL);
 7614     testq(tmp1, tmp1);
 7615     jcc(Assembler::zero, SAME_TILL_END);
 7616 
 7617     //bind(VECTOR64_TAIL);
 7618     // AVX512 code to compare up to 63 byte vectors.
 7619     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
 7620     shlxq(tmp2, tmp2, tmp1);
 7621     notq(tmp2);
 7622     kmovql(k3, tmp2);
 7623 
 7624     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
 7625     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7626 
 7627     ktestql(k7, k3);
 7628     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
 7629 
 7630     bind(VECTOR64_NOT_EQUAL);
 7631     kmovql(tmp1, k7);
 7632     notq(tmp1);
 7633     tzcntq(tmp1, tmp1);
 7634     addq(result, tmp1);
 7635     shrq(result);
 7636     jmp(DONE);
 7637     bind(VECTOR32_TAIL);
 7638   }
 7639 
 7640   cmpq(length, 8);
 7641   jcc(Assembler::equal, VECTOR8_LOOP);
 7642   jcc(Assembler::less, VECTOR4_TAIL);
 7643 
 7644   if (UseAVX >= 2) {
 7645     Label VECTOR16_TAIL, VECTOR32_LOOP;
 7646 
 7647     cmpq(length, 16);
 7648     jcc(Assembler::equal, VECTOR16_LOOP);
 7649     jcc(Assembler::less, VECTOR8_LOOP);
 7650 
 7651     cmpq(length, 32);
 7652     jccb(Assembler::less, VECTOR16_TAIL);
 7653 
 7654     subq(length, 32);
 7655     bind(VECTOR32_LOOP);
 7656     vmovdqu(rymm0, Address(obja, result));
 7657     vmovdqu(rymm1, Address(objb, result));
 7658     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
 7659     vptest(rymm2, rymm2);
 7660     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
 7661     addq(result, 32);
 7662     subq(length, 32);
 7663     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
 7664     addq(length, 32);
 7665     jcc(Assembler::equal, SAME_TILL_END);
 7666     //falling through if less than 32 bytes left //close the branch here.
 7667 
 7668     bind(VECTOR16_TAIL);
 7669     cmpq(length, 16);
 7670     jccb(Assembler::less, VECTOR8_TAIL);
 7671     bind(VECTOR16_LOOP);
 7672     movdqu(rymm0, Address(obja, result));
 7673     movdqu(rymm1, Address(objb, result));
 7674     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
 7675     ptest(rymm2, rymm2);
 7676     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7677     addq(result, 16);
 7678     subq(length, 16);
 7679     jcc(Assembler::equal, SAME_TILL_END);
 7680     //falling through if less than 16 bytes left
 7681   } else {//regular intrinsics
 7682 
 7683     cmpq(length, 16);
 7684     jccb(Assembler::less, VECTOR8_TAIL);
 7685 
 7686     subq(length, 16);
 7687     bind(VECTOR16_LOOP);
 7688     movdqu(rymm0, Address(obja, result));
 7689     movdqu(rymm1, Address(objb, result));
 7690     pxor(rymm0, rymm1);
 7691     ptest(rymm0, rymm0);
 7692     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7693     addq(result, 16);
 7694     subq(length, 16);
 7695     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
 7696     addq(length, 16);
 7697     jcc(Assembler::equal, SAME_TILL_END);
 7698     //falling through if less than 16 bytes left
 7699   }
 7700 
 7701   bind(VECTOR8_TAIL);
 7702   cmpq(length, 8);
 7703   jccb(Assembler::less, VECTOR4_TAIL);
 7704   bind(VECTOR8_LOOP);
 7705   movq(tmp1, Address(obja, result));
 7706   movq(tmp2, Address(objb, result));
 7707   xorq(tmp1, tmp2);
 7708   testq(tmp1, tmp1);
 7709   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
 7710   addq(result, 8);
 7711   subq(length, 8);
 7712   jcc(Assembler::equal, SAME_TILL_END);
 7713   //falling through if less than 8 bytes left
 7714 
 7715   bind(VECTOR4_TAIL);
 7716   cmpq(length, 4);
 7717   jccb(Assembler::less, BYTES_TAIL);
 7718   bind(VECTOR4_LOOP);
 7719   movl(tmp1, Address(obja, result));
 7720   xorl(tmp1, Address(objb, result));
 7721   testl(tmp1, tmp1);
 7722   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
 7723   addq(result, 4);
 7724   subq(length, 4);
 7725   jcc(Assembler::equal, SAME_TILL_END);
 7726   //falling through if less than 4 bytes left
 7727 
 7728   bind(BYTES_TAIL);
 7729   bind(BYTES_LOOP);
 7730   load_unsigned_byte(tmp1, Address(obja, result));
 7731   load_unsigned_byte(tmp2, Address(objb, result));
 7732   xorl(tmp1, tmp2);
 7733   testl(tmp1, tmp1);
 7734   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7735   decq(length);
 7736   jcc(Assembler::zero, SAME_TILL_END);
 7737   incq(result);
 7738   load_unsigned_byte(tmp1, Address(obja, result));
 7739   load_unsigned_byte(tmp2, Address(objb, result));
 7740   xorl(tmp1, tmp2);
 7741   testl(tmp1, tmp1);
 7742   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7743   decq(length);
 7744   jcc(Assembler::zero, SAME_TILL_END);
 7745   incq(result);
 7746   load_unsigned_byte(tmp1, Address(obja, result));
 7747   load_unsigned_byte(tmp2, Address(objb, result));
 7748   xorl(tmp1, tmp2);
 7749   testl(tmp1, tmp1);
 7750   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7751   jmp(SAME_TILL_END);
 7752 
 7753   if (UseAVX >= 2) {
 7754     bind(VECTOR32_NOT_EQUAL);
 7755     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
 7756     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
 7757     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
 7758     vpmovmskb(tmp1, rymm0);
 7759     bsfq(tmp1, tmp1);
 7760     addq(result, tmp1);
 7761     shrq(result);
 7762     jmp(DONE);
 7763   }
 7764 
 7765   bind(VECTOR16_NOT_EQUAL);
 7766   if (UseAVX >= 2) {
 7767     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
 7768     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
 7769     pxor(rymm0, rymm2);
 7770   } else {
 7771     pcmpeqb(rymm2, rymm2);
 7772     pxor(rymm0, rymm1);
 7773     pcmpeqb(rymm0, rymm1);
 7774     pxor(rymm0, rymm2);
 7775   }
 7776   pmovmskb(tmp1, rymm0);
 7777   bsfq(tmp1, tmp1);
 7778   addq(result, tmp1);
 7779   shrq(result);
 7780   jmpb(DONE);
 7781 
 7782   bind(VECTOR8_NOT_EQUAL);
 7783   bind(VECTOR4_NOT_EQUAL);
 7784   bsfq(tmp1, tmp1);
 7785   shrq(tmp1, 3);
 7786   addq(result, tmp1);
 7787   bind(BYTES_NOT_EQUAL);
 7788   shrq(result);
 7789   jmpb(DONE);
 7790 
 7791   bind(SAME_TILL_END);
 7792   mov64(result, -1);
 7793 
 7794   bind(DONE);
 7795 }
 7796 
 7797 //Helper functions for square_to_len()
 7798 
 7799 /**
 7800  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
 7801  * Preserves x and z and modifies rest of the registers.
 7802  */
 7803 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7804   // Perform square and right shift by 1
 7805   // Handle odd xlen case first, then for even xlen do the following
 7806   // jlong carry = 0;
 7807   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
 7808   //     huge_128 product = x[j:j+1] * x[j:j+1];
 7809   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
 7810   //     z[i+2:i+3] = (jlong)(product >>> 1);
 7811   //     carry = (jlong)product;
 7812   // }
 7813 
 7814   xorq(tmp5, tmp5);     // carry
 7815   xorq(rdxReg, rdxReg);
 7816   xorl(tmp1, tmp1);     // index for x
 7817   xorl(tmp4, tmp4);     // index for z
 7818 
 7819   Label L_first_loop, L_first_loop_exit;
 7820 
 7821   testl(xlen, 1);
 7822   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
 7823 
 7824   // Square and right shift by 1 the odd element using 32 bit multiply
 7825   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
 7826   imulq(raxReg, raxReg);
 7827   shrq(raxReg, 1);
 7828   adcq(tmp5, 0);
 7829   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
 7830   incrementl(tmp1);
 7831   addl(tmp4, 2);
 7832 
 7833   // Square and  right shift by 1 the rest using 64 bit multiply
 7834   bind(L_first_loop);
 7835   cmpptr(tmp1, xlen);
 7836   jccb(Assembler::equal, L_first_loop_exit);
 7837 
 7838   // Square
 7839   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
 7840   rorq(raxReg, 32);    // convert big-endian to little-endian
 7841   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
 7842 
 7843   // Right shift by 1 and save carry
 7844   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
 7845   rcrq(rdxReg, 1);
 7846   rcrq(raxReg, 1);
 7847   adcq(tmp5, 0);
 7848 
 7849   // Store result in z
 7850   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
 7851   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
 7852 
 7853   // Update indices for x and z
 7854   addl(tmp1, 2);
 7855   addl(tmp4, 4);
 7856   jmp(L_first_loop);
 7857 
 7858   bind(L_first_loop_exit);
 7859 }
 7860 
 7861 
 7862 /**
 7863  * Perform the following multiply add operation using BMI2 instructions
 7864  * carry:sum = sum + op1*op2 + carry
 7865  * op2 should be in rdx
 7866  * op2 is preserved, all other registers are modified
 7867  */
 7868 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
 7869   // assert op2 is rdx
 7870   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
 7871   addq(sum, carry);
 7872   adcq(tmp2, 0);
 7873   addq(sum, op1);
 7874   adcq(tmp2, 0);
 7875   movq(carry, tmp2);
 7876 }
 7877 
 7878 /**
 7879  * Perform the following multiply add operation:
 7880  * carry:sum = sum + op1*op2 + carry
 7881  * Preserves op1, op2 and modifies rest of registers
 7882  */
 7883 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
 7884   // rdx:rax = op1 * op2
 7885   movq(raxReg, op2);
 7886   mulq(op1);
 7887 
 7888   //  rdx:rax = sum + carry + rdx:rax
 7889   addq(sum, carry);
 7890   adcq(rdxReg, 0);
 7891   addq(sum, raxReg);
 7892   adcq(rdxReg, 0);
 7893 
 7894   // carry:sum = rdx:sum
 7895   movq(carry, rdxReg);
 7896 }
 7897 
 7898 /**
 7899  * Add 64 bit long carry into z[] with carry propagation.
 7900  * Preserves z and carry register values and modifies rest of registers.
 7901  *
 7902  */
 7903 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
 7904   Label L_fourth_loop, L_fourth_loop_exit;
 7905 
 7906   movl(tmp1, 1);
 7907   subl(zlen, 2);
 7908   addq(Address(z, zlen, Address::times_4, 0), carry);
 7909 
 7910   bind(L_fourth_loop);
 7911   jccb(Assembler::carryClear, L_fourth_loop_exit);
 7912   subl(zlen, 2);
 7913   jccb(Assembler::negative, L_fourth_loop_exit);
 7914   addq(Address(z, zlen, Address::times_4, 0), tmp1);
 7915   jmp(L_fourth_loop);
 7916   bind(L_fourth_loop_exit);
 7917 }
 7918 
 7919 /**
 7920  * Shift z[] left by 1 bit.
 7921  * Preserves x, len, z and zlen registers and modifies rest of the registers.
 7922  *
 7923  */
 7924 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
 7925 
 7926   Label L_fifth_loop, L_fifth_loop_exit;
 7927 
 7928   // Fifth loop
 7929   // Perform primitiveLeftShift(z, zlen, 1)
 7930 
 7931   const Register prev_carry = tmp1;
 7932   const Register new_carry = tmp4;
 7933   const Register value = tmp2;
 7934   const Register zidx = tmp3;
 7935 
 7936   // int zidx, carry;
 7937   // long value;
 7938   // carry = 0;
 7939   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
 7940   //    (carry:value)  = (z[i] << 1) | carry ;
 7941   //    z[i] = value;
 7942   // }
 7943 
 7944   movl(zidx, zlen);
 7945   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
 7946 
 7947   bind(L_fifth_loop);
 7948   decl(zidx);  // Use decl to preserve carry flag
 7949   decl(zidx);
 7950   jccb(Assembler::negative, L_fifth_loop_exit);
 7951 
 7952   if (UseBMI2Instructions) {
 7953      movq(value, Address(z, zidx, Address::times_4, 0));
 7954      rclq(value, 1);
 7955      rorxq(value, value, 32);
 7956      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7957   }
 7958   else {
 7959     // clear new_carry
 7960     xorl(new_carry, new_carry);
 7961 
 7962     // Shift z[i] by 1, or in previous carry and save new carry
 7963     movq(value, Address(z, zidx, Address::times_4, 0));
 7964     shlq(value, 1);
 7965     adcl(new_carry, 0);
 7966 
 7967     orq(value, prev_carry);
 7968     rorq(value, 0x20);
 7969     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7970 
 7971     // Set previous carry = new carry
 7972     movl(prev_carry, new_carry);
 7973   }
 7974   jmp(L_fifth_loop);
 7975 
 7976   bind(L_fifth_loop_exit);
 7977 }
 7978 
 7979 
 7980 /**
 7981  * Code for BigInteger::squareToLen() intrinsic
 7982  *
 7983  * rdi: x
 7984  * rsi: len
 7985  * r8:  z
 7986  * rcx: zlen
 7987  * r12: tmp1
 7988  * r13: tmp2
 7989  * r14: tmp3
 7990  * r15: tmp4
 7991  * rbx: tmp5
 7992  *
 7993  */
 7994 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7995 
 7996   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
 7997   push(tmp1);
 7998   push(tmp2);
 7999   push(tmp3);
 8000   push(tmp4);
 8001   push(tmp5);
 8002 
 8003   // First loop
 8004   // Store the squares, right shifted one bit (i.e., divided by 2).
 8005   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
 8006 
 8007   // Add in off-diagonal sums.
 8008   //
 8009   // Second, third (nested) and fourth loops.
 8010   // zlen +=2;
 8011   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
 8012   //    carry = 0;
 8013   //    long op2 = x[xidx:xidx+1];
 8014   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
 8015   //       k -= 2;
 8016   //       long op1 = x[j:j+1];
 8017   //       long sum = z[k:k+1];
 8018   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
 8019   //       z[k:k+1] = sum;
 8020   //    }
 8021   //    add_one_64(z, k, carry, tmp_regs);
 8022   // }
 8023 
 8024   const Register carry = tmp5;
 8025   const Register sum = tmp3;
 8026   const Register op1 = tmp4;
 8027   Register op2 = tmp2;
 8028 
 8029   push(zlen);
 8030   push(len);
 8031   addl(zlen,2);
 8032   bind(L_second_loop);
 8033   xorq(carry, carry);
 8034   subl(zlen, 4);
 8035   subl(len, 2);
 8036   push(zlen);
 8037   push(len);
 8038   cmpl(len, 0);
 8039   jccb(Assembler::lessEqual, L_second_loop_exit);
 8040 
 8041   // Multiply an array by one 64 bit long.
 8042   if (UseBMI2Instructions) {
 8043     op2 = rdxReg;
 8044     movq(op2, Address(x, len, Address::times_4,  0));
 8045     rorxq(op2, op2, 32);
 8046   }
 8047   else {
 8048     movq(op2, Address(x, len, Address::times_4,  0));
 8049     rorq(op2, 32);
 8050   }
 8051 
 8052   bind(L_third_loop);
 8053   decrementl(len);
 8054   jccb(Assembler::negative, L_third_loop_exit);
 8055   decrementl(len);
 8056   jccb(Assembler::negative, L_last_x);
 8057 
 8058   movq(op1, Address(x, len, Address::times_4,  0));
 8059   rorq(op1, 32);
 8060 
 8061   bind(L_multiply);
 8062   subl(zlen, 2);
 8063   movq(sum, Address(z, zlen, Address::times_4,  0));
 8064 
 8065   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
 8066   if (UseBMI2Instructions) {
 8067     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
 8068   }
 8069   else {
 8070     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8071   }
 8072 
 8073   movq(Address(z, zlen, Address::times_4, 0), sum);
 8074 
 8075   jmp(L_third_loop);
 8076   bind(L_third_loop_exit);
 8077 
 8078   // Fourth loop
 8079   // Add 64 bit long carry into z with carry propagation.
 8080   // Uses offsetted zlen.
 8081   add_one_64(z, zlen, carry, tmp1);
 8082 
 8083   pop(len);
 8084   pop(zlen);
 8085   jmp(L_second_loop);
 8086 
 8087   // Next infrequent code is moved outside loops.
 8088   bind(L_last_x);
 8089   movl(op1, Address(x, 0));
 8090   jmp(L_multiply);
 8091 
 8092   bind(L_second_loop_exit);
 8093   pop(len);
 8094   pop(zlen);
 8095   pop(len);
 8096   pop(zlen);
 8097 
 8098   // Fifth loop
 8099   // Shift z left 1 bit.
 8100   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
 8101 
 8102   // z[zlen-1] |= x[len-1] & 1;
 8103   movl(tmp3, Address(x, len, Address::times_4, -4));
 8104   andl(tmp3, 1);
 8105   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
 8106 
 8107   pop(tmp5);
 8108   pop(tmp4);
 8109   pop(tmp3);
 8110   pop(tmp2);
 8111   pop(tmp1);
 8112 }
 8113 
 8114 /**
 8115  * Helper function for mul_add()
 8116  * Multiply the in[] by int k and add to out[] starting at offset offs using
 8117  * 128 bit by 32 bit multiply and return the carry in tmp5.
 8118  * Only quad int aligned length of in[] is operated on in this function.
 8119  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
 8120  * This function preserves out, in and k registers.
 8121  * len and offset point to the appropriate index in "in" & "out" correspondingly
 8122  * tmp5 has the carry.
 8123  * other registers are temporary and are modified.
 8124  *
 8125  */
 8126 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
 8127   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
 8128   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8129 
 8130   Label L_first_loop, L_first_loop_exit;
 8131 
 8132   movl(tmp1, len);
 8133   shrl(tmp1, 2);
 8134 
 8135   bind(L_first_loop);
 8136   subl(tmp1, 1);
 8137   jccb(Assembler::negative, L_first_loop_exit);
 8138 
 8139   subl(len, 4);
 8140   subl(offset, 4);
 8141 
 8142   Register op2 = tmp2;
 8143   const Register sum = tmp3;
 8144   const Register op1 = tmp4;
 8145   const Register carry = tmp5;
 8146 
 8147   if (UseBMI2Instructions) {
 8148     op2 = rdxReg;
 8149   }
 8150 
 8151   movq(op1, Address(in, len, Address::times_4,  8));
 8152   rorq(op1, 32);
 8153   movq(sum, Address(out, offset, Address::times_4,  8));
 8154   rorq(sum, 32);
 8155   if (UseBMI2Instructions) {
 8156     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8157   }
 8158   else {
 8159     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8160   }
 8161   // Store back in big endian from little endian
 8162   rorq(sum, 0x20);
 8163   movq(Address(out, offset, Address::times_4,  8), sum);
 8164 
 8165   movq(op1, Address(in, len, Address::times_4,  0));
 8166   rorq(op1, 32);
 8167   movq(sum, Address(out, offset, Address::times_4,  0));
 8168   rorq(sum, 32);
 8169   if (UseBMI2Instructions) {
 8170     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8171   }
 8172   else {
 8173     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8174   }
 8175   // Store back in big endian from little endian
 8176   rorq(sum, 0x20);
 8177   movq(Address(out, offset, Address::times_4,  0), sum);
 8178 
 8179   jmp(L_first_loop);
 8180   bind(L_first_loop_exit);
 8181 }
 8182 
 8183 /**
 8184  * Code for BigInteger::mulAdd() intrinsic
 8185  *
 8186  * rdi: out
 8187  * rsi: in
 8188  * r11: offs (out.length - offset)
 8189  * rcx: len
 8190  * r8:  k
 8191  * r12: tmp1
 8192  * r13: tmp2
 8193  * r14: tmp3
 8194  * r15: tmp4
 8195  * rbx: tmp5
 8196  * Multiply the in[] by word k and add to out[], return the carry in rax
 8197  */
 8198 void MacroAssembler::mul_add(Register out, Register in, Register offs,
 8199    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
 8200    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8201 
 8202   Label L_carry, L_last_in, L_done;
 8203 
 8204 // carry = 0;
 8205 // for (int j=len-1; j >= 0; j--) {
 8206 //    long product = (in[j] & LONG_MASK) * kLong +
 8207 //                   (out[offs] & LONG_MASK) + carry;
 8208 //    out[offs--] = (int)product;
 8209 //    carry = product >>> 32;
 8210 // }
 8211 //
 8212   push(tmp1);
 8213   push(tmp2);
 8214   push(tmp3);
 8215   push(tmp4);
 8216   push(tmp5);
 8217 
 8218   Register op2 = tmp2;
 8219   const Register sum = tmp3;
 8220   const Register op1 = tmp4;
 8221   const Register carry =  tmp5;
 8222 
 8223   if (UseBMI2Instructions) {
 8224     op2 = rdxReg;
 8225     movl(op2, k);
 8226   }
 8227   else {
 8228     movl(op2, k);
 8229   }
 8230 
 8231   xorq(carry, carry);
 8232 
 8233   //First loop
 8234 
 8235   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
 8236   //The carry is in tmp5
 8237   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
 8238 
 8239   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
 8240   decrementl(len);
 8241   jccb(Assembler::negative, L_carry);
 8242   decrementl(len);
 8243   jccb(Assembler::negative, L_last_in);
 8244 
 8245   movq(op1, Address(in, len, Address::times_4,  0));
 8246   rorq(op1, 32);
 8247 
 8248   subl(offs, 2);
 8249   movq(sum, Address(out, offs, Address::times_4,  0));
 8250   rorq(sum, 32);
 8251 
 8252   if (UseBMI2Instructions) {
 8253     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8254   }
 8255   else {
 8256     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8257   }
 8258 
 8259   // Store back in big endian from little endian
 8260   rorq(sum, 0x20);
 8261   movq(Address(out, offs, Address::times_4,  0), sum);
 8262 
 8263   testl(len, len);
 8264   jccb(Assembler::zero, L_carry);
 8265 
 8266   //Multiply the last in[] entry, if any
 8267   bind(L_last_in);
 8268   movl(op1, Address(in, 0));
 8269   movl(sum, Address(out, offs, Address::times_4,  -4));
 8270 
 8271   movl(raxReg, k);
 8272   mull(op1); //tmp4 * eax -> edx:eax
 8273   addl(sum, carry);
 8274   adcl(rdxReg, 0);
 8275   addl(sum, raxReg);
 8276   adcl(rdxReg, 0);
 8277   movl(carry, rdxReg);
 8278 
 8279   movl(Address(out, offs, Address::times_4,  -4), sum);
 8280 
 8281   bind(L_carry);
 8282   //return tmp5/carry as carry in rax
 8283   movl(rax, carry);
 8284 
 8285   bind(L_done);
 8286   pop(tmp5);
 8287   pop(tmp4);
 8288   pop(tmp3);
 8289   pop(tmp2);
 8290   pop(tmp1);
 8291 }
 8292 #endif
 8293 
 8294 /**
 8295  * Emits code to update CRC-32 with a byte value according to constants in table
 8296  *
 8297  * @param [in,out]crc   Register containing the crc.
 8298  * @param [in]val       Register containing the byte to fold into the CRC.
 8299  * @param [in]table     Register containing the table of crc constants.
 8300  *
 8301  * uint32_t crc;
 8302  * val = crc_table[(val ^ crc) & 0xFF];
 8303  * crc = val ^ (crc >> 8);
 8304  *
 8305  */
 8306 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
 8307   xorl(val, crc);
 8308   andl(val, 0xFF);
 8309   shrl(crc, 8); // unsigned shift
 8310   xorl(crc, Address(table, val, Address::times_4, 0));
 8311 }
 8312 
 8313 /**
 8314  * Fold 128-bit data chunk
 8315  */
 8316 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
 8317   if (UseAVX > 0) {
 8318     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
 8319     vpclmulldq(xcrc, xK, xcrc); // [63:0]
 8320     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
 8321     pxor(xcrc, xtmp);
 8322   } else {
 8323     movdqa(xtmp, xcrc);
 8324     pclmulhdq(xtmp, xK);   // [123:64]
 8325     pclmulldq(xcrc, xK);   // [63:0]
 8326     pxor(xcrc, xtmp);
 8327     movdqu(xtmp, Address(buf, offset));
 8328     pxor(xcrc, xtmp);
 8329   }
 8330 }
 8331 
 8332 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
 8333   if (UseAVX > 0) {
 8334     vpclmulhdq(xtmp, xK, xcrc);
 8335     vpclmulldq(xcrc, xK, xcrc);
 8336     pxor(xcrc, xbuf);
 8337     pxor(xcrc, xtmp);
 8338   } else {
 8339     movdqa(xtmp, xcrc);
 8340     pclmulhdq(xtmp, xK);
 8341     pclmulldq(xcrc, xK);
 8342     pxor(xcrc, xbuf);
 8343     pxor(xcrc, xtmp);
 8344   }
 8345 }
 8346 
 8347 /**
 8348  * 8-bit folds to compute 32-bit CRC
 8349  *
 8350  * uint64_t xcrc;
 8351  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
 8352  */
 8353 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
 8354   movdl(tmp, xcrc);
 8355   andl(tmp, 0xFF);
 8356   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
 8357   psrldq(xcrc, 1); // unsigned shift one byte
 8358   pxor(xcrc, xtmp);
 8359 }
 8360 
 8361 /**
 8362  * uint32_t crc;
 8363  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
 8364  */
 8365 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
 8366   movl(tmp, crc);
 8367   andl(tmp, 0xFF);
 8368   shrl(crc, 8);
 8369   xorl(crc, Address(table, tmp, Address::times_4, 0));
 8370 }
 8371 
 8372 /**
 8373  * @param crc   register containing existing CRC (32-bit)
 8374  * @param buf   register pointing to input byte buffer (byte*)
 8375  * @param len   register containing number of bytes
 8376  * @param table register that will contain address of CRC table
 8377  * @param tmp   scratch register
 8378  */
 8379 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
 8380   assert_different_registers(crc, buf, len, table, tmp, rax);
 8381 
 8382   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 8383   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 8384 
 8385   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 8386   // context for the registers used, where all instructions below are using 128-bit mode
 8387   // On EVEX without VL and BW, these instructions will all be AVX.
 8388   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
 8389   notl(crc); // ~crc
 8390   cmpl(len, 16);
 8391   jcc(Assembler::less, L_tail);
 8392 
 8393   // Align buffer to 16 bytes
 8394   movl(tmp, buf);
 8395   andl(tmp, 0xF);
 8396   jccb(Assembler::zero, L_aligned);
 8397   subl(tmp,  16);
 8398   addl(len, tmp);
 8399 
 8400   align(4);
 8401   BIND(L_align_loop);
 8402   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 8403   update_byte_crc32(crc, rax, table);
 8404   increment(buf);
 8405   incrementl(tmp);
 8406   jccb(Assembler::less, L_align_loop);
 8407 
 8408   BIND(L_aligned);
 8409   movl(tmp, len); // save
 8410   shrl(len, 4);
 8411   jcc(Assembler::zero, L_tail_restore);
 8412 
 8413   // Fold crc into first bytes of vector
 8414   movdqa(xmm1, Address(buf, 0));
 8415   movdl(rax, xmm1);
 8416   xorl(crc, rax);
 8417   if (VM_Version::supports_sse4_1()) {
 8418     pinsrd(xmm1, crc, 0);
 8419   } else {
 8420     pinsrw(xmm1, crc, 0);
 8421     shrl(crc, 16);
 8422     pinsrw(xmm1, crc, 1);
 8423   }
 8424   addptr(buf, 16);
 8425   subl(len, 4); // len > 0
 8426   jcc(Assembler::less, L_fold_tail);
 8427 
 8428   movdqa(xmm2, Address(buf,  0));
 8429   movdqa(xmm3, Address(buf, 16));
 8430   movdqa(xmm4, Address(buf, 32));
 8431   addptr(buf, 48);
 8432   subl(len, 3);
 8433   jcc(Assembler::lessEqual, L_fold_512b);
 8434 
 8435   // Fold total 512 bits of polynomial on each iteration,
 8436   // 128 bits per each of 4 parallel streams.
 8437   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1);
 8438 
 8439   align32();
 8440   BIND(L_fold_512b_loop);
 8441   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8442   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
 8443   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
 8444   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
 8445   addptr(buf, 64);
 8446   subl(len, 4);
 8447   jcc(Assembler::greater, L_fold_512b_loop);
 8448 
 8449   // Fold 512 bits to 128 bits.
 8450   BIND(L_fold_512b);
 8451   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8452   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
 8453   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
 8454   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
 8455 
 8456   // Fold the rest of 128 bits data chunks
 8457   BIND(L_fold_tail);
 8458   addl(len, 3);
 8459   jccb(Assembler::lessEqual, L_fold_128b);
 8460   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8461 
 8462   BIND(L_fold_tail_loop);
 8463   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8464   addptr(buf, 16);
 8465   decrementl(len);
 8466   jccb(Assembler::greater, L_fold_tail_loop);
 8467 
 8468   // Fold 128 bits in xmm1 down into 32 bits in crc register.
 8469   BIND(L_fold_128b);
 8470   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1);
 8471   if (UseAVX > 0) {
 8472     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
 8473     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
 8474     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
 8475   } else {
 8476     movdqa(xmm2, xmm0);
 8477     pclmulqdq(xmm2, xmm1, 0x1);
 8478     movdqa(xmm3, xmm0);
 8479     pand(xmm3, xmm2);
 8480     pclmulqdq(xmm0, xmm3, 0x1);
 8481   }
 8482   psrldq(xmm1, 8);
 8483   psrldq(xmm2, 4);
 8484   pxor(xmm0, xmm1);
 8485   pxor(xmm0, xmm2);
 8486 
 8487   // 8 8-bit folds to compute 32-bit CRC.
 8488   for (int j = 0; j < 4; j++) {
 8489     fold_8bit_crc32(xmm0, table, xmm1, rax);
 8490   }
 8491   movdl(crc, xmm0); // mov 32 bits to general register
 8492   for (int j = 0; j < 4; j++) {
 8493     fold_8bit_crc32(crc, table, rax);
 8494   }
 8495 
 8496   BIND(L_tail_restore);
 8497   movl(len, tmp); // restore
 8498   BIND(L_tail);
 8499   andl(len, 0xf);
 8500   jccb(Assembler::zero, L_exit);
 8501 
 8502   // Fold the rest of bytes
 8503   align(4);
 8504   BIND(L_tail_loop);
 8505   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 8506   update_byte_crc32(crc, rax, table);
 8507   increment(buf);
 8508   decrementl(len);
 8509   jccb(Assembler::greater, L_tail_loop);
 8510 
 8511   BIND(L_exit);
 8512   notl(crc); // ~c
 8513 }
 8514 
 8515 #ifdef _LP64
 8516 // Helper function for AVX 512 CRC32
 8517 // Fold 512-bit data chunks
 8518 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
 8519                                              Register pos, int offset) {
 8520   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
 8521   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
 8522   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
 8523   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
 8524   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
 8525 }
 8526 
 8527 // Helper function for AVX 512 CRC32
 8528 // Compute CRC32 for < 256B buffers
 8529 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos,
 8530                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
 8531                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
 8532 
 8533   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
 8534   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
 8535   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
 8536 
 8537   // check if there is enough buffer to be able to fold 16B at a time
 8538   cmpl(len, 32);
 8539   jcc(Assembler::less, L_less_than_32);
 8540 
 8541   // if there is, load the constants
 8542   movdqu(xmm10, Address(table, 1 * 16));    //rk1 and rk2 in xmm10
 8543   movdl(xmm0, crc);                        // get the initial crc value
 8544   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8545   pxor(xmm7, xmm0);
 8546 
 8547   // update the buffer pointer
 8548   addl(pos, 16);
 8549   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
 8550   subl(len, 32);
 8551   jmp(L_16B_reduction_loop);
 8552 
 8553   bind(L_less_than_32);
 8554   //mov initial crc to the return value. this is necessary for zero - length buffers.
 8555   movl(rax, crc);
 8556   testl(len, len);
 8557   jcc(Assembler::equal, L_cleanup);
 8558 
 8559   movdl(xmm0, crc);                        //get the initial crc value
 8560 
 8561   cmpl(len, 16);
 8562   jcc(Assembler::equal, L_exact_16_left);
 8563   jcc(Assembler::less, L_less_than_16_left);
 8564 
 8565   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8566   pxor(xmm7, xmm0);                       //xor the initial crc value
 8567   addl(pos, 16);
 8568   subl(len, 16);
 8569   movdqu(xmm10, Address(table, 1 * 16));    // rk1 and rk2 in xmm10
 8570   jmp(L_get_last_two_xmms);
 8571 
 8572   bind(L_less_than_16_left);
 8573   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
 8574   pxor(xmm1, xmm1);
 8575   movptr(tmp1, rsp);
 8576   movdqu(Address(tmp1, 0 * 16), xmm1);
 8577 
 8578   cmpl(len, 4);
 8579   jcc(Assembler::less, L_only_less_than_4);
 8580 
 8581   //backup the counter value
 8582   movl(tmp2, len);
 8583   cmpl(len, 8);
 8584   jcc(Assembler::less, L_less_than_8_left);
 8585 
 8586   //load 8 Bytes
 8587   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
 8588   movq(Address(tmp1, 0 * 16), rax);
 8589   addptr(tmp1, 8);
 8590   subl(len, 8);
 8591   addl(pos, 8);
 8592 
 8593   bind(L_less_than_8_left);
 8594   cmpl(len, 4);
 8595   jcc(Assembler::less, L_less_than_4_left);
 8596 
 8597   //load 4 Bytes
 8598   movl(rax, Address(buf, pos, Address::times_1, 0));
 8599   movl(Address(tmp1, 0 * 16), rax);
 8600   addptr(tmp1, 4);
 8601   subl(len, 4);
 8602   addl(pos, 4);
 8603 
 8604   bind(L_less_than_4_left);
 8605   cmpl(len, 2);
 8606   jcc(Assembler::less, L_less_than_2_left);
 8607 
 8608   // load 2 Bytes
 8609   movw(rax, Address(buf, pos, Address::times_1, 0));
 8610   movl(Address(tmp1, 0 * 16), rax);
 8611   addptr(tmp1, 2);
 8612   subl(len, 2);
 8613   addl(pos, 2);
 8614 
 8615   bind(L_less_than_2_left);
 8616   cmpl(len, 1);
 8617   jcc(Assembler::less, L_zero_left);
 8618 
 8619   // load 1 Byte
 8620   movb(rax, Address(buf, pos, Address::times_1, 0));
 8621   movb(Address(tmp1, 0 * 16), rax);
 8622 
 8623   bind(L_zero_left);
 8624   movdqu(xmm7, Address(rsp, 0));
 8625   pxor(xmm7, xmm0);                       //xor the initial crc value
 8626 
 8627   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8628   movdqu(xmm0, Address(rax, tmp2));
 8629   pshufb(xmm7, xmm0);
 8630   jmp(L_128_done);
 8631 
 8632   bind(L_exact_16_left);
 8633   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
 8634   pxor(xmm7, xmm0);                       //xor the initial crc value
 8635   jmp(L_128_done);
 8636 
 8637   bind(L_only_less_than_4);
 8638   cmpl(len, 3);
 8639   jcc(Assembler::less, L_only_less_than_3);
 8640 
 8641   // load 3 Bytes
 8642   movb(rax, Address(buf, pos, Address::times_1, 0));
 8643   movb(Address(tmp1, 0), rax);
 8644 
 8645   movb(rax, Address(buf, pos, Address::times_1, 1));
 8646   movb(Address(tmp1, 1), rax);
 8647 
 8648   movb(rax, Address(buf, pos, Address::times_1, 2));
 8649   movb(Address(tmp1, 2), rax);
 8650 
 8651   movdqu(xmm7, Address(rsp, 0));
 8652   pxor(xmm7, xmm0);                     //xor the initial crc value
 8653 
 8654   pslldq(xmm7, 0x5);
 8655   jmp(L_barrett);
 8656   bind(L_only_less_than_3);
 8657   cmpl(len, 2);
 8658   jcc(Assembler::less, L_only_less_than_2);
 8659 
 8660   // load 2 Bytes
 8661   movb(rax, Address(buf, pos, Address::times_1, 0));
 8662   movb(Address(tmp1, 0), rax);
 8663 
 8664   movb(rax, Address(buf, pos, Address::times_1, 1));
 8665   movb(Address(tmp1, 1), rax);
 8666 
 8667   movdqu(xmm7, Address(rsp, 0));
 8668   pxor(xmm7, xmm0);                     //xor the initial crc value
 8669 
 8670   pslldq(xmm7, 0x6);
 8671   jmp(L_barrett);
 8672 
 8673   bind(L_only_less_than_2);
 8674   //load 1 Byte
 8675   movb(rax, Address(buf, pos, Address::times_1, 0));
 8676   movb(Address(tmp1, 0), rax);
 8677 
 8678   movdqu(xmm7, Address(rsp, 0));
 8679   pxor(xmm7, xmm0);                     //xor the initial crc value
 8680 
 8681   pslldq(xmm7, 0x7);
 8682 }
 8683 
 8684 /**
 8685 * Compute CRC32 using AVX512 instructions
 8686 * param crc   register containing existing CRC (32-bit)
 8687 * param buf   register pointing to input byte buffer (byte*)
 8688 * param len   register containing number of bytes
 8689 * param table address of crc or crc32c table
 8690 * param tmp1  scratch register
 8691 * param tmp2  scratch register
 8692 * return rax  result register
 8693 *
 8694 * This routine is identical for crc32c with the exception of the precomputed constant
 8695 * table which will be passed as the table argument.  The calculation steps are
 8696 * the same for both variants.
 8697 */
 8698 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) {
 8699   assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12);
 8700 
 8701   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 8702   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 8703   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
 8704   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
 8705   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
 8706 
 8707   const Register pos = r12;
 8708   push(r12);
 8709   subptr(rsp, 16 * 2 + 8);
 8710 
 8711   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 8712   // context for the registers used, where all instructions below are using 128-bit mode
 8713   // On EVEX without VL and BW, these instructions will all be AVX.
 8714   movl(pos, 0);
 8715 
 8716   // check if smaller than 256B
 8717   cmpl(len, 256);
 8718   jcc(Assembler::less, L_less_than_256);
 8719 
 8720   // load the initial crc value
 8721   movdl(xmm10, crc);
 8722 
 8723   // receive the initial 64B data, xor the initial crc value
 8724   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
 8725   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
 8726   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
 8727   evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
 8728 
 8729   subl(len, 256);
 8730   cmpl(len, 256);
 8731   jcc(Assembler::less, L_fold_128_B_loop);
 8732 
 8733   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
 8734   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
 8735   evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
 8736   subl(len, 256);
 8737 
 8738   bind(L_fold_256_B_loop);
 8739   addl(pos, 256);
 8740   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
 8741   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
 8742   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
 8743   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
 8744 
 8745   subl(len, 256);
 8746   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
 8747 
 8748   // Fold 256 into 128
 8749   addl(pos, 256);
 8750   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
 8751   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
 8752   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
 8753 
 8754   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
 8755   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
 8756   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
 8757 
 8758   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
 8759   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
 8760 
 8761   addl(len, 128);
 8762   jmp(L_fold_128_B_register);
 8763 
 8764   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
 8765   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
 8766 
 8767   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
 8768   bind(L_fold_128_B_loop);
 8769   addl(pos, 128);
 8770   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
 8771   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
 8772 
 8773   subl(len, 128);
 8774   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
 8775 
 8776   addl(pos, 128);
 8777 
 8778   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
 8779   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
 8780   bind(L_fold_128_B_register);
 8781   evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
 8782   evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
 8783   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
 8784   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
 8785   // save last that has no multiplicand
 8786   vextracti64x2(xmm7, xmm4, 3);
 8787 
 8788   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
 8789   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
 8790   // Needed later in reduction loop
 8791   movdqu(xmm10, Address(table, 1 * 16));
 8792   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
 8793   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
 8794 
 8795   // Swap 1,0,3,2 - 01 00 11 10
 8796   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
 8797   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
 8798   vextracti128(xmm5, xmm8, 1);
 8799   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
 8800 
 8801   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
 8802   // instead of a cmp instruction, we use the negative flag with the jl instruction
 8803   addl(len, 128 - 16);
 8804   jcc(Assembler::less, L_final_reduction_for_128);
 8805 
 8806   bind(L_16B_reduction_loop);
 8807   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8808   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8809   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8810   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
 8811   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8812   addl(pos, 16);
 8813   subl(len, 16);
 8814   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
 8815 
 8816   bind(L_final_reduction_for_128);
 8817   addl(len, 16);
 8818   jcc(Assembler::equal, L_128_done);
 8819 
 8820   bind(L_get_last_two_xmms);
 8821   movdqu(xmm2, xmm7);
 8822   addl(pos, len);
 8823   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
 8824   subl(pos, len);
 8825 
 8826   // get rid of the extra data that was loaded before
 8827   // load the shift constant
 8828   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8829   movdqu(xmm0, Address(rax, len));
 8830   addl(rax, len);
 8831 
 8832   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8833   //Change mask to 512
 8834   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
 8835   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
 8836 
 8837   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
 8838   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8839   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8840   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8841   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
 8842 
 8843   bind(L_128_done);
 8844   // compute crc of a 128-bit value
 8845   movdqu(xmm10, Address(table, 3 * 16));
 8846   movdqu(xmm0, xmm7);
 8847 
 8848   // 64b fold
 8849   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
 8850   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
 8851   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8852 
 8853   // 32b fold
 8854   movdqu(xmm0, xmm7);
 8855   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
 8856   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8857   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8858   jmp(L_barrett);
 8859 
 8860   bind(L_less_than_256);
 8861   kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
 8862 
 8863   //barrett reduction
 8864   bind(L_barrett);
 8865   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
 8866   movdqu(xmm1, xmm7);
 8867   movdqu(xmm2, xmm7);
 8868   movdqu(xmm10, Address(table, 4 * 16));
 8869 
 8870   pclmulqdq(xmm7, xmm10, 0x0);
 8871   pxor(xmm7, xmm2);
 8872   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
 8873   movdqu(xmm2, xmm7);
 8874   pclmulqdq(xmm7, xmm10, 0x10);
 8875   pxor(xmm7, xmm2);
 8876   pxor(xmm7, xmm1);
 8877   pextrd(crc, xmm7, 2);
 8878 
 8879   bind(L_cleanup);
 8880   addptr(rsp, 16 * 2 + 8);
 8881   pop(r12);
 8882 }
 8883 
 8884 // S. Gueron / Information Processing Letters 112 (2012) 184
 8885 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
 8886 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
 8887 // Output: the 64-bit carry-less product of B * CONST
 8888 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
 8889                                      Register tmp1, Register tmp2, Register tmp3) {
 8890   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8891   if (n > 0) {
 8892     addq(tmp3, n * 256 * 8);
 8893   }
 8894   //    Q1 = TABLEExt[n][B & 0xFF];
 8895   movl(tmp1, in);
 8896   andl(tmp1, 0x000000FF);
 8897   shll(tmp1, 3);
 8898   addq(tmp1, tmp3);
 8899   movq(tmp1, Address(tmp1, 0));
 8900 
 8901   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8902   movl(tmp2, in);
 8903   shrl(tmp2, 8);
 8904   andl(tmp2, 0x000000FF);
 8905   shll(tmp2, 3);
 8906   addq(tmp2, tmp3);
 8907   movq(tmp2, Address(tmp2, 0));
 8908 
 8909   shlq(tmp2, 8);
 8910   xorq(tmp1, tmp2);
 8911 
 8912   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8913   movl(tmp2, in);
 8914   shrl(tmp2, 16);
 8915   andl(tmp2, 0x000000FF);
 8916   shll(tmp2, 3);
 8917   addq(tmp2, tmp3);
 8918   movq(tmp2, Address(tmp2, 0));
 8919 
 8920   shlq(tmp2, 16);
 8921   xorq(tmp1, tmp2);
 8922 
 8923   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8924   shrl(in, 24);
 8925   andl(in, 0x000000FF);
 8926   shll(in, 3);
 8927   addq(in, tmp3);
 8928   movq(in, Address(in, 0));
 8929 
 8930   shlq(in, 24);
 8931   xorq(in, tmp1);
 8932   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8933 }
 8934 
 8935 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8936                                       Register in_out,
 8937                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8938                                       XMMRegister w_xtmp2,
 8939                                       Register tmp1,
 8940                                       Register n_tmp2, Register n_tmp3) {
 8941   if (is_pclmulqdq_supported) {
 8942     movdl(w_xtmp1, in_out); // modified blindly
 8943 
 8944     movl(tmp1, const_or_pre_comp_const_index);
 8945     movdl(w_xtmp2, tmp1);
 8946     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8947 
 8948     movdq(in_out, w_xtmp1);
 8949   } else {
 8950     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
 8951   }
 8952 }
 8953 
 8954 // Recombination Alternative 2: No bit-reflections
 8955 // T1 = (CRC_A * U1) << 1
 8956 // T2 = (CRC_B * U2) << 1
 8957 // C1 = T1 >> 32
 8958 // C2 = T2 >> 32
 8959 // T1 = T1 & 0xFFFFFFFF
 8960 // T2 = T2 & 0xFFFFFFFF
 8961 // T1 = CRC32(0, T1)
 8962 // T2 = CRC32(0, T2)
 8963 // C1 = C1 ^ T1
 8964 // C2 = C2 ^ T2
 8965 // CRC = C1 ^ C2 ^ CRC_C
 8966 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 8967                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8968                                      Register tmp1, Register tmp2,
 8969                                      Register n_tmp3) {
 8970   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8971   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8972   shlq(in_out, 1);
 8973   movl(tmp1, in_out);
 8974   shrq(in_out, 32);
 8975   xorl(tmp2, tmp2);
 8976   crc32(tmp2, tmp1, 4);
 8977   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
 8978   shlq(in1, 1);
 8979   movl(tmp1, in1);
 8980   shrq(in1, 32);
 8981   xorl(tmp2, tmp2);
 8982   crc32(tmp2, tmp1, 4);
 8983   xorl(in1, tmp2);
 8984   xorl(in_out, in1);
 8985   xorl(in_out, in2);
 8986 }
 8987 
 8988 // Set N to predefined value
 8989 // Subtract from a length of a buffer
 8990 // execute in a loop:
 8991 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
 8992 // for i = 1 to N do
 8993 //  CRC_A = CRC32(CRC_A, A[i])
 8994 //  CRC_B = CRC32(CRC_B, B[i])
 8995 //  CRC_C = CRC32(CRC_C, C[i])
 8996 // end for
 8997 // Recombine
 8998 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 8999                                        Register in_out1, Register in_out2, Register in_out3,
 9000                                        Register tmp1, Register tmp2, Register tmp3,
 9001                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9002                                        Register tmp4, Register tmp5,
 9003                                        Register n_tmp6) {
 9004   Label L_processPartitions;
 9005   Label L_processPartition;
 9006   Label L_exit;
 9007 
 9008   bind(L_processPartitions);
 9009   cmpl(in_out1, 3 * size);
 9010   jcc(Assembler::less, L_exit);
 9011     xorl(tmp1, tmp1);
 9012     xorl(tmp2, tmp2);
 9013     movq(tmp3, in_out2);
 9014     addq(tmp3, size);
 9015 
 9016     bind(L_processPartition);
 9017       crc32(in_out3, Address(in_out2, 0), 8);
 9018       crc32(tmp1, Address(in_out2, size), 8);
 9019       crc32(tmp2, Address(in_out2, size * 2), 8);
 9020       addq(in_out2, 8);
 9021       cmpq(in_out2, tmp3);
 9022       jcc(Assembler::less, L_processPartition);
 9023     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 9024             w_xtmp1, w_xtmp2, w_xtmp3,
 9025             tmp4, tmp5,
 9026             n_tmp6);
 9027     addq(in_out2, 2 * size);
 9028     subl(in_out1, 3 * size);
 9029     jmp(L_processPartitions);
 9030 
 9031   bind(L_exit);
 9032 }
 9033 #else
 9034 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
 9035                                      Register tmp1, Register tmp2, Register tmp3,
 9036                                      XMMRegister xtmp1, XMMRegister xtmp2) {
 9037   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 9038   if (n > 0) {
 9039     addl(tmp3, n * 256 * 8);
 9040   }
 9041   //    Q1 = TABLEExt[n][B & 0xFF];
 9042   movl(tmp1, in_out);
 9043   andl(tmp1, 0x000000FF);
 9044   shll(tmp1, 3);
 9045   addl(tmp1, tmp3);
 9046   movq(xtmp1, Address(tmp1, 0));
 9047 
 9048   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 9049   movl(tmp2, in_out);
 9050   shrl(tmp2, 8);
 9051   andl(tmp2, 0x000000FF);
 9052   shll(tmp2, 3);
 9053   addl(tmp2, tmp3);
 9054   movq(xtmp2, Address(tmp2, 0));
 9055 
 9056   psllq(xtmp2, 8);
 9057   pxor(xtmp1, xtmp2);
 9058 
 9059   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 9060   movl(tmp2, in_out);
 9061   shrl(tmp2, 16);
 9062   andl(tmp2, 0x000000FF);
 9063   shll(tmp2, 3);
 9064   addl(tmp2, tmp3);
 9065   movq(xtmp2, Address(tmp2, 0));
 9066 
 9067   psllq(xtmp2, 16);
 9068   pxor(xtmp1, xtmp2);
 9069 
 9070   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 9071   shrl(in_out, 24);
 9072   andl(in_out, 0x000000FF);
 9073   shll(in_out, 3);
 9074   addl(in_out, tmp3);
 9075   movq(xtmp2, Address(in_out, 0));
 9076 
 9077   psllq(xtmp2, 24);
 9078   pxor(xtmp1, xtmp2); // Result in CXMM
 9079   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 9080 }
 9081 
 9082 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 9083                                       Register in_out,
 9084                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 9085                                       XMMRegister w_xtmp2,
 9086                                       Register tmp1,
 9087                                       Register n_tmp2, Register n_tmp3) {
 9088   if (is_pclmulqdq_supported) {
 9089     movdl(w_xtmp1, in_out);
 9090 
 9091     movl(tmp1, const_or_pre_comp_const_index);
 9092     movdl(w_xtmp2, tmp1);
 9093     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 9094     // Keep result in XMM since GPR is 32 bit in length
 9095   } else {
 9096     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
 9097   }
 9098 }
 9099 
 9100 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 9101                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9102                                      Register tmp1, Register tmp2,
 9103                                      Register n_tmp3) {
 9104   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9105   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9106 
 9107   psllq(w_xtmp1, 1);
 9108   movdl(tmp1, w_xtmp1);
 9109   psrlq(w_xtmp1, 32);
 9110   movdl(in_out, w_xtmp1);
 9111 
 9112   xorl(tmp2, tmp2);
 9113   crc32(tmp2, tmp1, 4);
 9114   xorl(in_out, tmp2);
 9115 
 9116   psllq(w_xtmp2, 1);
 9117   movdl(tmp1, w_xtmp2);
 9118   psrlq(w_xtmp2, 32);
 9119   movdl(in1, w_xtmp2);
 9120 
 9121   xorl(tmp2, tmp2);
 9122   crc32(tmp2, tmp1, 4);
 9123   xorl(in1, tmp2);
 9124   xorl(in_out, in1);
 9125   xorl(in_out, in2);
 9126 }
 9127 
 9128 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 9129                                        Register in_out1, Register in_out2, Register in_out3,
 9130                                        Register tmp1, Register tmp2, Register tmp3,
 9131                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9132                                        Register tmp4, Register tmp5,
 9133                                        Register n_tmp6) {
 9134   Label L_processPartitions;
 9135   Label L_processPartition;
 9136   Label L_exit;
 9137 
 9138   bind(L_processPartitions);
 9139   cmpl(in_out1, 3 * size);
 9140   jcc(Assembler::less, L_exit);
 9141     xorl(tmp1, tmp1);
 9142     xorl(tmp2, tmp2);
 9143     movl(tmp3, in_out2);
 9144     addl(tmp3, size);
 9145 
 9146     bind(L_processPartition);
 9147       crc32(in_out3, Address(in_out2, 0), 4);
 9148       crc32(tmp1, Address(in_out2, size), 4);
 9149       crc32(tmp2, Address(in_out2, size*2), 4);
 9150       crc32(in_out3, Address(in_out2, 0+4), 4);
 9151       crc32(tmp1, Address(in_out2, size+4), 4);
 9152       crc32(tmp2, Address(in_out2, size*2+4), 4);
 9153       addl(in_out2, 8);
 9154       cmpl(in_out2, tmp3);
 9155       jcc(Assembler::less, L_processPartition);
 9156 
 9157         push(tmp3);
 9158         push(in_out1);
 9159         push(in_out2);
 9160         tmp4 = tmp3;
 9161         tmp5 = in_out1;
 9162         n_tmp6 = in_out2;
 9163 
 9164       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 9165             w_xtmp1, w_xtmp2, w_xtmp3,
 9166             tmp4, tmp5,
 9167             n_tmp6);
 9168 
 9169         pop(in_out2);
 9170         pop(in_out1);
 9171         pop(tmp3);
 9172 
 9173     addl(in_out2, 2 * size);
 9174     subl(in_out1, 3 * size);
 9175     jmp(L_processPartitions);
 9176 
 9177   bind(L_exit);
 9178 }
 9179 #endif //LP64
 9180 
 9181 #ifdef _LP64
 9182 // Algorithm 2: Pipelined usage of the CRC32 instruction.
 9183 // Input: A buffer I of L bytes.
 9184 // Output: the CRC32C value of the buffer.
 9185 // Notations:
 9186 // Write L = 24N + r, with N = floor (L/24).
 9187 // r = L mod 24 (0 <= r < 24).
 9188 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
 9189 // N quadwords, and R consists of r bytes.
 9190 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
 9191 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
 9192 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
 9193 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
 9194 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 9195                                           Register tmp1, Register tmp2, Register tmp3,
 9196                                           Register tmp4, Register tmp5, Register tmp6,
 9197                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9198                                           bool is_pclmulqdq_supported) {
 9199   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 9200   Label L_wordByWord;
 9201   Label L_byteByByteProlog;
 9202   Label L_byteByByte;
 9203   Label L_exit;
 9204 
 9205   if (is_pclmulqdq_supported ) {
 9206     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
 9207     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
 9208 
 9209     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
 9210     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
 9211 
 9212     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
 9213     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
 9214     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
 9215   } else {
 9216     const_or_pre_comp_const_index[0] = 1;
 9217     const_or_pre_comp_const_index[1] = 0;
 9218 
 9219     const_or_pre_comp_const_index[2] = 3;
 9220     const_or_pre_comp_const_index[3] = 2;
 9221 
 9222     const_or_pre_comp_const_index[4] = 5;
 9223     const_or_pre_comp_const_index[5] = 4;
 9224    }
 9225   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 9226                     in2, in1, in_out,
 9227                     tmp1, tmp2, tmp3,
 9228                     w_xtmp1, w_xtmp2, w_xtmp3,
 9229                     tmp4, tmp5,
 9230                     tmp6);
 9231   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 9232                     in2, in1, in_out,
 9233                     tmp1, tmp2, tmp3,
 9234                     w_xtmp1, w_xtmp2, w_xtmp3,
 9235                     tmp4, tmp5,
 9236                     tmp6);
 9237   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 9238                     in2, in1, in_out,
 9239                     tmp1, tmp2, tmp3,
 9240                     w_xtmp1, w_xtmp2, w_xtmp3,
 9241                     tmp4, tmp5,
 9242                     tmp6);
 9243   movl(tmp1, in2);
 9244   andl(tmp1, 0x00000007);
 9245   negl(tmp1);
 9246   addl(tmp1, in2);
 9247   addq(tmp1, in1);
 9248 
 9249   cmpq(in1, tmp1);
 9250   jccb(Assembler::greaterEqual, L_byteByByteProlog);
 9251   align(16);
 9252   BIND(L_wordByWord);
 9253     crc32(in_out, Address(in1, 0), 8);
 9254     addq(in1, 8);
 9255     cmpq(in1, tmp1);
 9256     jcc(Assembler::less, L_wordByWord);
 9257 
 9258   BIND(L_byteByByteProlog);
 9259   andl(in2, 0x00000007);
 9260   movl(tmp2, 1);
 9261 
 9262   cmpl(tmp2, in2);
 9263   jccb(Assembler::greater, L_exit);
 9264   BIND(L_byteByByte);
 9265     crc32(in_out, Address(in1, 0), 1);
 9266     incq(in1);
 9267     incl(tmp2);
 9268     cmpl(tmp2, in2);
 9269     jcc(Assembler::lessEqual, L_byteByByte);
 9270 
 9271   BIND(L_exit);
 9272 }
 9273 #else
 9274 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 9275                                           Register tmp1, Register  tmp2, Register tmp3,
 9276                                           Register tmp4, Register  tmp5, Register tmp6,
 9277                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9278                                           bool is_pclmulqdq_supported) {
 9279   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 9280   Label L_wordByWord;
 9281   Label L_byteByByteProlog;
 9282   Label L_byteByByte;
 9283   Label L_exit;
 9284 
 9285   if (is_pclmulqdq_supported) {
 9286     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
 9287     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
 9288 
 9289     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
 9290     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
 9291 
 9292     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
 9293     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
 9294   } else {
 9295     const_or_pre_comp_const_index[0] = 1;
 9296     const_or_pre_comp_const_index[1] = 0;
 9297 
 9298     const_or_pre_comp_const_index[2] = 3;
 9299     const_or_pre_comp_const_index[3] = 2;
 9300 
 9301     const_or_pre_comp_const_index[4] = 5;
 9302     const_or_pre_comp_const_index[5] = 4;
 9303   }
 9304   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 9305                     in2, in1, in_out,
 9306                     tmp1, tmp2, tmp3,
 9307                     w_xtmp1, w_xtmp2, w_xtmp3,
 9308                     tmp4, tmp5,
 9309                     tmp6);
 9310   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 9311                     in2, in1, in_out,
 9312                     tmp1, tmp2, tmp3,
 9313                     w_xtmp1, w_xtmp2, w_xtmp3,
 9314                     tmp4, tmp5,
 9315                     tmp6);
 9316   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 9317                     in2, in1, in_out,
 9318                     tmp1, tmp2, tmp3,
 9319                     w_xtmp1, w_xtmp2, w_xtmp3,
 9320                     tmp4, tmp5,
 9321                     tmp6);
 9322   movl(tmp1, in2);
 9323   andl(tmp1, 0x00000007);
 9324   negl(tmp1);
 9325   addl(tmp1, in2);
 9326   addl(tmp1, in1);
 9327 
 9328   BIND(L_wordByWord);
 9329   cmpl(in1, tmp1);
 9330   jcc(Assembler::greaterEqual, L_byteByByteProlog);
 9331     crc32(in_out, Address(in1,0), 4);
 9332     addl(in1, 4);
 9333     jmp(L_wordByWord);
 9334 
 9335   BIND(L_byteByByteProlog);
 9336   andl(in2, 0x00000007);
 9337   movl(tmp2, 1);
 9338 
 9339   BIND(L_byteByByte);
 9340   cmpl(tmp2, in2);
 9341   jccb(Assembler::greater, L_exit);
 9342     movb(tmp1, Address(in1, 0));
 9343     crc32(in_out, tmp1, 1);
 9344     incl(in1);
 9345     incl(tmp2);
 9346     jmp(L_byteByByte);
 9347 
 9348   BIND(L_exit);
 9349 }
 9350 #endif // LP64
 9351 #undef BIND
 9352 #undef BLOCK_COMMENT
 9353 
 9354 // Compress char[] array to byte[].
 9355 //   ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java
 9356 //   @IntrinsicCandidate
 9357 //   private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
 9358 //     for (int i = 0; i < len; i++) {
 9359 //       int c = src[srcOff++];
 9360 //       if (c >>> 8 != 0) {
 9361 //         return 0;
 9362 //       }
 9363 //       dst[dstOff++] = (byte)c;
 9364 //     }
 9365 //     return len;
 9366 //   }
 9367 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
 9368   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 9369   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 9370   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
 9371   Label copy_chars_loop, return_length, return_zero, done;
 9372 
 9373   // rsi: src
 9374   // rdi: dst
 9375   // rdx: len
 9376   // rcx: tmp5
 9377   // rax: result
 9378 
 9379   // rsi holds start addr of source char[] to be compressed
 9380   // rdi holds start addr of destination byte[]
 9381   // rdx holds length
 9382 
 9383   assert(len != result, "");
 9384 
 9385   // save length for return
 9386   push(len);
 9387 
 9388   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
 9389     VM_Version::supports_avx512vlbw() &&
 9390     VM_Version::supports_bmi2()) {
 9391 
 9392     Label copy_32_loop, copy_loop_tail, below_threshold;
 9393 
 9394     // alignment
 9395     Label post_alignment;
 9396 
 9397     // if length of the string is less than 16, handle it in an old fashioned way
 9398     testl(len, -32);
 9399     jcc(Assembler::zero, below_threshold);
 9400 
 9401     // First check whether a character is compressible ( <= 0xFF).
 9402     // Create mask to test for Unicode chars inside zmm vector
 9403     movl(result, 0x00FF);
 9404     evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit);
 9405 
 9406     testl(len, -64);
 9407     jcc(Assembler::zero, post_alignment);
 9408 
 9409     movl(tmp5, dst);
 9410     andl(tmp5, (32 - 1));
 9411     negl(tmp5);
 9412     andl(tmp5, (32 - 1));
 9413 
 9414     // bail out when there is nothing to be done
 9415     testl(tmp5, 0xFFFFFFFF);
 9416     jcc(Assembler::zero, post_alignment);
 9417 
 9418     // ~(~0 << len), where len is the # of remaining elements to process
 9419     movl(result, 0xFFFFFFFF);
 9420     shlxl(result, result, tmp5);
 9421     notl(result);
 9422     kmovdl(mask2, result);
 9423 
 9424     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9425     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9426     ktestd(mask1, mask2);
 9427     jcc(Assembler::carryClear, return_zero);
 9428 
 9429     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9430 
 9431     addptr(src, tmp5);
 9432     addptr(src, tmp5);
 9433     addptr(dst, tmp5);
 9434     subl(len, tmp5);
 9435 
 9436     bind(post_alignment);
 9437     // end of alignment
 9438 
 9439     movl(tmp5, len);
 9440     andl(tmp5, (32 - 1));    // tail count (in chars)
 9441     andl(len, ~(32 - 1));    // vector count (in chars)
 9442     jcc(Assembler::zero, copy_loop_tail);
 9443 
 9444     lea(src, Address(src, len, Address::times_2));
 9445     lea(dst, Address(dst, len, Address::times_1));
 9446     negptr(len);
 9447 
 9448     bind(copy_32_loop);
 9449     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
 9450     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
 9451     kortestdl(mask1, mask1);
 9452     jcc(Assembler::carryClear, return_zero);
 9453 
 9454     // All elements in current processed chunk are valid candidates for
 9455     // compression. Write a truncated byte elements to the memory.
 9456     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
 9457     addptr(len, 32);
 9458     jcc(Assembler::notZero, copy_32_loop);
 9459 
 9460     bind(copy_loop_tail);
 9461     // bail out when there is nothing to be done
 9462     testl(tmp5, 0xFFFFFFFF);
 9463     jcc(Assembler::zero, return_length);
 9464 
 9465     movl(len, tmp5);
 9466 
 9467     // ~(~0 << len), where len is the # of remaining elements to process
 9468     movl(result, 0xFFFFFFFF);
 9469     shlxl(result, result, len);
 9470     notl(result);
 9471 
 9472     kmovdl(mask2, result);
 9473 
 9474     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9475     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9476     ktestd(mask1, mask2);
 9477     jcc(Assembler::carryClear, return_zero);
 9478 
 9479     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9480     jmp(return_length);
 9481 
 9482     bind(below_threshold);
 9483   }
 9484 
 9485   if (UseSSE42Intrinsics) {
 9486     Label copy_32_loop, copy_16, copy_tail;
 9487 
 9488     movl(result, len);
 9489 
 9490     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
 9491 
 9492     // vectored compression
 9493     andl(len, 0xfffffff0);    // vector count (in chars)
 9494     andl(result, 0x0000000f);    // tail count (in chars)
 9495     testl(len, len);
 9496     jcc(Assembler::zero, copy_16);
 9497 
 9498     // compress 16 chars per iter
 9499     movdl(tmp1Reg, tmp5);
 9500     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
 9501     pxor(tmp4Reg, tmp4Reg);
 9502 
 9503     lea(src, Address(src, len, Address::times_2));
 9504     lea(dst, Address(dst, len, Address::times_1));
 9505     negptr(len);
 9506 
 9507     bind(copy_32_loop);
 9508     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
 9509     por(tmp4Reg, tmp2Reg);
 9510     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
 9511     por(tmp4Reg, tmp3Reg);
 9512     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
 9513     jcc(Assembler::notZero, return_zero);
 9514     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
 9515     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
 9516     addptr(len, 16);
 9517     jcc(Assembler::notZero, copy_32_loop);
 9518 
 9519     // compress next vector of 8 chars (if any)
 9520     bind(copy_16);
 9521     movl(len, result);
 9522     andl(len, 0xfffffff8);    // vector count (in chars)
 9523     andl(result, 0x00000007);    // tail count (in chars)
 9524     testl(len, len);
 9525     jccb(Assembler::zero, copy_tail);
 9526 
 9527     movdl(tmp1Reg, tmp5);
 9528     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
 9529     pxor(tmp3Reg, tmp3Reg);
 9530 
 9531     movdqu(tmp2Reg, Address(src, 0));
 9532     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
 9533     jccb(Assembler::notZero, return_zero);
 9534     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
 9535     movq(Address(dst, 0), tmp2Reg);
 9536     addptr(src, 16);
 9537     addptr(dst, 8);
 9538 
 9539     bind(copy_tail);
 9540     movl(len, result);
 9541   }
 9542   // compress 1 char per iter
 9543   testl(len, len);
 9544   jccb(Assembler::zero, return_length);
 9545   lea(src, Address(src, len, Address::times_2));
 9546   lea(dst, Address(dst, len, Address::times_1));
 9547   negptr(len);
 9548 
 9549   bind(copy_chars_loop);
 9550   load_unsigned_short(result, Address(src, len, Address::times_2));
 9551   testl(result, 0xff00);      // check if Unicode char
 9552   jccb(Assembler::notZero, return_zero);
 9553   movb(Address(dst, len, Address::times_1), result);  // ASCII char; compress to 1 byte
 9554   increment(len);
 9555   jcc(Assembler::notZero, copy_chars_loop);
 9556 
 9557   // if compression succeeded, return length
 9558   bind(return_length);
 9559   pop(result);
 9560   jmpb(done);
 9561 
 9562   // if compression failed, return 0
 9563   bind(return_zero);
 9564   xorl(result, result);
 9565   addptr(rsp, wordSize);
 9566 
 9567   bind(done);
 9568 }
 9569 
 9570 // Inflate byte[] array to char[].
 9571 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
 9572 //   @IntrinsicCandidate
 9573 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
 9574 //     for (int i = 0; i < len; i++) {
 9575 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
 9576 //     }
 9577 //   }
 9578 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
 9579   XMMRegister tmp1, Register tmp2, KRegister mask) {
 9580   Label copy_chars_loop, done, below_threshold, avx3_threshold;
 9581   // rsi: src
 9582   // rdi: dst
 9583   // rdx: len
 9584   // rcx: tmp2
 9585 
 9586   // rsi holds start addr of source byte[] to be inflated
 9587   // rdi holds start addr of destination char[]
 9588   // rdx holds length
 9589   assert_different_registers(src, dst, len, tmp2);
 9590   movl(tmp2, len);
 9591   if ((UseAVX > 2) && // AVX512
 9592     VM_Version::supports_avx512vlbw() &&
 9593     VM_Version::supports_bmi2()) {
 9594 
 9595     Label copy_32_loop, copy_tail;
 9596     Register tmp3_aliased = len;
 9597 
 9598     // if length of the string is less than 16, handle it in an old fashioned way
 9599     testl(len, -16);
 9600     jcc(Assembler::zero, below_threshold);
 9601 
 9602     testl(len, -1 * AVX3Threshold);
 9603     jcc(Assembler::zero, avx3_threshold);
 9604 
 9605     // In order to use only one arithmetic operation for the main loop we use
 9606     // this pre-calculation
 9607     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
 9608     andl(len, -32);     // vector count
 9609     jccb(Assembler::zero, copy_tail);
 9610 
 9611     lea(src, Address(src, len, Address::times_1));
 9612     lea(dst, Address(dst, len, Address::times_2));
 9613     negptr(len);
 9614 
 9615 
 9616     // inflate 32 chars per iter
 9617     bind(copy_32_loop);
 9618     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
 9619     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
 9620     addptr(len, 32);
 9621     jcc(Assembler::notZero, copy_32_loop);
 9622 
 9623     bind(copy_tail);
 9624     // bail out when there is nothing to be done
 9625     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
 9626     jcc(Assembler::zero, done);
 9627 
 9628     // ~(~0 << length), where length is the # of remaining elements to process
 9629     movl(tmp3_aliased, -1);
 9630     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
 9631     notl(tmp3_aliased);
 9632     kmovdl(mask, tmp3_aliased);
 9633     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
 9634     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
 9635 
 9636     jmp(done);
 9637     bind(avx3_threshold);
 9638   }
 9639   if (UseSSE42Intrinsics) {
 9640     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
 9641 
 9642     if (UseAVX > 1) {
 9643       andl(tmp2, (16 - 1));
 9644       andl(len, -16);
 9645       jccb(Assembler::zero, copy_new_tail);
 9646     } else {
 9647       andl(tmp2, 0x00000007);   // tail count (in chars)
 9648       andl(len, 0xfffffff8);    // vector count (in chars)
 9649       jccb(Assembler::zero, copy_tail);
 9650     }
 9651 
 9652     // vectored inflation
 9653     lea(src, Address(src, len, Address::times_1));
 9654     lea(dst, Address(dst, len, Address::times_2));
 9655     negptr(len);
 9656 
 9657     if (UseAVX > 1) {
 9658       bind(copy_16_loop);
 9659       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
 9660       vmovdqu(Address(dst, len, Address::times_2), tmp1);
 9661       addptr(len, 16);
 9662       jcc(Assembler::notZero, copy_16_loop);
 9663 
 9664       bind(below_threshold);
 9665       bind(copy_new_tail);
 9666       movl(len, tmp2);
 9667       andl(tmp2, 0x00000007);
 9668       andl(len, 0xFFFFFFF8);
 9669       jccb(Assembler::zero, copy_tail);
 9670 
 9671       pmovzxbw(tmp1, Address(src, 0));
 9672       movdqu(Address(dst, 0), tmp1);
 9673       addptr(src, 8);
 9674       addptr(dst, 2 * 8);
 9675 
 9676       jmp(copy_tail, true);
 9677     }
 9678 
 9679     // inflate 8 chars per iter
 9680     bind(copy_8_loop);
 9681     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
 9682     movdqu(Address(dst, len, Address::times_2), tmp1);
 9683     addptr(len, 8);
 9684     jcc(Assembler::notZero, copy_8_loop);
 9685 
 9686     bind(copy_tail);
 9687     movl(len, tmp2);
 9688 
 9689     cmpl(len, 4);
 9690     jccb(Assembler::less, copy_bytes);
 9691 
 9692     movdl(tmp1, Address(src, 0));  // load 4 byte chars
 9693     pmovzxbw(tmp1, tmp1);
 9694     movq(Address(dst, 0), tmp1);
 9695     subptr(len, 4);
 9696     addptr(src, 4);
 9697     addptr(dst, 8);
 9698 
 9699     bind(copy_bytes);
 9700   } else {
 9701     bind(below_threshold);
 9702   }
 9703 
 9704   testl(len, len);
 9705   jccb(Assembler::zero, done);
 9706   lea(src, Address(src, len, Address::times_1));
 9707   lea(dst, Address(dst, len, Address::times_2));
 9708   negptr(len);
 9709 
 9710   // inflate 1 char per iter
 9711   bind(copy_chars_loop);
 9712   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
 9713   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
 9714   increment(len);
 9715   jcc(Assembler::notZero, copy_chars_loop);
 9716 
 9717   bind(done);
 9718 }
 9719 
 9720 
 9721 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
 9722   switch(type) {
 9723     case T_BYTE:
 9724     case T_BOOLEAN:
 9725       evmovdqub(dst, kmask, src, merge, vector_len);
 9726       break;
 9727     case T_CHAR:
 9728     case T_SHORT:
 9729       evmovdquw(dst, kmask, src, merge, vector_len);
 9730       break;
 9731     case T_INT:
 9732     case T_FLOAT:
 9733       evmovdqul(dst, kmask, src, merge, vector_len);
 9734       break;
 9735     case T_LONG:
 9736     case T_DOUBLE:
 9737       evmovdquq(dst, kmask, src, merge, vector_len);
 9738       break;
 9739     default:
 9740       fatal("Unexpected type argument %s", type2name(type));
 9741       break;
 9742   }
 9743 }
 9744 
 9745 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
 9746   switch(type) {
 9747     case T_BYTE:
 9748     case T_BOOLEAN:
 9749       evmovdqub(dst, kmask, src, merge, vector_len);
 9750       break;
 9751     case T_CHAR:
 9752     case T_SHORT:
 9753       evmovdquw(dst, kmask, src, merge, vector_len);
 9754       break;
 9755     case T_INT:
 9756     case T_FLOAT:
 9757       evmovdqul(dst, kmask, src, merge, vector_len);
 9758       break;
 9759     case T_LONG:
 9760     case T_DOUBLE:
 9761       evmovdquq(dst, kmask, src, merge, vector_len);
 9762       break;
 9763     default:
 9764       fatal("Unexpected type argument %s", type2name(type));
 9765       break;
 9766   }
 9767 }
 9768 
 9769 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
 9770   switch(masklen) {
 9771     case 2:
 9772        knotbl(dst, src);
 9773        movl(rtmp, 3);
 9774        kmovbl(ktmp, rtmp);
 9775        kandbl(dst, ktmp, dst);
 9776        break;
 9777     case 4:
 9778        knotbl(dst, src);
 9779        movl(rtmp, 15);
 9780        kmovbl(ktmp, rtmp);
 9781        kandbl(dst, ktmp, dst);
 9782        break;
 9783     case 8:
 9784        knotbl(dst, src);
 9785        break;
 9786     case 16:
 9787        knotwl(dst, src);
 9788        break;
 9789     case 32:
 9790        knotdl(dst, src);
 9791        break;
 9792     case 64:
 9793        knotql(dst, src);
 9794        break;
 9795     default:
 9796       fatal("Unexpected vector length %d", masklen);
 9797       break;
 9798   }
 9799 }
 9800 
 9801 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9802   switch(type) {
 9803     case T_BOOLEAN:
 9804     case T_BYTE:
 9805        kandbl(dst, src1, src2);
 9806        break;
 9807     case T_CHAR:
 9808     case T_SHORT:
 9809        kandwl(dst, src1, src2);
 9810        break;
 9811     case T_INT:
 9812     case T_FLOAT:
 9813        kanddl(dst, src1, src2);
 9814        break;
 9815     case T_LONG:
 9816     case T_DOUBLE:
 9817        kandql(dst, src1, src2);
 9818        break;
 9819     default:
 9820       fatal("Unexpected type argument %s", type2name(type));
 9821       break;
 9822   }
 9823 }
 9824 
 9825 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9826   switch(type) {
 9827     case T_BOOLEAN:
 9828     case T_BYTE:
 9829        korbl(dst, src1, src2);
 9830        break;
 9831     case T_CHAR:
 9832     case T_SHORT:
 9833        korwl(dst, src1, src2);
 9834        break;
 9835     case T_INT:
 9836     case T_FLOAT:
 9837        kordl(dst, src1, src2);
 9838        break;
 9839     case T_LONG:
 9840     case T_DOUBLE:
 9841        korql(dst, src1, src2);
 9842        break;
 9843     default:
 9844       fatal("Unexpected type argument %s", type2name(type));
 9845       break;
 9846   }
 9847 }
 9848 
 9849 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9850   switch(type) {
 9851     case T_BOOLEAN:
 9852     case T_BYTE:
 9853        kxorbl(dst, src1, src2);
 9854        break;
 9855     case T_CHAR:
 9856     case T_SHORT:
 9857        kxorwl(dst, src1, src2);
 9858        break;
 9859     case T_INT:
 9860     case T_FLOAT:
 9861        kxordl(dst, src1, src2);
 9862        break;
 9863     case T_LONG:
 9864     case T_DOUBLE:
 9865        kxorql(dst, src1, src2);
 9866        break;
 9867     default:
 9868       fatal("Unexpected type argument %s", type2name(type));
 9869       break;
 9870   }
 9871 }
 9872 
 9873 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9874   switch(type) {
 9875     case T_BOOLEAN:
 9876     case T_BYTE:
 9877       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9878     case T_CHAR:
 9879     case T_SHORT:
 9880       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9881     case T_INT:
 9882     case T_FLOAT:
 9883       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9884     case T_LONG:
 9885     case T_DOUBLE:
 9886       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9887     default:
 9888       fatal("Unexpected type argument %s", type2name(type)); break;
 9889   }
 9890 }
 9891 
 9892 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9893   switch(type) {
 9894     case T_BOOLEAN:
 9895     case T_BYTE:
 9896       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9897     case T_CHAR:
 9898     case T_SHORT:
 9899       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9900     case T_INT:
 9901     case T_FLOAT:
 9902       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9903     case T_LONG:
 9904     case T_DOUBLE:
 9905       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9906     default:
 9907       fatal("Unexpected type argument %s", type2name(type)); break;
 9908   }
 9909 }
 9910 
 9911 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9912   switch(type) {
 9913     case T_BYTE:
 9914       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9915     case T_SHORT:
 9916       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9917     case T_INT:
 9918       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9919     case T_LONG:
 9920       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9921     default:
 9922       fatal("Unexpected type argument %s", type2name(type)); break;
 9923   }
 9924 }
 9925 
 9926 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9927   switch(type) {
 9928     case T_BYTE:
 9929       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9930     case T_SHORT:
 9931       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9932     case T_INT:
 9933       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9934     case T_LONG:
 9935       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9936     default:
 9937       fatal("Unexpected type argument %s", type2name(type)); break;
 9938   }
 9939 }
 9940 
 9941 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9942   switch(type) {
 9943     case T_BYTE:
 9944       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9945     case T_SHORT:
 9946       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9947     case T_INT:
 9948       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9949     case T_LONG:
 9950       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9951     default:
 9952       fatal("Unexpected type argument %s", type2name(type)); break;
 9953   }
 9954 }
 9955 
 9956 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9957   switch(type) {
 9958     case T_BYTE:
 9959       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
 9960     case T_SHORT:
 9961       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
 9962     case T_INT:
 9963       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
 9964     case T_LONG:
 9965       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
 9966     default:
 9967       fatal("Unexpected type argument %s", type2name(type)); break;
 9968   }
 9969 }
 9970 
 9971 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9972   switch(type) {
 9973     case T_INT:
 9974       evpxord(dst, mask, nds, src, merge, vector_len); break;
 9975     case T_LONG:
 9976       evpxorq(dst, mask, nds, src, merge, vector_len); break;
 9977     default:
 9978       fatal("Unexpected type argument %s", type2name(type)); break;
 9979   }
 9980 }
 9981 
 9982 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9983   switch(type) {
 9984     case T_INT:
 9985       evpxord(dst, mask, nds, src, merge, vector_len); break;
 9986     case T_LONG:
 9987       evpxorq(dst, mask, nds, src, merge, vector_len); break;
 9988     default:
 9989       fatal("Unexpected type argument %s", type2name(type)); break;
 9990   }
 9991 }
 9992 
 9993 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9994   switch(type) {
 9995     case T_INT:
 9996       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
 9997     case T_LONG:
 9998       evporq(dst, mask, nds, src, merge, vector_len); break;
 9999     default:
10000       fatal("Unexpected type argument %s", type2name(type)); break;
10001   }
10002 }
10003 
10004 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10005   switch(type) {
10006     case T_INT:
10007       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
10008     case T_LONG:
10009       evporq(dst, mask, nds, src, merge, vector_len); break;
10010     default:
10011       fatal("Unexpected type argument %s", type2name(type)); break;
10012   }
10013 }
10014 
10015 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10016   switch(type) {
10017     case T_INT:
10018       evpandd(dst, mask, nds, src, merge, vector_len); break;
10019     case T_LONG:
10020       evpandq(dst, mask, nds, src, merge, vector_len); break;
10021     default:
10022       fatal("Unexpected type argument %s", type2name(type)); break;
10023   }
10024 }
10025 
10026 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10027   switch(type) {
10028     case T_INT:
10029       evpandd(dst, mask, nds, src, merge, vector_len); break;
10030     case T_LONG:
10031       evpandq(dst, mask, nds, src, merge, vector_len); break;
10032     default:
10033       fatal("Unexpected type argument %s", type2name(type)); break;
10034   }
10035 }
10036 
10037 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
10038   switch(masklen) {
10039     case 8:
10040        kortestbl(src1, src2);
10041        break;
10042     case 16:
10043        kortestwl(src1, src2);
10044        break;
10045     case 32:
10046        kortestdl(src1, src2);
10047        break;
10048     case 64:
10049        kortestql(src1, src2);
10050        break;
10051     default:
10052       fatal("Unexpected mask length %d", masklen);
10053       break;
10054   }
10055 }
10056 
10057 
10058 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
10059   switch(masklen)  {
10060     case 8:
10061        ktestbl(src1, src2);
10062        break;
10063     case 16:
10064        ktestwl(src1, src2);
10065        break;
10066     case 32:
10067        ktestdl(src1, src2);
10068        break;
10069     case 64:
10070        ktestql(src1, src2);
10071        break;
10072     default:
10073       fatal("Unexpected mask length %d", masklen);
10074       break;
10075   }
10076 }
10077 
10078 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
10079   switch(type) {
10080     case T_INT:
10081       evprold(dst, mask, src, shift, merge, vlen_enc); break;
10082     case T_LONG:
10083       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
10084     default:
10085       fatal("Unexpected type argument %s", type2name(type)); break;
10086       break;
10087   }
10088 }
10089 
10090 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
10091   switch(type) {
10092     case T_INT:
10093       evprord(dst, mask, src, shift, merge, vlen_enc); break;
10094     case T_LONG:
10095       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
10096     default:
10097       fatal("Unexpected type argument %s", type2name(type)); break;
10098   }
10099 }
10100 
10101 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
10102   switch(type) {
10103     case T_INT:
10104       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
10105     case T_LONG:
10106       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
10107     default:
10108       fatal("Unexpected type argument %s", type2name(type)); break;
10109   }
10110 }
10111 
10112 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
10113   switch(type) {
10114     case T_INT:
10115       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
10116     case T_LONG:
10117       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
10118     default:
10119       fatal("Unexpected type argument %s", type2name(type)); break;
10120   }
10121 }
10122 
10123 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10124   assert(rscratch != noreg || always_reachable(src), "missing");
10125 
10126   if (reachable(src)) {
10127     evpandq(dst, nds, as_Address(src), vector_len);
10128   } else {
10129     lea(rscratch, src);
10130     evpandq(dst, nds, Address(rscratch, 0), vector_len);
10131   }
10132 }
10133 
10134 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
10135   assert(rscratch != noreg || always_reachable(src), "missing");
10136 
10137   if (reachable(src)) {
10138     Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len);
10139   } else {
10140     lea(rscratch, src);
10141     Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
10142   }
10143 }
10144 
10145 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10146   assert(rscratch != noreg || always_reachable(src), "missing");
10147 
10148   if (reachable(src)) {
10149     evporq(dst, nds, as_Address(src), vector_len);
10150   } else {
10151     lea(rscratch, src);
10152     evporq(dst, nds, Address(rscratch, 0), vector_len);
10153   }
10154 }
10155 
10156 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10157   assert(rscratch != noreg || always_reachable(src), "missing");
10158 
10159   if (reachable(src)) {
10160     vpshufb(dst, nds, as_Address(src), vector_len);
10161   } else {
10162     lea(rscratch, src);
10163     vpshufb(dst, nds, Address(rscratch, 0), vector_len);
10164   }
10165 }
10166 
10167 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) {
10168   assert(rscratch != noreg || always_reachable(src3), "missing");
10169 
10170   if (reachable(src3)) {
10171     vpternlogq(dst, imm8, src2, as_Address(src3), vector_len);
10172   } else {
10173     lea(rscratch, src3);
10174     vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len);
10175   }
10176 }
10177 
10178 #if COMPILER2_OR_JVMCI
10179 
10180 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
10181                                  Register length, Register temp, int vec_enc) {
10182   // Computing mask for predicated vector store.
10183   movptr(temp, -1);
10184   bzhiq(temp, temp, length);
10185   kmov(mask, temp);
10186   evmovdqu(bt, mask, dst, xmm, true, vec_enc);
10187 }
10188 
10189 // Set memory operation for length "less than" 64 bytes.
10190 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp,
10191                                        XMMRegister xmm, KRegister mask, Register length,
10192                                        Register temp, bool use64byteVector) {
10193   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10194   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
10195   if (!use64byteVector) {
10196     fill32(dst, disp, xmm);
10197     subptr(length, 32 >> shift);
10198     fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp);
10199   } else {
10200     assert(MaxVectorSize == 64, "vector length != 64");
10201     fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit);
10202   }
10203 }
10204 
10205 
10206 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp,
10207                                        XMMRegister xmm, KRegister mask, Register length,
10208                                        Register temp) {
10209   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10210   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
10211   fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit);
10212 }
10213 
10214 
10215 void MacroAssembler::fill32(Address dst, XMMRegister xmm) {
10216   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10217   vmovdqu(dst, xmm);
10218 }
10219 
10220 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) {
10221   fill32(Address(dst, disp), xmm);
10222 }
10223 
10224 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) {
10225   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10226   if (!use64byteVector) {
10227     fill32(dst, xmm);
10228     fill32(dst.plus_disp(32), xmm);
10229   } else {
10230     evmovdquq(dst, xmm, Assembler::AVX_512bit);
10231   }
10232 }
10233 
10234 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
10235   fill64(Address(dst, disp), xmm, use64byteVector);
10236 }
10237 
10238 #ifdef _LP64
10239 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value,
10240                                         Register count, Register rtmp, XMMRegister xtmp) {
10241   Label L_exit;
10242   Label L_fill_start;
10243   Label L_fill_64_bytes;
10244   Label L_fill_96_bytes;
10245   Label L_fill_128_bytes;
10246   Label L_fill_128_bytes_loop;
10247   Label L_fill_128_loop_header;
10248   Label L_fill_128_bytes_loop_header;
10249   Label L_fill_128_bytes_loop_pre_header;
10250   Label L_fill_zmm_sequence;
10251 
10252   int shift = -1;
10253   int avx3threshold = VM_Version::avx3_threshold();
10254   switch(type) {
10255     case T_BYTE:  shift = 0;
10256       break;
10257     case T_SHORT: shift = 1;
10258       break;
10259     case T_INT:   shift = 2;
10260       break;
10261     /* Uncomment when LONG fill stubs are supported.
10262     case T_LONG:  shift = 3;
10263       break;
10264     */
10265     default:
10266       fatal("Unhandled type: %s\n", type2name(type));
10267   }
10268 
10269   if ((avx3threshold != 0)  || (MaxVectorSize == 32)) {
10270 
10271     if (MaxVectorSize == 64) {
10272       cmpq(count, avx3threshold >> shift);
10273       jcc(Assembler::greater, L_fill_zmm_sequence);
10274     }
10275 
10276     evpbroadcast(type, xtmp, value, Assembler::AVX_256bit);
10277 
10278     bind(L_fill_start);
10279 
10280     cmpq(count, 32 >> shift);
10281     jccb(Assembler::greater, L_fill_64_bytes);
10282     fill32_masked(shift, to, 0, xtmp, k2, count, rtmp);
10283     jmp(L_exit);
10284 
10285     bind(L_fill_64_bytes);
10286     cmpq(count, 64 >> shift);
10287     jccb(Assembler::greater, L_fill_96_bytes);
10288     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp);
10289     jmp(L_exit);
10290 
10291     bind(L_fill_96_bytes);
10292     cmpq(count, 96 >> shift);
10293     jccb(Assembler::greater, L_fill_128_bytes);
10294     fill64(to, 0, xtmp);
10295     subq(count, 64 >> shift);
10296     fill32_masked(shift, to, 64, xtmp, k2, count, rtmp);
10297     jmp(L_exit);
10298 
10299     bind(L_fill_128_bytes);
10300     cmpq(count, 128 >> shift);
10301     jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header);
10302     fill64(to, 0, xtmp);
10303     fill32(to, 64, xtmp);
10304     subq(count, 96 >> shift);
10305     fill32_masked(shift, to, 96, xtmp, k2, count, rtmp);
10306     jmp(L_exit);
10307 
10308     bind(L_fill_128_bytes_loop_pre_header);
10309     {
10310       mov(rtmp, to);
10311       andq(rtmp, 31);
10312       jccb(Assembler::zero, L_fill_128_bytes_loop_header);
10313       negq(rtmp);
10314       addq(rtmp, 32);
10315       mov64(r8, -1L);
10316       bzhiq(r8, r8, rtmp);
10317       kmovql(k2, r8);
10318       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit);
10319       addq(to, rtmp);
10320       shrq(rtmp, shift);
10321       subq(count, rtmp);
10322     }
10323 
10324     cmpq(count, 128 >> shift);
10325     jcc(Assembler::less, L_fill_start);
10326 
10327     bind(L_fill_128_bytes_loop_header);
10328     subq(count, 128 >> shift);
10329 
10330     align32();
10331     bind(L_fill_128_bytes_loop);
10332       fill64(to, 0, xtmp);
10333       fill64(to, 64, xtmp);
10334       addq(to, 128);
10335       subq(count, 128 >> shift);
10336       jccb(Assembler::greaterEqual, L_fill_128_bytes_loop);
10337 
10338     addq(count, 128 >> shift);
10339     jcc(Assembler::zero, L_exit);
10340     jmp(L_fill_start);
10341   }
10342 
10343   if (MaxVectorSize == 64) {
10344     // Sequence using 64 byte ZMM register.
10345     Label L_fill_128_bytes_zmm;
10346     Label L_fill_192_bytes_zmm;
10347     Label L_fill_192_bytes_loop_zmm;
10348     Label L_fill_192_bytes_loop_header_zmm;
10349     Label L_fill_192_bytes_loop_pre_header_zmm;
10350     Label L_fill_start_zmm_sequence;
10351 
10352     bind(L_fill_zmm_sequence);
10353     evpbroadcast(type, xtmp, value, Assembler::AVX_512bit);
10354 
10355     bind(L_fill_start_zmm_sequence);
10356     cmpq(count, 64 >> shift);
10357     jccb(Assembler::greater, L_fill_128_bytes_zmm);
10358     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true);
10359     jmp(L_exit);
10360 
10361     bind(L_fill_128_bytes_zmm);
10362     cmpq(count, 128 >> shift);
10363     jccb(Assembler::greater, L_fill_192_bytes_zmm);
10364     fill64(to, 0, xtmp, true);
10365     subq(count, 64 >> shift);
10366     fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true);
10367     jmp(L_exit);
10368 
10369     bind(L_fill_192_bytes_zmm);
10370     cmpq(count, 192 >> shift);
10371     jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm);
10372     fill64(to, 0, xtmp, true);
10373     fill64(to, 64, xtmp, true);
10374     subq(count, 128 >> shift);
10375     fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true);
10376     jmp(L_exit);
10377 
10378     bind(L_fill_192_bytes_loop_pre_header_zmm);
10379     {
10380       movq(rtmp, to);
10381       andq(rtmp, 63);
10382       jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm);
10383       negq(rtmp);
10384       addq(rtmp, 64);
10385       mov64(r8, -1L);
10386       bzhiq(r8, r8, rtmp);
10387       kmovql(k2, r8);
10388       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit);
10389       addq(to, rtmp);
10390       shrq(rtmp, shift);
10391       subq(count, rtmp);
10392     }
10393 
10394     cmpq(count, 192 >> shift);
10395     jcc(Assembler::less, L_fill_start_zmm_sequence);
10396 
10397     bind(L_fill_192_bytes_loop_header_zmm);
10398     subq(count, 192 >> shift);
10399 
10400     align32();
10401     bind(L_fill_192_bytes_loop_zmm);
10402       fill64(to, 0, xtmp, true);
10403       fill64(to, 64, xtmp, true);
10404       fill64(to, 128, xtmp, true);
10405       addq(to, 192);
10406       subq(count, 192 >> shift);
10407       jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm);
10408 
10409     addq(count, 192 >> shift);
10410     jcc(Assembler::zero, L_exit);
10411     jmp(L_fill_start_zmm_sequence);
10412   }
10413   bind(L_exit);
10414 }
10415 #endif
10416 #endif //COMPILER2_OR_JVMCI
10417 
10418 
10419 #ifdef _LP64
10420 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
10421   Label done;
10422   cvttss2sil(dst, src);
10423   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10424   cmpl(dst, 0x80000000); // float_sign_flip
10425   jccb(Assembler::notEqual, done);
10426   subptr(rsp, 8);
10427   movflt(Address(rsp, 0), src);
10428   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10429   pop(dst);
10430   bind(done);
10431 }
10432 
10433 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
10434   Label done;
10435   cvttsd2sil(dst, src);
10436   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10437   cmpl(dst, 0x80000000); // float_sign_flip
10438   jccb(Assembler::notEqual, done);
10439   subptr(rsp, 8);
10440   movdbl(Address(rsp, 0), src);
10441   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10442   pop(dst);
10443   bind(done);
10444 }
10445 
10446 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
10447   Label done;
10448   cvttss2siq(dst, src);
10449   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10450   jccb(Assembler::notEqual, done);
10451   subptr(rsp, 8);
10452   movflt(Address(rsp, 0), src);
10453   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10454   pop(dst);
10455   bind(done);
10456 }
10457 
10458 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10459   // Following code is line by line assembly translation rounding algorithm.
10460   // Please refer to java.lang.Math.round(float) algorithm for details.
10461   const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000;
10462   const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24;
10463   const int32_t FloatConsts_EXP_BIAS = 127;
10464   const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF;
10465   const int32_t MINUS_32 = 0xFFFFFFE0;
10466   Label L_special_case, L_block1, L_exit;
10467   movl(rtmp, FloatConsts_EXP_BIT_MASK);
10468   movdl(dst, src);
10469   andl(dst, rtmp);
10470   sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1);
10471   movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS);
10472   subl(rtmp, dst);
10473   movl(rcx, rtmp);
10474   movl(dst, MINUS_32);
10475   testl(rtmp, dst);
10476   jccb(Assembler::notEqual, L_special_case);
10477   movdl(dst, src);
10478   andl(dst, FloatConsts_SIGNIF_BIT_MASK);
10479   orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1);
10480   movdl(rtmp, src);
10481   testl(rtmp, rtmp);
10482   jccb(Assembler::greaterEqual, L_block1);
10483   negl(dst);
10484   bind(L_block1);
10485   sarl(dst);
10486   addl(dst, 0x1);
10487   sarl(dst, 0x1);
10488   jmp(L_exit);
10489   bind(L_special_case);
10490   convert_f2i(dst, src);
10491   bind(L_exit);
10492 }
10493 
10494 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10495   // Following code is line by line assembly translation rounding algorithm.
10496   // Please refer to java.lang.Math.round(double) algorithm for details.
10497   const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L;
10498   const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53;
10499   const int64_t DoubleConsts_EXP_BIAS = 1023;
10500   const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL;
10501   const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L;
10502   Label L_special_case, L_block1, L_exit;
10503   mov64(rtmp, DoubleConsts_EXP_BIT_MASK);
10504   movq(dst, src);
10505   andq(dst, rtmp);
10506   sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1);
10507   mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS);
10508   subq(rtmp, dst);
10509   movq(rcx, rtmp);
10510   mov64(dst, MINUS_64);
10511   testq(rtmp, dst);
10512   jccb(Assembler::notEqual, L_special_case);
10513   movq(dst, src);
10514   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK);
10515   andq(dst, rtmp);
10516   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1);
10517   orq(dst, rtmp);
10518   movq(rtmp, src);
10519   testq(rtmp, rtmp);
10520   jccb(Assembler::greaterEqual, L_block1);
10521   negq(dst);
10522   bind(L_block1);
10523   sarq(dst);
10524   addq(dst, 0x1);
10525   sarq(dst, 0x1);
10526   jmp(L_exit);
10527   bind(L_special_case);
10528   convert_d2l(dst, src);
10529   bind(L_exit);
10530 }
10531 
10532 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
10533   Label done;
10534   cvttsd2siq(dst, src);
10535   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10536   jccb(Assembler::notEqual, done);
10537   subptr(rsp, 8);
10538   movdbl(Address(rsp, 0), src);
10539   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10540   pop(dst);
10541   bind(done);
10542 }
10543 
10544 void MacroAssembler::cache_wb(Address line)
10545 {
10546   // 64 bit cpus always support clflush
10547   assert(VM_Version::supports_clflush(), "clflush should be available");
10548   bool optimized = VM_Version::supports_clflushopt();
10549   bool no_evict = VM_Version::supports_clwb();
10550 
10551   // prefer clwb (writeback without evict) otherwise
10552   // prefer clflushopt (potentially parallel writeback with evict)
10553   // otherwise fallback on clflush (serial writeback with evict)
10554 
10555   if (optimized) {
10556     if (no_evict) {
10557       clwb(line);
10558     } else {
10559       clflushopt(line);
10560     }
10561   } else {
10562     // no need for fence when using CLFLUSH
10563     clflush(line);
10564   }
10565 }
10566 
10567 void MacroAssembler::cache_wbsync(bool is_pre)
10568 {
10569   assert(VM_Version::supports_clflush(), "clflush should be available");
10570   bool optimized = VM_Version::supports_clflushopt();
10571   bool no_evict = VM_Version::supports_clwb();
10572 
10573   // pick the correct implementation
10574 
10575   if (!is_pre && (optimized || no_evict)) {
10576     // need an sfence for post flush when using clflushopt or clwb
10577     // otherwise no no need for any synchroniaztion
10578 
10579     sfence();
10580   }
10581 }
10582 
10583 #endif // _LP64
10584 
10585 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
10586   switch (cond) {
10587     // Note some conditions are synonyms for others
10588     case Assembler::zero:         return Assembler::notZero;
10589     case Assembler::notZero:      return Assembler::zero;
10590     case Assembler::less:         return Assembler::greaterEqual;
10591     case Assembler::lessEqual:    return Assembler::greater;
10592     case Assembler::greater:      return Assembler::lessEqual;
10593     case Assembler::greaterEqual: return Assembler::less;
10594     case Assembler::below:        return Assembler::aboveEqual;
10595     case Assembler::belowEqual:   return Assembler::above;
10596     case Assembler::above:        return Assembler::belowEqual;
10597     case Assembler::aboveEqual:   return Assembler::below;
10598     case Assembler::overflow:     return Assembler::noOverflow;
10599     case Assembler::noOverflow:   return Assembler::overflow;
10600     case Assembler::negative:     return Assembler::positive;
10601     case Assembler::positive:     return Assembler::negative;
10602     case Assembler::parity:       return Assembler::noParity;
10603     case Assembler::noParity:     return Assembler::parity;
10604   }
10605   ShouldNotReachHere(); return Assembler::overflow;
10606 }
10607 
10608 SkipIfEqual::SkipIfEqual(
10609     MacroAssembler* masm, const bool* flag_addr, bool value, Register rscratch) {
10610   _masm = masm;
10611   _masm->cmp8(ExternalAddress((address)flag_addr), value, rscratch);
10612   _masm->jcc(Assembler::equal, _label);
10613 }
10614 
10615 SkipIfEqual::~SkipIfEqual() {
10616   _masm->bind(_label);
10617 }
10618 
10619 // 32-bit Windows has its own fast-path implementation
10620 // of get_thread
10621 #if !defined(WIN32) || defined(_LP64)
10622 
10623 // This is simply a call to Thread::current()
10624 void MacroAssembler::get_thread(Register thread) {
10625   if (thread != rax) {
10626     push(rax);
10627   }
10628   LP64_ONLY(push(rdi);)
10629   LP64_ONLY(push(rsi);)
10630   push(rdx);
10631   push(rcx);
10632 #ifdef _LP64
10633   push(r8);
10634   push(r9);
10635   push(r10);
10636   push(r11);
10637 #endif
10638 
10639   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
10640 
10641 #ifdef _LP64
10642   pop(r11);
10643   pop(r10);
10644   pop(r9);
10645   pop(r8);
10646 #endif
10647   pop(rcx);
10648   pop(rdx);
10649   LP64_ONLY(pop(rsi);)
10650   LP64_ONLY(pop(rdi);)
10651   if (thread != rax) {
10652     mov(thread, rax);
10653     pop(rax);
10654   }
10655 }
10656 
10657 
10658 #endif // !WIN32 || _LP64
10659 
10660 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) {
10661   Label L_stack_ok;
10662   if (bias == 0) {
10663     testptr(sp, 2 * wordSize - 1);
10664   } else {
10665     // lea(tmp, Address(rsp, bias);
10666     mov(tmp, sp);
10667     addptr(tmp, bias);
10668     testptr(tmp, 2 * wordSize - 1);
10669   }
10670   jcc(Assembler::equal, L_stack_ok);
10671   block_comment(msg);
10672   stop(msg);
10673   bind(L_stack_ok);
10674 }
10675 
10676 // Implements lightweight-locking.
10677 // Branches to slow upon failure to lock the object, with ZF cleared.
10678 // Falls through upon success with unspecified ZF.
10679 //
10680 // obj: the object to be locked
10681 // hdr: the (pre-loaded) header of the object, must be rax
10682 // thread: the thread which attempts to lock obj
10683 // tmp: a temporary register
10684 void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register thread, Register tmp, Label& slow) {
10685   assert(hdr == rax, "header must be in rax for cmpxchg");
10686   assert_different_registers(obj, hdr, thread, tmp);
10687 
10688   // First we need to check if the lock-stack has room for pushing the object reference.
10689   // Note: we subtract 1 from the end-offset so that we can do a 'greater' comparison, instead
10690   // of 'greaterEqual' below, which readily clears the ZF. This makes C2 code a little simpler and
10691   // avoids one branch.
10692   cmpl(Address(thread, JavaThread::lock_stack_top_offset()), LockStack::end_offset() - 1);
10693   jcc(Assembler::greater, slow);
10694 
10695   // Now we attempt to take the fast-lock.
10696   // Clear lock_mask bits (locked state).
10697   andptr(hdr, ~(int32_t)markWord::lock_mask_in_place);
10698   movptr(tmp, hdr);
10699   // Set unlocked_value bit.
10700   orptr(hdr, markWord::unlocked_value);
10701   if (EnableValhalla) {
10702     // Mask inline_type bit such that we go to the slow path if object is an inline type
10703     andptr(hdr, ~((int) markWord::inline_type_bit_in_place));
10704   }
10705   lock();
10706   cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10707   jcc(Assembler::notEqual, slow);
10708 
10709   // If successful, push object to lock-stack.
10710   movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
10711   movptr(Address(thread, tmp), obj);
10712   incrementl(tmp, oopSize);
10713   movl(Address(thread, JavaThread::lock_stack_top_offset()), tmp);
10714 }
10715 
10716 // Implements lightweight-unlocking.
10717 // Branches to slow upon failure, with ZF cleared.
10718 // Falls through upon success, with unspecified ZF.
10719 //
10720 // obj: the object to be unlocked
10721 // hdr: the (pre-loaded) header of the object, must be rax
10722 // tmp: a temporary register
10723 void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register tmp, Label& slow) {
10724   assert(hdr == rax, "header must be in rax for cmpxchg");
10725   assert_different_registers(obj, hdr, tmp);
10726 
10727   // Mark-word must be lock_mask now, try to swing it back to unlocked_value.
10728   movptr(tmp, hdr); // The expected old value
10729   orptr(tmp, markWord::unlocked_value);
10730   lock();
10731   cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10732   jcc(Assembler::notEqual, slow);
10733   // Pop the lock object from the lock-stack.
10734 #ifdef _LP64
10735   const Register thread = r15_thread;
10736 #else
10737   const Register thread = rax;
10738   get_thread(thread);
10739 #endif
10740   subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10741 #ifdef ASSERT
10742   movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
10743   movptr(Address(thread, tmp), 0);
10744 #endif
10745 }