1 /*
    2  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
    3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
    4  *
    5  * This code is free software; you can redistribute it and/or modify it
    6  * under the terms of the GNU General Public License version 2 only, as
    7  * published by the Free Software Foundation.
    8  *
    9  * This code is distributed in the hope that it will be useful, but WITHOUT
   10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "asm/assembler.hpp"
   26 #include "asm/assembler.inline.hpp"
   27 #include "code/SCCache.hpp"
   28 #include "code/compiledIC.hpp"
   29 #include "compiler/compiler_globals.hpp"
   30 #include "compiler/disassembler.hpp"
   31 #include "crc32c.h"
   32 #include "gc/shared/barrierSet.hpp"
   33 #include "gc/shared/barrierSetAssembler.hpp"
   34 #include "gc/shared/collectedHeap.inline.hpp"
   35 #include "gc/shared/tlab_globals.hpp"
   36 #include "interpreter/bytecodeHistogram.hpp"
   37 #include "interpreter/interpreter.hpp"
   38 #include "interpreter/interpreterRuntime.hpp"
   39 #include "jvm.h"
   40 #include "memory/resourceArea.hpp"
   41 #include "memory/universe.hpp"
   42 #include "oops/accessDecorators.hpp"
   43 #include "oops/compressedKlass.inline.hpp"
   44 #include "oops/compressedOops.inline.hpp"
   45 #include "oops/klass.inline.hpp"
   46 #include "prims/methodHandles.hpp"
   47 #include "runtime/continuation.hpp"
   48 #include "runtime/interfaceSupport.inline.hpp"
   49 #include "runtime/javaThread.hpp"
   50 #include "runtime/jniHandles.hpp"
   51 #include "runtime/objectMonitor.hpp"
   52 #include "runtime/os.hpp"
   53 #include "runtime/safepoint.hpp"
   54 #include "runtime/safepointMechanism.hpp"
   55 #include "runtime/sharedRuntime.hpp"
   56 #include "runtime/stubRoutines.hpp"
   57 #include "utilities/checkedCast.hpp"
   58 #include "utilities/macros.hpp"
   59 
   60 #ifdef PRODUCT
   61 #define BLOCK_COMMENT(str) /* nothing */
   62 #define STOP(error) stop(error)
   63 #else
   64 #define BLOCK_COMMENT(str) block_comment(str)
   65 #define STOP(error) block_comment(error); stop(error)
   66 #endif
   67 
   68 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   69 
   70 #ifdef ASSERT
   71 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   72 #endif
   73 
   74 static const Assembler::Condition reverse[] = {
   75     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   76     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   77     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   78     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
   79     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
   80     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
   81     Assembler::above          /* belowEqual    = 0x6 */ ,
   82     Assembler::belowEqual     /* above         = 0x7 */ ,
   83     Assembler::positive       /* negative      = 0x8 */ ,
   84     Assembler::negative       /* positive      = 0x9 */ ,
   85     Assembler::noParity       /* parity        = 0xa */ ,
   86     Assembler::parity         /* noParity      = 0xb */ ,
   87     Assembler::greaterEqual   /* less          = 0xc */ ,
   88     Assembler::less           /* greaterEqual  = 0xd */ ,
   89     Assembler::greater        /* lessEqual     = 0xe */ ,
   90     Assembler::lessEqual      /* greater       = 0xf, */
   91 
   92 };
   93 
   94 
   95 // Implementation of MacroAssembler
   96 
   97 // First all the versions that have distinct versions depending on 32/64 bit
   98 // Unless the difference is trivial (1 line or so).
   99 
  100 #ifndef _LP64
  101 
  102 // 32bit versions
  103 
  104 Address MacroAssembler::as_Address(AddressLiteral adr) {
  105   return Address(adr.target(), adr.rspec());
  106 }
  107 
  108 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  109   assert(rscratch == noreg, "");
  110   return Address::make_array(adr);
  111 }
  112 
  113 void MacroAssembler::call_VM_leaf_base(address entry_point,
  114                                        int number_of_arguments) {
  115   call(RuntimeAddress(entry_point));
  116   increment(rsp, number_of_arguments * wordSize);
  117 }
  118 
  119 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
  120   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  121 }
  122 
  123 
  124 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
  125   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  126 }
  127 
  128 void MacroAssembler::cmpoop(Address src1, jobject obj) {
  129   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  130 }
  131 
  132 void MacroAssembler::cmpoop(Register src1, jobject obj, Register rscratch) {
  133   assert(rscratch == noreg, "redundant");
  134   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
  135 }
  136 
  137 void MacroAssembler::extend_sign(Register hi, Register lo) {
  138   // According to Intel Doc. AP-526, "Integer Divide", p.18.
  139   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
  140     cdql();
  141   } else {
  142     movl(hi, lo);
  143     sarl(hi, 31);
  144   }
  145 }
  146 
  147 void MacroAssembler::jC2(Register tmp, Label& L) {
  148   // set parity bit if FPU flag C2 is set (via rax)
  149   save_rax(tmp);
  150   fwait(); fnstsw_ax();
  151   sahf();
  152   restore_rax(tmp);
  153   // branch
  154   jcc(Assembler::parity, L);
  155 }
  156 
  157 void MacroAssembler::jnC2(Register tmp, Label& L) {
  158   // set parity bit if FPU flag C2 is set (via rax)
  159   save_rax(tmp);
  160   fwait(); fnstsw_ax();
  161   sahf();
  162   restore_rax(tmp);
  163   // branch
  164   jcc(Assembler::noParity, L);
  165 }
  166 
  167 // 32bit can do a case table jump in one instruction but we no longer allow the base
  168 // to be installed in the Address class
  169 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  170   assert(rscratch == noreg, "not needed");
  171   jmp(as_Address(entry, noreg));
  172 }
  173 
  174 // Note: y_lo will be destroyed
  175 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  176   // Long compare for Java (semantics as described in JVM spec.)
  177   Label high, low, done;
  178 
  179   cmpl(x_hi, y_hi);
  180   jcc(Assembler::less, low);
  181   jcc(Assembler::greater, high);
  182   // x_hi is the return register
  183   xorl(x_hi, x_hi);
  184   cmpl(x_lo, y_lo);
  185   jcc(Assembler::below, low);
  186   jcc(Assembler::equal, done);
  187 
  188   bind(high);
  189   xorl(x_hi, x_hi);
  190   increment(x_hi);
  191   jmp(done);
  192 
  193   bind(low);
  194   xorl(x_hi, x_hi);
  195   decrementl(x_hi);
  196 
  197   bind(done);
  198 }
  199 
  200 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  201   mov_literal32(dst, (int32_t)src.target(), src.rspec());
  202 }
  203 
  204 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  205   assert(rscratch == noreg, "not needed");
  206 
  207   // leal(dst, as_Address(adr));
  208   // see note in movl as to why we must use a move
  209   mov_literal32(dst, (int32_t)adr.target(), adr.rspec());
  210 }
  211 
  212 void MacroAssembler::leave() {
  213   mov(rsp, rbp);
  214   pop(rbp);
  215 }
  216 
  217 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
  218   // Multiplication of two Java long values stored on the stack
  219   // as illustrated below. Result is in rdx:rax.
  220   //
  221   // rsp ---> [  ??  ] \               \
  222   //            ....    | y_rsp_offset  |
  223   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
  224   //          [ y_hi ]                  | (in bytes)
  225   //            ....                    |
  226   //          [ x_lo ]                 /
  227   //          [ x_hi ]
  228   //            ....
  229   //
  230   // Basic idea: lo(result) = lo(x_lo * y_lo)
  231   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
  232   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
  233   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
  234   Label quick;
  235   // load x_hi, y_hi and check if quick
  236   // multiplication is possible
  237   movl(rbx, x_hi);
  238   movl(rcx, y_hi);
  239   movl(rax, rbx);
  240   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
  241   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
  242   // do full multiplication
  243   // 1st step
  244   mull(y_lo);                                    // x_hi * y_lo
  245   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
  246   // 2nd step
  247   movl(rax, x_lo);
  248   mull(rcx);                                     // x_lo * y_hi
  249   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
  250   // 3rd step
  251   bind(quick);                                   // note: rbx, = 0 if quick multiply!
  252   movl(rax, x_lo);
  253   mull(y_lo);                                    // x_lo * y_lo
  254   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
  255 }
  256 
  257 void MacroAssembler::lneg(Register hi, Register lo) {
  258   negl(lo);
  259   adcl(hi, 0);
  260   negl(hi);
  261 }
  262 
  263 void MacroAssembler::lshl(Register hi, Register lo) {
  264   // Java shift left long support (semantics as described in JVM spec., p.305)
  265   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
  266   // shift value is in rcx !
  267   assert(hi != rcx, "must not use rcx");
  268   assert(lo != rcx, "must not use rcx");
  269   const Register s = rcx;                        // shift count
  270   const int      n = BitsPerWord;
  271   Label L;
  272   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  273   cmpl(s, n);                                    // if (s < n)
  274   jcc(Assembler::less, L);                       // else (s >= n)
  275   movl(hi, lo);                                  // x := x << n
  276   xorl(lo, lo);
  277   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  278   bind(L);                                       // s (mod n) < n
  279   shldl(hi, lo);                                 // x := x << s
  280   shll(lo);
  281 }
  282 
  283 
  284 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
  285   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
  286   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
  287   assert(hi != rcx, "must not use rcx");
  288   assert(lo != rcx, "must not use rcx");
  289   const Register s = rcx;                        // shift count
  290   const int      n = BitsPerWord;
  291   Label L;
  292   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
  293   cmpl(s, n);                                    // if (s < n)
  294   jcc(Assembler::less, L);                       // else (s >= n)
  295   movl(lo, hi);                                  // x := x >> n
  296   if (sign_extension) sarl(hi, 31);
  297   else                xorl(hi, hi);
  298   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
  299   bind(L);                                       // s (mod n) < n
  300   shrdl(lo, hi);                                 // x := x >> s
  301   if (sign_extension) sarl(hi);
  302   else                shrl(hi);
  303 }
  304 
  305 void MacroAssembler::movoop(Register dst, jobject obj) {
  306   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  307 }
  308 
  309 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  310   assert(rscratch == noreg, "redundant");
  311   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
  312 }
  313 
  314 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  315   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  316 }
  317 
  318 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  319   assert(rscratch == noreg, "redundant");
  320   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
  321 }
  322 
  323 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  324   if (src.is_lval()) {
  325     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
  326   } else {
  327     movl(dst, as_Address(src));
  328   }
  329 }
  330 
  331 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  332   assert(rscratch == noreg, "redundant");
  333   movl(as_Address(dst, noreg), src);
  334 }
  335 
  336 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  337   movl(dst, as_Address(src, noreg));
  338 }
  339 
  340 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  341   assert(rscratch == noreg, "redundant");
  342   movl(dst, src);
  343 }
  344 
  345 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  346   assert(rscratch == noreg, "redundant");
  347   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
  348 }
  349 
  350 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  351   assert(rscratch == noreg, "redundant");
  352   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
  353 }
  354 
  355 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  356   assert(rscratch == noreg, "redundant");
  357   if (src.is_lval()) {
  358     push_literal32((int32_t)src.target(), src.rspec());
  359   } else {
  360     pushl(as_Address(src));
  361   }
  362 }
  363 
  364 static void pass_arg0(MacroAssembler* masm, Register arg) {
  365   masm->push(arg);
  366 }
  367 
  368 static void pass_arg1(MacroAssembler* masm, Register arg) {
  369   masm->push(arg);
  370 }
  371 
  372 static void pass_arg2(MacroAssembler* masm, Register arg) {
  373   masm->push(arg);
  374 }
  375 
  376 static void pass_arg3(MacroAssembler* masm, Register arg) {
  377   masm->push(arg);
  378 }
  379 
  380 #ifndef PRODUCT
  381 extern "C" void findpc(intptr_t x);
  382 #endif
  383 
  384 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
  385   // In order to get locks to work, we need to fake a in_VM state
  386   JavaThread* thread = JavaThread::current();
  387   JavaThreadState saved_state = thread->thread_state();
  388   thread->set_thread_state(_thread_in_vm);
  389   if (ShowMessageBoxOnError) {
  390     JavaThread* thread = JavaThread::current();
  391     JavaThreadState saved_state = thread->thread_state();
  392     thread->set_thread_state(_thread_in_vm);
  393     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  394       ttyLocker ttyl;
  395       BytecodeCounter::print();
  396     }
  397     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  398     // This is the value of eip which points to where verify_oop will return.
  399     if (os::message_box(msg, "Execution stopped, print registers?")) {
  400       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
  401       BREAKPOINT;
  402     }
  403   }
  404   fatal("DEBUG MESSAGE: %s", msg);
  405 }
  406 
  407 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
  408   ttyLocker ttyl;
  409   DebuggingContext debugging{};
  410   tty->print_cr("eip = 0x%08x", eip);
  411 #ifndef PRODUCT
  412   if ((WizardMode || Verbose) && PrintMiscellaneous) {
  413     tty->cr();
  414     findpc(eip);
  415     tty->cr();
  416   }
  417 #endif
  418 #define PRINT_REG(rax) \
  419   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
  420   PRINT_REG(rax);
  421   PRINT_REG(rbx);
  422   PRINT_REG(rcx);
  423   PRINT_REG(rdx);
  424   PRINT_REG(rdi);
  425   PRINT_REG(rsi);
  426   PRINT_REG(rbp);
  427   PRINT_REG(rsp);
  428 #undef PRINT_REG
  429   // Print some words near top of staack.
  430   int* dump_sp = (int*) rsp;
  431   for (int col1 = 0; col1 < 8; col1++) {
  432     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  433     os::print_location(tty, *dump_sp++);
  434   }
  435   for (int row = 0; row < 16; row++) {
  436     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  437     for (int col = 0; col < 8; col++) {
  438       tty->print(" 0x%08x", *dump_sp++);
  439     }
  440     tty->cr();
  441   }
  442   // Print some instructions around pc:
  443   Disassembler::decode((address)eip-64, (address)eip);
  444   tty->print_cr("--------");
  445   Disassembler::decode((address)eip, (address)eip+32);
  446 }
  447 
  448 void MacroAssembler::stop(const char* msg) {
  449   // push address of message
  450   ExternalAddress message((address)msg);
  451   pushptr(message.addr(), noreg);
  452   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  453   pusha();                                            // push registers
  454   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
  455   hlt();
  456 }
  457 
  458 void MacroAssembler::warn(const char* msg) {
  459   push_CPU_state();
  460 
  461   // push address of message
  462   ExternalAddress message((address)msg);
  463   pushptr(message.addr(), noreg);
  464 
  465   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  466   addl(rsp, wordSize);       // discard argument
  467   pop_CPU_state();
  468 }
  469 
  470 void MacroAssembler::print_state() {
  471   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
  472   pusha();                                            // push registers
  473 
  474   push_CPU_state();
  475   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
  476   pop_CPU_state();
  477 
  478   popa();
  479   addl(rsp, wordSize);
  480 }
  481 
  482 #else // _LP64
  483 
  484 // 64 bit versions
  485 
  486 Address MacroAssembler::as_Address(AddressLiteral adr) {
  487   // amd64 always does this as a pc-rel
  488   // we can be absolute or disp based on the instruction type
  489   // jmp/call are displacements others are absolute
  490   assert(!adr.is_lval(), "must be rval");
  491   assert(reachable(adr), "must be");
  492   return Address(checked_cast<int32_t>(adr.target() - pc()), adr.target(), adr.reloc());
  493 
  494 }
  495 
  496 Address MacroAssembler::as_Address(ArrayAddress adr, Register rscratch) {
  497   AddressLiteral base = adr.base();
  498   lea(rscratch, base);
  499   Address index = adr.index();
  500   assert(index._disp == 0, "must not have disp"); // maybe it can?
  501   Address array(rscratch, index._index, index._scale, index._disp);
  502   return array;
  503 }
  504 
  505 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
  506   Label L, E;
  507 
  508 #ifdef _WIN64
  509   // Windows always allocates space for it's register args
  510   assert(num_args <= 4, "only register arguments supported");
  511   subq(rsp,  frame::arg_reg_save_area_bytes);
  512 #endif
  513 
  514   // Align stack if necessary
  515   testl(rsp, 15);
  516   jcc(Assembler::zero, L);
  517 
  518   subq(rsp, 8);
  519   call(RuntimeAddress(entry_point));
  520   addq(rsp, 8);
  521   jmp(E);
  522 
  523   bind(L);
  524   call(RuntimeAddress(entry_point));
  525 
  526   bind(E);
  527 
  528 #ifdef _WIN64
  529   // restore stack pointer
  530   addq(rsp, frame::arg_reg_save_area_bytes);
  531 #endif
  532 }
  533 
  534 void MacroAssembler::cmp64(Register src1, AddressLiteral src2, Register rscratch) {
  535   assert(!src2.is_lval(), "should use cmpptr");
  536   assert(rscratch != noreg || always_reachable(src2), "missing");
  537 
  538   if (reachable(src2)) {
  539     cmpq(src1, as_Address(src2));
  540   } else {
  541     lea(rscratch, src2);
  542     Assembler::cmpq(src1, Address(rscratch, 0));
  543   }
  544 }
  545 
  546 int MacroAssembler::corrected_idivq(Register reg) {
  547   // Full implementation of Java ldiv and lrem; checks for special
  548   // case as described in JVM spec., p.243 & p.271.  The function
  549   // returns the (pc) offset of the idivl instruction - may be needed
  550   // for implicit exceptions.
  551   //
  552   //         normal case                           special case
  553   //
  554   // input : rax: dividend                         min_long
  555   //         reg: divisor   (may not be eax/edx)   -1
  556   //
  557   // output: rax: quotient  (= rax idiv reg)       min_long
  558   //         rdx: remainder (= rax irem reg)       0
  559   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
  560   static const int64_t min_long = 0x8000000000000000;
  561   Label normal_case, special_case;
  562 
  563   // check for special case
  564   cmp64(rax, ExternalAddress((address) &min_long), rdx /*rscratch*/);
  565   jcc(Assembler::notEqual, normal_case);
  566   xorl(rdx, rdx); // prepare rdx for possible special case (where
  567                   // remainder = 0)
  568   cmpq(reg, -1);
  569   jcc(Assembler::equal, special_case);
  570 
  571   // handle normal case
  572   bind(normal_case);
  573   cdqq();
  574   int idivq_offset = offset();
  575   idivq(reg);
  576 
  577   // normal and special case exit
  578   bind(special_case);
  579 
  580   return idivq_offset;
  581 }
  582 
  583 void MacroAssembler::decrementq(Register reg, int value) {
  584   if (value == min_jint) { subq(reg, value); return; }
  585   if (value <  0) { incrementq(reg, -value); return; }
  586   if (value == 0) {                        ; return; }
  587   if (value == 1 && UseIncDec) { decq(reg) ; return; }
  588   /* else */      { subq(reg, value)       ; return; }
  589 }
  590 
  591 void MacroAssembler::decrementq(Address dst, int value) {
  592   if (value == min_jint) { subq(dst, value); return; }
  593   if (value <  0) { incrementq(dst, -value); return; }
  594   if (value == 0) {                        ; return; }
  595   if (value == 1 && UseIncDec) { decq(dst) ; return; }
  596   /* else */      { subq(dst, value)       ; return; }
  597 }
  598 
  599 void MacroAssembler::incrementq(AddressLiteral dst, Register rscratch) {
  600   assert(rscratch != noreg || always_reachable(dst), "missing");
  601 
  602   if (reachable(dst)) {
  603     incrementq(as_Address(dst));
  604   } else {
  605     lea(rscratch, dst);
  606     incrementq(Address(rscratch, 0));
  607   }
  608 }
  609 
  610 void MacroAssembler::incrementq(Register reg, int value) {
  611   if (value == min_jint) { addq(reg, value); return; }
  612   if (value <  0) { decrementq(reg, -value); return; }
  613   if (value == 0) {                        ; return; }
  614   if (value == 1 && UseIncDec) { incq(reg) ; return; }
  615   /* else */      { addq(reg, value)       ; return; }
  616 }
  617 
  618 void MacroAssembler::incrementq(Address dst, int value) {
  619   if (value == min_jint) { addq(dst, value); return; }
  620   if (value <  0) { decrementq(dst, -value); return; }
  621   if (value == 0) {                        ; return; }
  622   if (value == 1 && UseIncDec) { incq(dst) ; return; }
  623   /* else */      { addq(dst, value)       ; return; }
  624 }
  625 
  626 // 32bit can do a case table jump in one instruction but we no longer allow the base
  627 // to be installed in the Address class
  628 void MacroAssembler::jump(ArrayAddress entry, Register rscratch) {
  629   lea(rscratch, entry.base());
  630   Address dispatch = entry.index();
  631   assert(dispatch._base == noreg, "must be");
  632   dispatch._base = rscratch;
  633   jmp(dispatch);
  634 }
  635 
  636 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
  637   ShouldNotReachHere(); // 64bit doesn't use two regs
  638   cmpq(x_lo, y_lo);
  639 }
  640 
  641 void MacroAssembler::lea(Register dst, AddressLiteral src) {
  642   mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  643 }
  644 
  645 void MacroAssembler::lea(Address dst, AddressLiteral adr, Register rscratch) {
  646   lea(rscratch, adr);
  647   movptr(dst, rscratch);
  648 }
  649 
  650 void MacroAssembler::leave() {
  651   // %%% is this really better? Why not on 32bit too?
  652   emit_int8((unsigned char)0xC9); // LEAVE
  653 }
  654 
  655 void MacroAssembler::lneg(Register hi, Register lo) {
  656   ShouldNotReachHere(); // 64bit doesn't use two regs
  657   negq(lo);
  658 }
  659 
  660 void MacroAssembler::movoop(Register dst, jobject obj) {
  661   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  662 }
  663 
  664 void MacroAssembler::movoop(Address dst, jobject obj, Register rscratch) {
  665   mov_literal64(rscratch, (intptr_t)obj, oop_Relocation::spec_for_immediate());
  666   movq(dst, rscratch);
  667 }
  668 
  669 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
  670   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  671 }
  672 
  673 void MacroAssembler::mov_metadata(Address dst, Metadata* obj, Register rscratch) {
  674   mov_literal64(rscratch, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
  675   movq(dst, rscratch);
  676 }
  677 
  678 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
  679   if (src.is_lval()) {
  680     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
  681   } else {
  682     if (reachable(src)) {
  683       movq(dst, as_Address(src));
  684     } else {
  685       lea(dst, src);
  686       movq(dst, Address(dst, 0));
  687     }
  688   }
  689 }
  690 
  691 void MacroAssembler::movptr(ArrayAddress dst, Register src, Register rscratch) {
  692   movq(as_Address(dst, rscratch), src);
  693 }
  694 
  695 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
  696   movq(dst, as_Address(src, dst /*rscratch*/));
  697 }
  698 
  699 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
  700 void MacroAssembler::movptr(Address dst, intptr_t src, Register rscratch) {
  701   if (is_simm32(src)) {
  702     movptr(dst, checked_cast<int32_t>(src));
  703   } else {
  704     mov64(rscratch, src);
  705     movq(dst, rscratch);
  706   }
  707 }
  708 
  709 void MacroAssembler::pushoop(jobject obj, Register rscratch) {
  710   movoop(rscratch, obj);
  711   push(rscratch);
  712 }
  713 
  714 void MacroAssembler::pushklass(Metadata* obj, Register rscratch) {
  715   mov_metadata(rscratch, obj);
  716   push(rscratch);
  717 }
  718 
  719 void MacroAssembler::pushptr(AddressLiteral src, Register rscratch) {
  720   lea(rscratch, src);
  721   if (src.is_lval()) {
  722     push(rscratch);
  723   } else {
  724     pushq(Address(rscratch, 0));
  725   }
  726 }
  727 
  728 static void pass_arg0(MacroAssembler* masm, Register arg) {
  729   if (c_rarg0 != arg ) {
  730     masm->mov(c_rarg0, arg);
  731   }
  732 }
  733 
  734 static void pass_arg1(MacroAssembler* masm, Register arg) {
  735   if (c_rarg1 != arg ) {
  736     masm->mov(c_rarg1, arg);
  737   }
  738 }
  739 
  740 static void pass_arg2(MacroAssembler* masm, Register arg) {
  741   if (c_rarg2 != arg ) {
  742     masm->mov(c_rarg2, arg);
  743   }
  744 }
  745 
  746 static void pass_arg3(MacroAssembler* masm, Register arg) {
  747   if (c_rarg3 != arg ) {
  748     masm->mov(c_rarg3, arg);
  749   }
  750 }
  751 
  752 void MacroAssembler::stop(const char* msg) {
  753   if (ShowMessageBoxOnError) {
  754     address rip = pc();
  755     pusha(); // get regs on stack
  756     lea(c_rarg1, InternalAddress(rip));
  757     movq(c_rarg2, rsp); // pass pointer to regs array
  758   }
  759   lea(c_rarg0, ExternalAddress((address) msg));
  760   andq(rsp, -16); // align stack as required by ABI
  761   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
  762   hlt();
  763   SCCache::add_C_string(msg);
  764 }
  765 
  766 void MacroAssembler::warn(const char* msg) {
  767   push(rbp);
  768   movq(rbp, rsp);
  769   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  770   push_CPU_state();   // keeps alignment at 16 bytes
  771 
  772 #ifdef _WIN64
  773   // Windows always allocates space for its register args
  774   subq(rsp,  frame::arg_reg_save_area_bytes);
  775 #endif
  776   lea(c_rarg0, ExternalAddress((address) msg));
  777   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
  778 
  779 #ifdef _WIN64
  780   // restore stack pointer
  781   addq(rsp, frame::arg_reg_save_area_bytes);
  782 #endif
  783   pop_CPU_state();
  784   mov(rsp, rbp);
  785   pop(rbp);
  786 }
  787 
  788 void MacroAssembler::print_state() {
  789   address rip = pc();
  790   pusha();            // get regs on stack
  791   push(rbp);
  792   movq(rbp, rsp);
  793   andq(rsp, -16);     // align stack as required by push_CPU_state and call
  794   push_CPU_state();   // keeps alignment at 16 bytes
  795 
  796   lea(c_rarg0, InternalAddress(rip));
  797   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
  798   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
  799 
  800   pop_CPU_state();
  801   mov(rsp, rbp);
  802   pop(rbp);
  803   popa();
  804 }
  805 
  806 #ifndef PRODUCT
  807 extern "C" void findpc(intptr_t x);
  808 #endif
  809 
  810 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
  811   // In order to get locks to work, we need to fake a in_VM state
  812   if (ShowMessageBoxOnError) {
  813     JavaThread* thread = JavaThread::current();
  814     JavaThreadState saved_state = thread->thread_state();
  815     thread->set_thread_state(_thread_in_vm);
  816 #ifndef PRODUCT
  817     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
  818       ttyLocker ttyl;
  819       BytecodeCounter::print();
  820     }
  821 #endif
  822     // To see where a verify_oop failed, get $ebx+40/X for this frame.
  823     // XXX correct this offset for amd64
  824     // This is the value of eip which points to where verify_oop will return.
  825     if (os::message_box(msg, "Execution stopped, print registers?")) {
  826       print_state64(pc, regs);
  827       BREAKPOINT;
  828     }
  829   }
  830   fatal("DEBUG MESSAGE: %s", msg);
  831 }
  832 
  833 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
  834   ttyLocker ttyl;
  835   DebuggingContext debugging{};
  836   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
  837 #ifndef PRODUCT
  838   tty->cr();
  839   findpc(pc);
  840   tty->cr();
  841 #endif
  842 #define PRINT_REG(rax, value) \
  843   { tty->print("%s = ", #rax); os::print_location(tty, value); }
  844   PRINT_REG(rax, regs[15]);
  845   PRINT_REG(rbx, regs[12]);
  846   PRINT_REG(rcx, regs[14]);
  847   PRINT_REG(rdx, regs[13]);
  848   PRINT_REG(rdi, regs[8]);
  849   PRINT_REG(rsi, regs[9]);
  850   PRINT_REG(rbp, regs[10]);
  851   // rsp is actually not stored by pusha(), compute the old rsp from regs (rsp after pusha): regs + 16 = old rsp
  852   PRINT_REG(rsp, (intptr_t)(&regs[16]));
  853   PRINT_REG(r8 , regs[7]);
  854   PRINT_REG(r9 , regs[6]);
  855   PRINT_REG(r10, regs[5]);
  856   PRINT_REG(r11, regs[4]);
  857   PRINT_REG(r12, regs[3]);
  858   PRINT_REG(r13, regs[2]);
  859   PRINT_REG(r14, regs[1]);
  860   PRINT_REG(r15, regs[0]);
  861 #undef PRINT_REG
  862   // Print some words near the top of the stack.
  863   int64_t* rsp = &regs[16];
  864   int64_t* dump_sp = rsp;
  865   for (int col1 = 0; col1 < 8; col1++) {
  866     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  867     os::print_location(tty, *dump_sp++);
  868   }
  869   for (int row = 0; row < 25; row++) {
  870     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
  871     for (int col = 0; col < 4; col++) {
  872       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
  873     }
  874     tty->cr();
  875   }
  876   // Print some instructions around pc:
  877   Disassembler::decode((address)pc-64, (address)pc);
  878   tty->print_cr("--------");
  879   Disassembler::decode((address)pc, (address)pc+32);
  880 }
  881 
  882 // The java_calling_convention describes stack locations as ideal slots on
  883 // a frame with no abi restrictions. Since we must observe abi restrictions
  884 // (like the placement of the register window) the slots must be biased by
  885 // the following value.
  886 static int reg2offset_in(VMReg r) {
  887   // Account for saved rbp and return address
  888   // This should really be in_preserve_stack_slots
  889   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
  890 }
  891 
  892 static int reg2offset_out(VMReg r) {
  893   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
  894 }
  895 
  896 // A long move
  897 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  898 
  899   // The calling conventions assures us that each VMregpair is either
  900   // all really one physical register or adjacent stack slots.
  901 
  902   if (src.is_single_phys_reg() ) {
  903     if (dst.is_single_phys_reg()) {
  904       if (dst.first() != src.first()) {
  905         mov(dst.first()->as_Register(), src.first()->as_Register());
  906       }
  907     } else {
  908       assert(dst.is_single_reg(), "not a stack pair: (%s, %s), (%s, %s)",
  909              src.first()->name(), src.second()->name(), dst.first()->name(), dst.second()->name());
  910       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  911     }
  912   } else if (dst.is_single_phys_reg()) {
  913     assert(src.is_single_reg(),  "not a stack pair");
  914     movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  915   } else {
  916     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  917     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  918     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  919   }
  920 }
  921 
  922 // A double move
  923 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  924 
  925   // The calling conventions assures us that each VMregpair is either
  926   // all really one physical register or adjacent stack slots.
  927 
  928   if (src.is_single_phys_reg() ) {
  929     if (dst.is_single_phys_reg()) {
  930       // In theory these overlap but the ordering is such that this is likely a nop
  931       if ( src.first() != dst.first()) {
  932         movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
  933       }
  934     } else {
  935       assert(dst.is_single_reg(), "not a stack pair");
  936       movdbl(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  937     }
  938   } else if (dst.is_single_phys_reg()) {
  939     assert(src.is_single_reg(),  "not a stack pair");
  940     movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  941   } else {
  942     assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
  943     movq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  944     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  945   }
  946 }
  947 
  948 
  949 // A float arg may have to do float reg int reg conversion
  950 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  951   assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
  952 
  953   // The calling conventions assures us that each VMregpair is either
  954   // all really one physical register or adjacent stack slots.
  955 
  956   if (src.first()->is_stack()) {
  957     if (dst.first()->is_stack()) {
  958       movl(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  959       movptr(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  960     } else {
  961       // stack to reg
  962       assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  963       movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  964     }
  965   } else if (dst.first()->is_stack()) {
  966     // reg to stack
  967     assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
  968     movflt(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_XMMRegister());
  969   } else {
  970     // reg to reg
  971     // In theory these overlap but the ordering is such that this is likely a nop
  972     if ( src.first() != dst.first()) {
  973       movdbl(dst.first()->as_XMMRegister(),  src.first()->as_XMMRegister());
  974     }
  975   }
  976 }
  977 
  978 // On 64 bit we will store integer like items to the stack as
  979 // 64 bits items (x86_32/64 abi) even though java would only store
  980 // 32bits for a parameter. On 32bit it will simply be 32 bits
  981 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
  982 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp, int in_stk_bias, int out_stk_bias) {
  983   if (src.first()->is_stack()) {
  984     if (dst.first()->is_stack()) {
  985       // stack to stack
  986       movslq(tmp, Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  987       movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), tmp);
  988     } else {
  989       // stack to reg
  990       movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first()) + in_stk_bias));
  991     }
  992   } else if (dst.first()->is_stack()) {
  993     // reg to stack
  994     // Do we really have to sign extend???
  995     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
  996     movq(Address(rsp, reg2offset_out(dst.first()) + out_stk_bias), src.first()->as_Register());
  997   } else {
  998     // Do we really have to sign extend???
  999     // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
 1000     if (dst.first() != src.first()) {
 1001       movq(dst.first()->as_Register(), src.first()->as_Register());
 1002     }
 1003   }
 1004 }
 1005 
 1006 void MacroAssembler::move_ptr(VMRegPair src, VMRegPair dst) {
 1007   if (src.first()->is_stack()) {
 1008     if (dst.first()->is_stack()) {
 1009       // stack to stack
 1010       movq(rax, Address(rbp, reg2offset_in(src.first())));
 1011       movq(Address(rsp, reg2offset_out(dst.first())), rax);
 1012     } else {
 1013       // stack to reg
 1014       movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
 1015     }
 1016   } else if (dst.first()->is_stack()) {
 1017     // reg to stack
 1018     movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
 1019   } else {
 1020     if (dst.first() != src.first()) {
 1021       movq(dst.first()->as_Register(), src.first()->as_Register());
 1022     }
 1023   }
 1024 }
 1025 
 1026 // An oop arg. Must pass a handle not the oop itself
 1027 void MacroAssembler::object_move(OopMap* map,
 1028                         int oop_handle_offset,
 1029                         int framesize_in_slots,
 1030                         VMRegPair src,
 1031                         VMRegPair dst,
 1032                         bool is_receiver,
 1033                         int* receiver_offset) {
 1034 
 1035   // must pass a handle. First figure out the location we use as a handle
 1036 
 1037   Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
 1038 
 1039   // See if oop is null if it is we need no handle
 1040 
 1041   if (src.first()->is_stack()) {
 1042 
 1043     // Oop is already on the stack as an argument
 1044     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 1045     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 1046     if (is_receiver) {
 1047       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 1048     }
 1049 
 1050     cmpptr(Address(rbp, reg2offset_in(src.first())), NULL_WORD);
 1051     lea(rHandle, Address(rbp, reg2offset_in(src.first())));
 1052     // conditionally move a null
 1053     cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
 1054   } else {
 1055 
 1056     // Oop is in a register we must store it to the space we reserve
 1057     // on the stack for oop_handles and pass a handle if oop is non-null
 1058 
 1059     const Register rOop = src.first()->as_Register();
 1060     int oop_slot;
 1061     if (rOop == j_rarg0)
 1062       oop_slot = 0;
 1063     else if (rOop == j_rarg1)
 1064       oop_slot = 1;
 1065     else if (rOop == j_rarg2)
 1066       oop_slot = 2;
 1067     else if (rOop == j_rarg3)
 1068       oop_slot = 3;
 1069     else if (rOop == j_rarg4)
 1070       oop_slot = 4;
 1071     else {
 1072       assert(rOop == j_rarg5, "wrong register");
 1073       oop_slot = 5;
 1074     }
 1075 
 1076     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 1077     int offset = oop_slot*VMRegImpl::stack_slot_size;
 1078 
 1079     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 1080     // Store oop in handle area, may be null
 1081     movptr(Address(rsp, offset), rOop);
 1082     if (is_receiver) {
 1083       *receiver_offset = offset;
 1084     }
 1085 
 1086     cmpptr(rOop, NULL_WORD);
 1087     lea(rHandle, Address(rsp, offset));
 1088     // conditionally move a null from the handle area where it was just stored
 1089     cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
 1090   }
 1091 
 1092   // If arg is on the stack then place it otherwise it is already in correct reg.
 1093   if (dst.first()->is_stack()) {
 1094     movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
 1095   }
 1096 }
 1097 
 1098 #endif // _LP64
 1099 
 1100 // Now versions that are common to 32/64 bit
 1101 
 1102 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 1103   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 1104 }
 1105 
 1106 void MacroAssembler::addptr(Register dst, Register src) {
 1107   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1108 }
 1109 
 1110 void MacroAssembler::addptr(Address dst, Register src) {
 1111   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 1112 }
 1113 
 1114 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1115   assert(rscratch != noreg || always_reachable(src), "missing");
 1116 
 1117   if (reachable(src)) {
 1118     Assembler::addsd(dst, as_Address(src));
 1119   } else {
 1120     lea(rscratch, src);
 1121     Assembler::addsd(dst, Address(rscratch, 0));
 1122   }
 1123 }
 1124 
 1125 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1126   assert(rscratch != noreg || always_reachable(src), "missing");
 1127 
 1128   if (reachable(src)) {
 1129     addss(dst, as_Address(src));
 1130   } else {
 1131     lea(rscratch, src);
 1132     addss(dst, Address(rscratch, 0));
 1133   }
 1134 }
 1135 
 1136 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1137   assert(rscratch != noreg || always_reachable(src), "missing");
 1138 
 1139   if (reachable(src)) {
 1140     Assembler::addpd(dst, as_Address(src));
 1141   } else {
 1142     lea(rscratch, src);
 1143     Assembler::addpd(dst, Address(rscratch, 0));
 1144   }
 1145 }
 1146 
 1147 // See 8273459.  Function for ensuring 64-byte alignment, intended for stubs only.
 1148 // Stub code is generated once and never copied.
 1149 // NMethods can't use this because they get copied and we can't force alignment > 32 bytes.
 1150 void MacroAssembler::align64() {
 1151   align(64, (uint)(uintptr_t)pc());
 1152 }
 1153 
 1154 void MacroAssembler::align32() {
 1155   align(32, (uint)(uintptr_t)pc());
 1156 }
 1157 
 1158 void MacroAssembler::align(uint modulus) {
 1159   // 8273459: Ensure alignment is possible with current segment alignment
 1160   assert(modulus <= (uintx)CodeEntryAlignment, "Alignment must be <= CodeEntryAlignment");
 1161   align(modulus, offset());
 1162 }
 1163 
 1164 void MacroAssembler::align(uint modulus, uint target) {
 1165   if (target % modulus != 0) {
 1166     nop(modulus - (target % modulus));
 1167   }
 1168 }
 1169 
 1170 void MacroAssembler::push_f(XMMRegister r) {
 1171   subptr(rsp, wordSize);
 1172   movflt(Address(rsp, 0), r);
 1173 }
 1174 
 1175 void MacroAssembler::pop_f(XMMRegister r) {
 1176   movflt(r, Address(rsp, 0));
 1177   addptr(rsp, wordSize);
 1178 }
 1179 
 1180 void MacroAssembler::push_d(XMMRegister r) {
 1181   subptr(rsp, 2 * wordSize);
 1182   movdbl(Address(rsp, 0), r);
 1183 }
 1184 
 1185 void MacroAssembler::pop_d(XMMRegister r) {
 1186   movdbl(r, Address(rsp, 0));
 1187   addptr(rsp, 2 * Interpreter::stackElementSize);
 1188 }
 1189 
 1190 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1191   // Used in sign-masking with aligned address.
 1192   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1193   assert(rscratch != noreg || always_reachable(src), "missing");
 1194 
 1195   if (UseAVX > 2 &&
 1196       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 1197       (dst->encoding() >= 16)) {
 1198     vpand(dst, dst, src, AVX_512bit, rscratch);
 1199   } else if (reachable(src)) {
 1200     Assembler::andpd(dst, as_Address(src));
 1201   } else {
 1202     lea(rscratch, src);
 1203     Assembler::andpd(dst, Address(rscratch, 0));
 1204   }
 1205 }
 1206 
 1207 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1208   // Used in sign-masking with aligned address.
 1209   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 1210   assert(rscratch != noreg || always_reachable(src), "missing");
 1211 
 1212   if (reachable(src)) {
 1213     Assembler::andps(dst, as_Address(src));
 1214   } else {
 1215     lea(rscratch, src);
 1216     Assembler::andps(dst, Address(rscratch, 0));
 1217   }
 1218 }
 1219 
 1220 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 1221   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 1222 }
 1223 
 1224 #ifdef _LP64
 1225 void MacroAssembler::andq(Register dst, AddressLiteral src, Register rscratch) {
 1226   assert(rscratch != noreg || always_reachable(src), "missing");
 1227 
 1228   if (reachable(src)) {
 1229     andq(dst, as_Address(src));
 1230   } else {
 1231     lea(rscratch, src);
 1232     andq(dst, Address(rscratch, 0));
 1233   }
 1234 }
 1235 #endif
 1236 
 1237 void MacroAssembler::atomic_incl(Address counter_addr) {
 1238   lock();
 1239   incrementl(counter_addr);
 1240 }
 1241 
 1242 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register rscratch) {
 1243   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1244 
 1245   if (reachable(counter_addr)) {
 1246     atomic_incl(as_Address(counter_addr));
 1247   } else {
 1248     lea(rscratch, counter_addr);
 1249     atomic_incl(Address(rscratch, 0));
 1250   }
 1251 }
 1252 
 1253 #ifdef _LP64
 1254 void MacroAssembler::atomic_incq(Address counter_addr) {
 1255   lock();
 1256   incrementq(counter_addr);
 1257 }
 1258 
 1259 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register rscratch) {
 1260   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1261 
 1262   if (reachable(counter_addr)) {
 1263     atomic_incq(as_Address(counter_addr));
 1264   } else {
 1265     lea(rscratch, counter_addr);
 1266     atomic_incq(Address(rscratch, 0));
 1267   }
 1268 }
 1269 #endif
 1270 
 1271 // Writes to stack successive pages until offset reached to check for
 1272 // stack overflow + shadow pages.  This clobbers tmp.
 1273 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 1274   movptr(tmp, rsp);
 1275   // Bang stack for total size given plus shadow page size.
 1276   // Bang one page at a time because large size can bang beyond yellow and
 1277   // red zones.
 1278   Label loop;
 1279   bind(loop);
 1280   movl(Address(tmp, (-(int)os::vm_page_size())), size );
 1281   subptr(tmp, (int)os::vm_page_size());
 1282   subl(size, (int)os::vm_page_size());
 1283   jcc(Assembler::greater, loop);
 1284 
 1285   // Bang down shadow pages too.
 1286   // At this point, (tmp-0) is the last address touched, so don't
 1287   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
 1288   // was post-decremented.)  Skip this address by starting at i=1, and
 1289   // touch a few more pages below.  N.B.  It is important to touch all
 1290   // the way down including all pages in the shadow zone.
 1291   for (int i = 1; i < ((int)StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()); i++) {
 1292     // this could be any sized move but this is can be a debugging crumb
 1293     // so the bigger the better.
 1294     movptr(Address(tmp, (-i*(int)os::vm_page_size())), size );
 1295   }
 1296 }
 1297 
 1298 void MacroAssembler::reserved_stack_check() {
 1299   // testing if reserved zone needs to be enabled
 1300   Label no_reserved_zone_enabling;
 1301   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 1302   NOT_LP64(get_thread(rsi);)
 1303 
 1304   cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
 1305   jcc(Assembler::below, no_reserved_zone_enabling);
 1306 
 1307   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
 1308   jump(RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry()));
 1309   should_not_reach_here();
 1310 
 1311   bind(no_reserved_zone_enabling);
 1312 }
 1313 
 1314 void MacroAssembler::c2bool(Register x) {
 1315   // implements x == 0 ? 0 : 1
 1316   // note: must only look at least-significant byte of x
 1317   //       since C-style booleans are stored in one byte
 1318   //       only! (was bug)
 1319   andl(x, 0xFF);
 1320   setb(Assembler::notZero, x);
 1321 }
 1322 
 1323 // Wouldn't need if AddressLiteral version had new name
 1324 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
 1325   Assembler::call(L, rtype);
 1326 }
 1327 
 1328 void MacroAssembler::call(Register entry) {
 1329   Assembler::call(entry);
 1330 }
 1331 
 1332 void MacroAssembler::call(AddressLiteral entry, Register rscratch) {
 1333   assert(rscratch != noreg || always_reachable(entry), "missing");
 1334 
 1335   if (reachable(entry)) {
 1336     Assembler::call_literal(entry.target(), entry.rspec());
 1337   } else {
 1338     lea(rscratch, entry);
 1339     Assembler::call(rscratch);
 1340   }
 1341 }
 1342 
 1343 void MacroAssembler::ic_call(address entry, jint method_index) {
 1344   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 1345 #ifdef _LP64
 1346   // Needs full 64-bit immediate for later patching.
 1347   mov64(rax, (int64_t)Universe::non_oop_word());
 1348 #else
 1349   movptr(rax, (intptr_t)Universe::non_oop_word());
 1350 #endif
 1351   call(AddressLiteral(entry, rh));
 1352 }
 1353 
 1354 int MacroAssembler::ic_check_size() {
 1355   return
 1356       LP64_ONLY(UseCompactObjectHeaders ? 17 : 14) NOT_LP64(12);
 1357 }
 1358 
 1359 int MacroAssembler::ic_check(int end_alignment) {
 1360   Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx);
 1361   Register data = rax;
 1362   Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx);
 1363 
 1364   // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
 1365   // before the inline cache check, so we don't have to execute any nop instructions when dispatching
 1366   // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
 1367   // before the inline cache check here, and not after
 1368   align(end_alignment, offset() + ic_check_size());
 1369 
 1370   int uep_offset = offset();
 1371 
 1372 #ifdef _LP64
 1373   if (UseCompactObjectHeaders) {
 1374     load_narrow_klass_compact(temp, receiver);
 1375     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1376   } else
 1377 #endif
 1378   if (UseCompressedClassPointers) {
 1379     movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1380     cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1381   } else {
 1382     movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
 1383     cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
 1384   }
 1385 
 1386   // if inline cache check fails, then jump to runtime routine
 1387   jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 1388   assert((offset() % end_alignment) == 0, "Misaligned verified entry point (%d, %d, %d)", uep_offset, offset(), end_alignment);
 1389 
 1390   return uep_offset;
 1391 }
 1392 
 1393 void MacroAssembler::emit_static_call_stub() {
 1394   // Static stub relocation also tags the Method* in the code-stream.
 1395   mov_metadata(rbx, (Metadata*) nullptr);  // Method is zapped till fixup time.
 1396   // This is recognized as unresolved by relocs/nativeinst/ic code.
 1397   jump(RuntimeAddress(pc()));
 1398 }
 1399 
 1400 // Implementation of call_VM versions
 1401 
 1402 void MacroAssembler::call_VM(Register oop_result,
 1403                              address entry_point,
 1404                              bool check_exceptions) {
 1405   Label C, E;
 1406   call(C, relocInfo::none);
 1407   jmp(E);
 1408 
 1409   bind(C);
 1410   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 1411   ret(0);
 1412 
 1413   bind(E);
 1414 }
 1415 
 1416 void MacroAssembler::call_VM(Register oop_result,
 1417                              address entry_point,
 1418                              Register arg_1,
 1419                              bool check_exceptions) {
 1420   Label C, E;
 1421   call(C, relocInfo::none);
 1422   jmp(E);
 1423 
 1424   bind(C);
 1425   pass_arg1(this, arg_1);
 1426   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 1427   ret(0);
 1428 
 1429   bind(E);
 1430 }
 1431 
 1432 void MacroAssembler::call_VM(Register oop_result,
 1433                              address entry_point,
 1434                              Register arg_1,
 1435                              Register arg_2,
 1436                              bool check_exceptions) {
 1437   Label C, E;
 1438   call(C, relocInfo::none);
 1439   jmp(E);
 1440 
 1441   bind(C);
 1442 
 1443   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1444 
 1445   pass_arg2(this, arg_2);
 1446   pass_arg1(this, arg_1);
 1447   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 1448   ret(0);
 1449 
 1450   bind(E);
 1451 }
 1452 
 1453 void MacroAssembler::call_VM(Register oop_result,
 1454                              address entry_point,
 1455                              Register arg_1,
 1456                              Register arg_2,
 1457                              Register arg_3,
 1458                              bool check_exceptions) {
 1459   Label C, E;
 1460   call(C, relocInfo::none);
 1461   jmp(E);
 1462 
 1463   bind(C);
 1464 
 1465   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1466   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1467   pass_arg3(this, arg_3);
 1468   pass_arg2(this, arg_2);
 1469   pass_arg1(this, arg_1);
 1470   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 1471   ret(0);
 1472 
 1473   bind(E);
 1474 }
 1475 
 1476 void MacroAssembler::call_VM(Register oop_result,
 1477                              Register last_java_sp,
 1478                              address entry_point,
 1479                              int number_of_arguments,
 1480                              bool check_exceptions) {
 1481   call_VM_base(oop_result, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1482 }
 1483 
 1484 void MacroAssembler::call_VM(Register oop_result,
 1485                              Register last_java_sp,
 1486                              address entry_point,
 1487                              Register arg_1,
 1488                              bool check_exceptions) {
 1489   pass_arg1(this, arg_1);
 1490   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1491 }
 1492 
 1493 void MacroAssembler::call_VM(Register oop_result,
 1494                              Register last_java_sp,
 1495                              address entry_point,
 1496                              Register arg_1,
 1497                              Register arg_2,
 1498                              bool check_exceptions) {
 1499 
 1500   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1501   pass_arg2(this, arg_2);
 1502   pass_arg1(this, arg_1);
 1503   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1504 }
 1505 
 1506 void MacroAssembler::call_VM(Register oop_result,
 1507                              Register last_java_sp,
 1508                              address entry_point,
 1509                              Register arg_1,
 1510                              Register arg_2,
 1511                              Register arg_3,
 1512                              bool check_exceptions) {
 1513   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1514   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1515   pass_arg3(this, arg_3);
 1516   pass_arg2(this, arg_2);
 1517   pass_arg1(this, arg_1);
 1518   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1519 }
 1520 
 1521 void MacroAssembler::super_call_VM(Register oop_result,
 1522                                    Register last_java_sp,
 1523                                    address entry_point,
 1524                                    int number_of_arguments,
 1525                                    bool check_exceptions) {
 1526   MacroAssembler::call_VM_base(oop_result, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 1527 }
 1528 
 1529 void MacroAssembler::super_call_VM(Register oop_result,
 1530                                    Register last_java_sp,
 1531                                    address entry_point,
 1532                                    Register arg_1,
 1533                                    bool check_exceptions) {
 1534   pass_arg1(this, arg_1);
 1535   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 1536 }
 1537 
 1538 void MacroAssembler::super_call_VM(Register oop_result,
 1539                                    Register last_java_sp,
 1540                                    address entry_point,
 1541                                    Register arg_1,
 1542                                    Register arg_2,
 1543                                    bool check_exceptions) {
 1544 
 1545   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1546   pass_arg2(this, arg_2);
 1547   pass_arg1(this, arg_1);
 1548   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 1549 }
 1550 
 1551 void MacroAssembler::super_call_VM(Register oop_result,
 1552                                    Register last_java_sp,
 1553                                    address entry_point,
 1554                                    Register arg_1,
 1555                                    Register arg_2,
 1556                                    Register arg_3,
 1557                                    bool check_exceptions) {
 1558   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1559   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1560   pass_arg3(this, arg_3);
 1561   pass_arg2(this, arg_2);
 1562   pass_arg1(this, arg_1);
 1563   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 1564 }
 1565 
 1566 void MacroAssembler::call_VM_base(Register oop_result,
 1567                                   Register last_java_sp,
 1568                                   address  entry_point,
 1569                                   int      number_of_arguments,
 1570                                   bool     check_exceptions) {
 1571   Register java_thread = r15_thread;
 1572 
 1573   // determine last_java_sp register
 1574   if (!last_java_sp->is_valid()) {
 1575     last_java_sp = rsp;
 1576   }
 1577   // debugging support
 1578   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 1579 #ifdef ASSERT
 1580   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
 1581   // r12 is the heapbase.
 1582   if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
 1583 #endif // ASSERT
 1584 
 1585   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 1586   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 1587 
 1588   // push java thread (becomes first argument of C function)
 1589 
 1590   mov(c_rarg0, r15_thread);
 1591 
 1592   // set last Java frame before call
 1593   assert(last_java_sp != rbp, "can't use ebp/rbp");
 1594 
 1595   // Only interpreter should have to set fp
 1596   set_last_Java_frame(last_java_sp, rbp, nullptr, rscratch1);
 1597 
 1598   // do the call, remove parameters
 1599   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
 1600 
 1601 #ifdef ASSERT
 1602   // Check that thread register is not clobbered.
 1603   guarantee(java_thread != rax, "change this code");
 1604   push(rax);
 1605   { Label L;
 1606     get_thread_slow(rax);
 1607     cmpptr(java_thread, rax);
 1608     jcc(Assembler::equal, L);
 1609     STOP("MacroAssembler::call_VM_base: java_thread not callee saved?");
 1610     bind(L);
 1611   }
 1612   pop(rax);
 1613 #endif
 1614 
 1615   // reset last Java frame
 1616   // Only interpreter should have to clear fp
 1617   reset_last_Java_frame(true);
 1618 
 1619    // C++ interp handles this in the interpreter
 1620   check_and_handle_popframe();
 1621   check_and_handle_earlyret();
 1622 
 1623   if (check_exceptions) {
 1624     // check for pending exceptions (java_thread is set upon return)
 1625     cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
 1626     // This used to conditionally jump to forward_exception however it is
 1627     // possible if we relocate that the branch will not reach. So we must jump
 1628     // around so we can always reach
 1629 
 1630     Label ok;
 1631     jcc(Assembler::equal, ok);
 1632     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 1633     bind(ok);
 1634   }
 1635 
 1636   // get oop result if there is one and reset the value in the thread
 1637   if (oop_result->is_valid()) {
 1638     get_vm_result(oop_result);
 1639   }
 1640 }
 1641 
 1642 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 1643   // Calculate the value for last_Java_sp somewhat subtle.
 1644   // call_VM does an intermediate call which places a return address on
 1645   // the stack just under the stack pointer as the user finished with it.
 1646   // This allows use to retrieve last_Java_pc from last_Java_sp[-1].
 1647 
 1648   // We've pushed one address, correct last_Java_sp
 1649   lea(rax, Address(rsp, wordSize));
 1650 
 1651   call_VM_base(oop_result, rax, entry_point, number_of_arguments, check_exceptions);
 1652 }
 1653 
 1654 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
 1655 void MacroAssembler::call_VM_leaf0(address entry_point) {
 1656   MacroAssembler::call_VM_leaf_base(entry_point, 0);
 1657 }
 1658 
 1659 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
 1660   call_VM_leaf_base(entry_point, number_of_arguments);
 1661 }
 1662 
 1663 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
 1664   pass_arg0(this, arg_0);
 1665   call_VM_leaf(entry_point, 1);
 1666 }
 1667 
 1668 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1669 
 1670   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1671   pass_arg1(this, arg_1);
 1672   pass_arg0(this, arg_0);
 1673   call_VM_leaf(entry_point, 2);
 1674 }
 1675 
 1676 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1677   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1678   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1679   pass_arg2(this, arg_2);
 1680   pass_arg1(this, arg_1);
 1681   pass_arg0(this, arg_0);
 1682   call_VM_leaf(entry_point, 3);
 1683 }
 1684 
 1685 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1686   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1687   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1688   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1689   pass_arg3(this, arg_3);
 1690   pass_arg2(this, arg_2);
 1691   pass_arg1(this, arg_1);
 1692   pass_arg0(this, arg_0);
 1693   call_VM_leaf(entry_point, 3);
 1694 }
 1695 
 1696 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1697   pass_arg0(this, arg_0);
 1698   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1699 }
 1700 
 1701 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1702   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1703   pass_arg1(this, arg_1);
 1704   pass_arg0(this, arg_0);
 1705   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1706 }
 1707 
 1708 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1709   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1710   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1711   pass_arg2(this, arg_2);
 1712   pass_arg1(this, arg_1);
 1713   pass_arg0(this, arg_0);
 1714   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 1715 }
 1716 
 1717 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1718   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1719   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1720   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1721   pass_arg3(this, arg_3);
 1722   pass_arg2(this, arg_2);
 1723   pass_arg1(this, arg_1);
 1724   pass_arg0(this, arg_0);
 1725   MacroAssembler::call_VM_leaf_base(entry_point, 4);
 1726 }
 1727 
 1728 void MacroAssembler::get_vm_result(Register oop_result) {
 1729   movptr(oop_result, Address(r15_thread, JavaThread::vm_result_offset()));
 1730   movptr(Address(r15_thread, JavaThread::vm_result_offset()), NULL_WORD);
 1731   verify_oop_msg(oop_result, "broken oop in call_VM_base");
 1732 }
 1733 
 1734 void MacroAssembler::get_vm_result_2(Register metadata_result) {
 1735   movptr(metadata_result, Address(r15_thread, JavaThread::vm_result_2_offset()));
 1736   movptr(Address(r15_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
 1737 }
 1738 
 1739 void MacroAssembler::check_and_handle_earlyret() {
 1740 }
 1741 
 1742 void MacroAssembler::check_and_handle_popframe() {
 1743 }
 1744 
 1745 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm, Register rscratch) {
 1746   assert(rscratch != noreg || always_reachable(src1), "missing");
 1747 
 1748   if (reachable(src1)) {
 1749     cmpl(as_Address(src1), imm);
 1750   } else {
 1751     lea(rscratch, src1);
 1752     cmpl(Address(rscratch, 0), imm);
 1753   }
 1754 }
 1755 
 1756 void MacroAssembler::cmp32(Register src1, AddressLiteral src2, Register rscratch) {
 1757   assert(!src2.is_lval(), "use cmpptr");
 1758   assert(rscratch != noreg || always_reachable(src2), "missing");
 1759 
 1760   if (reachable(src2)) {
 1761     cmpl(src1, as_Address(src2));
 1762   } else {
 1763     lea(rscratch, src2);
 1764     cmpl(src1, Address(rscratch, 0));
 1765   }
 1766 }
 1767 
 1768 void MacroAssembler::cmp32(Register src1, int32_t imm) {
 1769   Assembler::cmpl(src1, imm);
 1770 }
 1771 
 1772 void MacroAssembler::cmp32(Register src1, Address src2) {
 1773   Assembler::cmpl(src1, src2);
 1774 }
 1775 
 1776 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1777   ucomisd(opr1, opr2);
 1778 
 1779   Label L;
 1780   if (unordered_is_less) {
 1781     movl(dst, -1);
 1782     jcc(Assembler::parity, L);
 1783     jcc(Assembler::below , L);
 1784     movl(dst, 0);
 1785     jcc(Assembler::equal , L);
 1786     increment(dst);
 1787   } else { // unordered is greater
 1788     movl(dst, 1);
 1789     jcc(Assembler::parity, L);
 1790     jcc(Assembler::above , L);
 1791     movl(dst, 0);
 1792     jcc(Assembler::equal , L);
 1793     decrementl(dst);
 1794   }
 1795   bind(L);
 1796 }
 1797 
 1798 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
 1799   ucomiss(opr1, opr2);
 1800 
 1801   Label L;
 1802   if (unordered_is_less) {
 1803     movl(dst, -1);
 1804     jcc(Assembler::parity, L);
 1805     jcc(Assembler::below , L);
 1806     movl(dst, 0);
 1807     jcc(Assembler::equal , L);
 1808     increment(dst);
 1809   } else { // unordered is greater
 1810     movl(dst, 1);
 1811     jcc(Assembler::parity, L);
 1812     jcc(Assembler::above , L);
 1813     movl(dst, 0);
 1814     jcc(Assembler::equal , L);
 1815     decrementl(dst);
 1816   }
 1817   bind(L);
 1818 }
 1819 
 1820 
 1821 void MacroAssembler::cmp8(AddressLiteral src1, int imm, Register rscratch) {
 1822   assert(rscratch != noreg || always_reachable(src1), "missing");
 1823 
 1824   if (reachable(src1)) {
 1825     cmpb(as_Address(src1), imm);
 1826   } else {
 1827     lea(rscratch, src1);
 1828     cmpb(Address(rscratch, 0), imm);
 1829   }
 1830 }
 1831 
 1832 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2, Register rscratch) {
 1833 #ifdef _LP64
 1834   assert(rscratch != noreg || always_reachable(src2), "missing");
 1835 
 1836   if (src2.is_lval()) {
 1837     movptr(rscratch, src2);
 1838     Assembler::cmpq(src1, rscratch);
 1839   } else if (reachable(src2)) {
 1840     cmpq(src1, as_Address(src2));
 1841   } else {
 1842     lea(rscratch, src2);
 1843     Assembler::cmpq(src1, Address(rscratch, 0));
 1844   }
 1845 #else
 1846   assert(rscratch == noreg, "not needed");
 1847   if (src2.is_lval()) {
 1848     cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1849   } else {
 1850     cmpl(src1, as_Address(src2));
 1851   }
 1852 #endif // _LP64
 1853 }
 1854 
 1855 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2, Register rscratch) {
 1856   assert(src2.is_lval(), "not a mem-mem compare");
 1857 #ifdef _LP64
 1858   // moves src2's literal address
 1859   movptr(rscratch, src2);
 1860   Assembler::cmpq(src1, rscratch);
 1861 #else
 1862   assert(rscratch == noreg, "not needed");
 1863   cmp_literal32(src1, (int32_t)src2.target(), src2.rspec());
 1864 #endif // _LP64
 1865 }
 1866 
 1867 void MacroAssembler::cmpoop(Register src1, Register src2) {
 1868   cmpptr(src1, src2);
 1869 }
 1870 
 1871 void MacroAssembler::cmpoop(Register src1, Address src2) {
 1872   cmpptr(src1, src2);
 1873 }
 1874 
 1875 #ifdef _LP64
 1876 void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
 1877   movoop(rscratch, src2);
 1878   cmpptr(src1, rscratch);
 1879 }
 1880 #endif
 1881 
 1882 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
 1883   assert(rscratch != noreg || always_reachable(adr), "missing");
 1884 
 1885   if (reachable(adr)) {
 1886     lock();
 1887     cmpxchgptr(reg, as_Address(adr));
 1888   } else {
 1889     lea(rscratch, adr);
 1890     lock();
 1891     cmpxchgptr(reg, Address(rscratch, 0));
 1892   }
 1893 }
 1894 
 1895 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
 1896   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
 1897 }
 1898 
 1899 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1900   assert(rscratch != noreg || always_reachable(src), "missing");
 1901 
 1902   if (reachable(src)) {
 1903     Assembler::comisd(dst, as_Address(src));
 1904   } else {
 1905     lea(rscratch, src);
 1906     Assembler::comisd(dst, Address(rscratch, 0));
 1907   }
 1908 }
 1909 
 1910 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 1911   assert(rscratch != noreg || always_reachable(src), "missing");
 1912 
 1913   if (reachable(src)) {
 1914     Assembler::comiss(dst, as_Address(src));
 1915   } else {
 1916     lea(rscratch, src);
 1917     Assembler::comiss(dst, Address(rscratch, 0));
 1918   }
 1919 }
 1920 
 1921 
 1922 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch) {
 1923   assert(rscratch != noreg || always_reachable(counter_addr), "missing");
 1924 
 1925   Condition negated_cond = negate_condition(cond);
 1926   Label L;
 1927   jcc(negated_cond, L);
 1928   pushf(); // Preserve flags
 1929   atomic_incl(counter_addr, rscratch);
 1930   popf();
 1931   bind(L);
 1932 }
 1933 
 1934 int MacroAssembler::corrected_idivl(Register reg) {
 1935   // Full implementation of Java idiv and irem; checks for
 1936   // special case as described in JVM spec., p.243 & p.271.
 1937   // The function returns the (pc) offset of the idivl
 1938   // instruction - may be needed for implicit exceptions.
 1939   //
 1940   //         normal case                           special case
 1941   //
 1942   // input : rax,: dividend                         min_int
 1943   //         reg: divisor   (may not be rax,/rdx)   -1
 1944   //
 1945   // output: rax,: quotient  (= rax, idiv reg)       min_int
 1946   //         rdx: remainder (= rax, irem reg)       0
 1947   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
 1948   const int min_int = 0x80000000;
 1949   Label normal_case, special_case;
 1950 
 1951   // check for special case
 1952   cmpl(rax, min_int);
 1953   jcc(Assembler::notEqual, normal_case);
 1954   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
 1955   cmpl(reg, -1);
 1956   jcc(Assembler::equal, special_case);
 1957 
 1958   // handle normal case
 1959   bind(normal_case);
 1960   cdql();
 1961   int idivl_offset = offset();
 1962   idivl(reg);
 1963 
 1964   // normal and special case exit
 1965   bind(special_case);
 1966 
 1967   return idivl_offset;
 1968 }
 1969 
 1970 
 1971 
 1972 void MacroAssembler::decrementl(Register reg, int value) {
 1973   if (value == min_jint) {subl(reg, value) ; return; }
 1974   if (value <  0) { incrementl(reg, -value); return; }
 1975   if (value == 0) {                        ; return; }
 1976   if (value == 1 && UseIncDec) { decl(reg) ; return; }
 1977   /* else */      { subl(reg, value)       ; return; }
 1978 }
 1979 
 1980 void MacroAssembler::decrementl(Address dst, int value) {
 1981   if (value == min_jint) {subl(dst, value) ; return; }
 1982   if (value <  0) { incrementl(dst, -value); return; }
 1983   if (value == 0) {                        ; return; }
 1984   if (value == 1 && UseIncDec) { decl(dst) ; return; }
 1985   /* else */      { subl(dst, value)       ; return; }
 1986 }
 1987 
 1988 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
 1989   assert(shift_value > 0, "illegal shift value");
 1990   Label _is_positive;
 1991   testl (reg, reg);
 1992   jcc (Assembler::positive, _is_positive);
 1993   int offset = (1 << shift_value) - 1 ;
 1994 
 1995   if (offset == 1) {
 1996     incrementl(reg);
 1997   } else {
 1998     addl(reg, offset);
 1999   }
 2000 
 2001   bind (_is_positive);
 2002   sarl(reg, shift_value);
 2003 }
 2004 
 2005 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2006   assert(rscratch != noreg || always_reachable(src), "missing");
 2007 
 2008   if (reachable(src)) {
 2009     Assembler::divsd(dst, as_Address(src));
 2010   } else {
 2011     lea(rscratch, src);
 2012     Assembler::divsd(dst, Address(rscratch, 0));
 2013   }
 2014 }
 2015 
 2016 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2017   assert(rscratch != noreg || always_reachable(src), "missing");
 2018 
 2019   if (reachable(src)) {
 2020     Assembler::divss(dst, as_Address(src));
 2021   } else {
 2022     lea(rscratch, src);
 2023     Assembler::divss(dst, Address(rscratch, 0));
 2024   }
 2025 }
 2026 
 2027 void MacroAssembler::enter() {
 2028   push(rbp);
 2029   mov(rbp, rsp);
 2030 }
 2031 
 2032 void MacroAssembler::post_call_nop() {
 2033   if (!Continuations::enabled()) {
 2034     return;
 2035   }
 2036   InstructionMark im(this);
 2037   relocate(post_call_nop_Relocation::spec());
 2038   InlineSkippedInstructionsCounter skipCounter(this);
 2039   emit_int8((uint8_t)0x0f);
 2040   emit_int8((uint8_t)0x1f);
 2041   emit_int8((uint8_t)0x84);
 2042   emit_int8((uint8_t)0x00);
 2043   emit_int32(0x00);
 2044 }
 2045 
 2046 // A 5 byte nop that is safe for patching (see patch_verified_entry)
 2047 void MacroAssembler::fat_nop() {
 2048   if (UseAddressNop) {
 2049     addr_nop_5();
 2050   } else {
 2051     emit_int8((uint8_t)0x26); // es:
 2052     emit_int8((uint8_t)0x2e); // cs:
 2053     emit_int8((uint8_t)0x64); // fs:
 2054     emit_int8((uint8_t)0x65); // gs:
 2055     emit_int8((uint8_t)0x90);
 2056   }
 2057 }
 2058 
 2059 #ifndef _LP64
 2060 void MacroAssembler::fcmp(Register tmp) {
 2061   fcmp(tmp, 1, true, true);
 2062 }
 2063 
 2064 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
 2065   assert(!pop_right || pop_left, "usage error");
 2066   if (VM_Version::supports_cmov()) {
 2067     assert(tmp == noreg, "unneeded temp");
 2068     if (pop_left) {
 2069       fucomip(index);
 2070     } else {
 2071       fucomi(index);
 2072     }
 2073     if (pop_right) {
 2074       fpop();
 2075     }
 2076   } else {
 2077     assert(tmp != noreg, "need temp");
 2078     if (pop_left) {
 2079       if (pop_right) {
 2080         fcompp();
 2081       } else {
 2082         fcomp(index);
 2083       }
 2084     } else {
 2085       fcom(index);
 2086     }
 2087     // convert FPU condition into eflags condition via rax,
 2088     save_rax(tmp);
 2089     fwait(); fnstsw_ax();
 2090     sahf();
 2091     restore_rax(tmp);
 2092   }
 2093   // condition codes set as follows:
 2094   //
 2095   // CF (corresponds to C0) if x < y
 2096   // PF (corresponds to C2) if unordered
 2097   // ZF (corresponds to C3) if x = y
 2098 }
 2099 
 2100 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
 2101   fcmp2int(dst, unordered_is_less, 1, true, true);
 2102 }
 2103 
 2104 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
 2105   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
 2106   Label L;
 2107   if (unordered_is_less) {
 2108     movl(dst, -1);
 2109     jcc(Assembler::parity, L);
 2110     jcc(Assembler::below , L);
 2111     movl(dst, 0);
 2112     jcc(Assembler::equal , L);
 2113     increment(dst);
 2114   } else { // unordered is greater
 2115     movl(dst, 1);
 2116     jcc(Assembler::parity, L);
 2117     jcc(Assembler::above , L);
 2118     movl(dst, 0);
 2119     jcc(Assembler::equal , L);
 2120     decrementl(dst);
 2121   }
 2122   bind(L);
 2123 }
 2124 
 2125 void MacroAssembler::fld_d(AddressLiteral src) {
 2126   fld_d(as_Address(src));
 2127 }
 2128 
 2129 void MacroAssembler::fld_s(AddressLiteral src) {
 2130   fld_s(as_Address(src));
 2131 }
 2132 
 2133 void MacroAssembler::fldcw(AddressLiteral src) {
 2134   fldcw(as_Address(src));
 2135 }
 2136 
 2137 void MacroAssembler::fpop() {
 2138   ffree();
 2139   fincstp();
 2140 }
 2141 
 2142 void MacroAssembler::fremr(Register tmp) {
 2143   save_rax(tmp);
 2144   { Label L;
 2145     bind(L);
 2146     fprem();
 2147     fwait(); fnstsw_ax();
 2148     sahf();
 2149     jcc(Assembler::parity, L);
 2150   }
 2151   restore_rax(tmp);
 2152   // Result is in ST0.
 2153   // Note: fxch & fpop to get rid of ST1
 2154   // (otherwise FPU stack could overflow eventually)
 2155   fxch(1);
 2156   fpop();
 2157 }
 2158 
 2159 void MacroAssembler::empty_FPU_stack() {
 2160   if (VM_Version::supports_mmx()) {
 2161     emms();
 2162   } else {
 2163     for (int i = 8; i-- > 0; ) ffree(i);
 2164   }
 2165 }
 2166 #endif // !LP64
 2167 
 2168 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2169   assert(rscratch != noreg || always_reachable(src), "missing");
 2170   if (reachable(src)) {
 2171     Assembler::mulpd(dst, as_Address(src));
 2172   } else {
 2173     lea(rscratch, src);
 2174     Assembler::mulpd(dst, Address(rscratch, 0));
 2175   }
 2176 }
 2177 
 2178 void MacroAssembler::load_float(Address src) {
 2179 #ifdef _LP64
 2180   movflt(xmm0, src);
 2181 #else
 2182   if (UseSSE >= 1) {
 2183     movflt(xmm0, src);
 2184   } else {
 2185     fld_s(src);
 2186   }
 2187 #endif // LP64
 2188 }
 2189 
 2190 void MacroAssembler::store_float(Address dst) {
 2191 #ifdef _LP64
 2192   movflt(dst, xmm0);
 2193 #else
 2194   if (UseSSE >= 1) {
 2195     movflt(dst, xmm0);
 2196   } else {
 2197     fstp_s(dst);
 2198   }
 2199 #endif // LP64
 2200 }
 2201 
 2202 void MacroAssembler::load_double(Address src) {
 2203 #ifdef _LP64
 2204   movdbl(xmm0, src);
 2205 #else
 2206   if (UseSSE >= 2) {
 2207     movdbl(xmm0, src);
 2208   } else {
 2209     fld_d(src);
 2210   }
 2211 #endif // LP64
 2212 }
 2213 
 2214 void MacroAssembler::store_double(Address dst) {
 2215 #ifdef _LP64
 2216   movdbl(dst, xmm0);
 2217 #else
 2218   if (UseSSE >= 2) {
 2219     movdbl(dst, xmm0);
 2220   } else {
 2221     fstp_d(dst);
 2222   }
 2223 #endif // LP64
 2224 }
 2225 
 2226 // dst = c = a * b + c
 2227 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2228   Assembler::vfmadd231sd(c, a, b);
 2229   if (dst != c) {
 2230     movdbl(dst, c);
 2231   }
 2232 }
 2233 
 2234 // dst = c = a * b + c
 2235 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
 2236   Assembler::vfmadd231ss(c, a, b);
 2237   if (dst != c) {
 2238     movflt(dst, c);
 2239   }
 2240 }
 2241 
 2242 // dst = c = a * b + c
 2243 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2244   Assembler::vfmadd231pd(c, a, b, vector_len);
 2245   if (dst != c) {
 2246     vmovdqu(dst, c);
 2247   }
 2248 }
 2249 
 2250 // dst = c = a * b + c
 2251 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
 2252   Assembler::vfmadd231ps(c, a, b, vector_len);
 2253   if (dst != c) {
 2254     vmovdqu(dst, c);
 2255   }
 2256 }
 2257 
 2258 // dst = c = a * b + c
 2259 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2260   Assembler::vfmadd231pd(c, a, b, vector_len);
 2261   if (dst != c) {
 2262     vmovdqu(dst, c);
 2263   }
 2264 }
 2265 
 2266 // dst = c = a * b + c
 2267 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
 2268   Assembler::vfmadd231ps(c, a, b, vector_len);
 2269   if (dst != c) {
 2270     vmovdqu(dst, c);
 2271   }
 2272 }
 2273 
 2274 void MacroAssembler::incrementl(AddressLiteral dst, Register rscratch) {
 2275   assert(rscratch != noreg || always_reachable(dst), "missing");
 2276 
 2277   if (reachable(dst)) {
 2278     incrementl(as_Address(dst));
 2279   } else {
 2280     lea(rscratch, dst);
 2281     incrementl(Address(rscratch, 0));
 2282   }
 2283 }
 2284 
 2285 void MacroAssembler::incrementl(ArrayAddress dst, Register rscratch) {
 2286   incrementl(as_Address(dst, rscratch));
 2287 }
 2288 
 2289 void MacroAssembler::incrementl(Register reg, int value) {
 2290   if (value == min_jint) {addl(reg, value) ; return; }
 2291   if (value <  0) { decrementl(reg, -value); return; }
 2292   if (value == 0) {                        ; return; }
 2293   if (value == 1 && UseIncDec) { incl(reg) ; return; }
 2294   /* else */      { addl(reg, value)       ; return; }
 2295 }
 2296 
 2297 void MacroAssembler::incrementl(Address dst, int value) {
 2298   if (value == min_jint) {addl(dst, value) ; return; }
 2299   if (value <  0) { decrementl(dst, -value); return; }
 2300   if (value == 0) {                        ; return; }
 2301   if (value == 1 && UseIncDec) { incl(dst) ; return; }
 2302   /* else */      { addl(dst, value)       ; return; }
 2303 }
 2304 
 2305 void MacroAssembler::jump(AddressLiteral dst, Register rscratch) {
 2306   assert(rscratch != noreg || always_reachable(dst), "missing");
 2307   assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump");
 2308   if (reachable(dst)) {
 2309     jmp_literal(dst.target(), dst.rspec());
 2310   } else {
 2311     lea(rscratch, dst);
 2312     jmp(rscratch);
 2313   }
 2314 }
 2315 
 2316 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst, Register rscratch) {
 2317   assert(rscratch != noreg || always_reachable(dst), "missing");
 2318   assert(!dst.rspec().reloc()->is_data(), "should not use ExternalAddress for jump_cc");
 2319   if (reachable(dst)) {
 2320     InstructionMark im(this);
 2321     relocate(dst.reloc());
 2322     const int short_size = 2;
 2323     const int long_size = 6;
 2324     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
 2325     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
 2326       // 0111 tttn #8-bit disp
 2327       emit_int8(0x70 | cc);
 2328       emit_int8((offs - short_size) & 0xFF);
 2329     } else {
 2330       // 0000 1111 1000 tttn #32-bit disp
 2331       emit_int8(0x0F);
 2332       emit_int8((unsigned char)(0x80 | cc));
 2333       emit_int32(offs - long_size);
 2334     }
 2335   } else {
 2336 #ifdef ASSERT
 2337     warning("reversing conditional branch");
 2338 #endif /* ASSERT */
 2339     Label skip;
 2340     jccb(reverse[cc], skip);
 2341     lea(rscratch, dst);
 2342     Assembler::jmp(rscratch);
 2343     bind(skip);
 2344   }
 2345 }
 2346 
 2347 void MacroAssembler::cmp32_mxcsr_std(Address mxcsr_save, Register tmp, Register rscratch) {
 2348   ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
 2349   assert(rscratch != noreg || always_reachable(mxcsr_std), "missing");
 2350 
 2351   stmxcsr(mxcsr_save);
 2352   movl(tmp, mxcsr_save);
 2353   if (EnableX86ECoreOpts) {
 2354     // The mxcsr_std has status bits set for performance on ECore
 2355     orl(tmp, 0x003f);
 2356   } else {
 2357     // Mask out status bits (only check control and mask bits)
 2358     andl(tmp, 0xFFC0);
 2359   }
 2360   cmp32(tmp, mxcsr_std, rscratch);
 2361 }
 2362 
 2363 void MacroAssembler::ldmxcsr(AddressLiteral src, Register rscratch) {
 2364   assert(rscratch != noreg || always_reachable(src), "missing");
 2365 
 2366   if (reachable(src)) {
 2367     Assembler::ldmxcsr(as_Address(src));
 2368   } else {
 2369     lea(rscratch, src);
 2370     Assembler::ldmxcsr(Address(rscratch, 0));
 2371   }
 2372 }
 2373 
 2374 int MacroAssembler::load_signed_byte(Register dst, Address src) {
 2375   int off;
 2376   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2377     off = offset();
 2378     movsbl(dst, src); // movsxb
 2379   } else {
 2380     off = load_unsigned_byte(dst, src);
 2381     shll(dst, 24);
 2382     sarl(dst, 24);
 2383   }
 2384   return off;
 2385 }
 2386 
 2387 // Note: load_signed_short used to be called load_signed_word.
 2388 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
 2389 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
 2390 // The term "word" in HotSpot means a 32- or 64-bit machine word.
 2391 int MacroAssembler::load_signed_short(Register dst, Address src) {
 2392   int off;
 2393   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 2394     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
 2395     // version but this is what 64bit has always done. This seems to imply
 2396     // that users are only using 32bits worth.
 2397     off = offset();
 2398     movswl(dst, src); // movsxw
 2399   } else {
 2400     off = load_unsigned_short(dst, src);
 2401     shll(dst, 16);
 2402     sarl(dst, 16);
 2403   }
 2404   return off;
 2405 }
 2406 
 2407 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
 2408   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2409   // and "3.9 Partial Register Penalties", p. 22).
 2410   int off;
 2411   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
 2412     off = offset();
 2413     movzbl(dst, src); // movzxb
 2414   } else {
 2415     xorl(dst, dst);
 2416     off = offset();
 2417     movb(dst, src);
 2418   }
 2419   return off;
 2420 }
 2421 
 2422 // Note: load_unsigned_short used to be called load_unsigned_word.
 2423 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
 2424   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
 2425   // and "3.9 Partial Register Penalties", p. 22).
 2426   int off;
 2427   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
 2428     off = offset();
 2429     movzwl(dst, src); // movzxw
 2430   } else {
 2431     xorl(dst, dst);
 2432     off = offset();
 2433     movw(dst, src);
 2434   }
 2435   return off;
 2436 }
 2437 
 2438 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
 2439   switch (size_in_bytes) {
 2440 #ifndef _LP64
 2441   case  8:
 2442     assert(dst2 != noreg, "second dest register required");
 2443     movl(dst,  src);
 2444     movl(dst2, src.plus_disp(BytesPerInt));
 2445     break;
 2446 #else
 2447   case  8:  movq(dst, src); break;
 2448 #endif
 2449   case  4:  movl(dst, src); break;
 2450   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
 2451   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
 2452   default:  ShouldNotReachHere();
 2453   }
 2454 }
 2455 
 2456 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
 2457   switch (size_in_bytes) {
 2458 #ifndef _LP64
 2459   case  8:
 2460     assert(src2 != noreg, "second source register required");
 2461     movl(dst,                        src);
 2462     movl(dst.plus_disp(BytesPerInt), src2);
 2463     break;
 2464 #else
 2465   case  8:  movq(dst, src); break;
 2466 #endif
 2467   case  4:  movl(dst, src); break;
 2468   case  2:  movw(dst, src); break;
 2469   case  1:  movb(dst, src); break;
 2470   default:  ShouldNotReachHere();
 2471   }
 2472 }
 2473 
 2474 void MacroAssembler::mov32(AddressLiteral dst, Register src, Register rscratch) {
 2475   assert(rscratch != noreg || always_reachable(dst), "missing");
 2476 
 2477   if (reachable(dst)) {
 2478     movl(as_Address(dst), src);
 2479   } else {
 2480     lea(rscratch, dst);
 2481     movl(Address(rscratch, 0), src);
 2482   }
 2483 }
 2484 
 2485 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
 2486   if (reachable(src)) {
 2487     movl(dst, as_Address(src));
 2488   } else {
 2489     lea(dst, src);
 2490     movl(dst, Address(dst, 0));
 2491   }
 2492 }
 2493 
 2494 // C++ bool manipulation
 2495 
 2496 void MacroAssembler::movbool(Register dst, Address src) {
 2497   if(sizeof(bool) == 1)
 2498     movb(dst, src);
 2499   else if(sizeof(bool) == 2)
 2500     movw(dst, src);
 2501   else if(sizeof(bool) == 4)
 2502     movl(dst, src);
 2503   else
 2504     // unsupported
 2505     ShouldNotReachHere();
 2506 }
 2507 
 2508 void MacroAssembler::movbool(Address dst, bool boolconst) {
 2509   if(sizeof(bool) == 1)
 2510     movb(dst, (int) boolconst);
 2511   else if(sizeof(bool) == 2)
 2512     movw(dst, (int) boolconst);
 2513   else if(sizeof(bool) == 4)
 2514     movl(dst, (int) boolconst);
 2515   else
 2516     // unsupported
 2517     ShouldNotReachHere();
 2518 }
 2519 
 2520 void MacroAssembler::movbool(Address dst, Register src) {
 2521   if(sizeof(bool) == 1)
 2522     movb(dst, src);
 2523   else if(sizeof(bool) == 2)
 2524     movw(dst, src);
 2525   else if(sizeof(bool) == 4)
 2526     movl(dst, src);
 2527   else
 2528     // unsupported
 2529     ShouldNotReachHere();
 2530 }
 2531 
 2532 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2533   assert(rscratch != noreg || always_reachable(src), "missing");
 2534 
 2535   if (reachable(src)) {
 2536     movdl(dst, as_Address(src));
 2537   } else {
 2538     lea(rscratch, src);
 2539     movdl(dst, Address(rscratch, 0));
 2540   }
 2541 }
 2542 
 2543 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2544   assert(rscratch != noreg || always_reachable(src), "missing");
 2545 
 2546   if (reachable(src)) {
 2547     movq(dst, as_Address(src));
 2548   } else {
 2549     lea(rscratch, src);
 2550     movq(dst, Address(rscratch, 0));
 2551   }
 2552 }
 2553 
 2554 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2555   assert(rscratch != noreg || always_reachable(src), "missing");
 2556 
 2557   if (reachable(src)) {
 2558     if (UseXmmLoadAndClearUpper) {
 2559       movsd (dst, as_Address(src));
 2560     } else {
 2561       movlpd(dst, as_Address(src));
 2562     }
 2563   } else {
 2564     lea(rscratch, src);
 2565     if (UseXmmLoadAndClearUpper) {
 2566       movsd (dst, Address(rscratch, 0));
 2567     } else {
 2568       movlpd(dst, Address(rscratch, 0));
 2569     }
 2570   }
 2571 }
 2572 
 2573 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2574   assert(rscratch != noreg || always_reachable(src), "missing");
 2575 
 2576   if (reachable(src)) {
 2577     movss(dst, as_Address(src));
 2578   } else {
 2579     lea(rscratch, src);
 2580     movss(dst, Address(rscratch, 0));
 2581   }
 2582 }
 2583 
 2584 void MacroAssembler::movptr(Register dst, Register src) {
 2585   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2586 }
 2587 
 2588 void MacroAssembler::movptr(Register dst, Address src) {
 2589   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2590 }
 2591 
 2592 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 2593 void MacroAssembler::movptr(Register dst, intptr_t src) {
 2594 #ifdef _LP64
 2595   if (is_uimm32(src)) {
 2596     movl(dst, checked_cast<uint32_t>(src));
 2597   } else if (is_simm32(src)) {
 2598     movq(dst, checked_cast<int32_t>(src));
 2599   } else {
 2600     mov64(dst, src);
 2601   }
 2602 #else
 2603   movl(dst, src);
 2604 #endif
 2605 }
 2606 
 2607 void MacroAssembler::movptr(Address dst, Register src) {
 2608   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 2609 }
 2610 
 2611 void MacroAssembler::movptr(Address dst, int32_t src) {
 2612   LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src));
 2613 }
 2614 
 2615 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
 2616   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2617   Assembler::movdqu(dst, src);
 2618 }
 2619 
 2620 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
 2621   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2622   Assembler::movdqu(dst, src);
 2623 }
 2624 
 2625 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
 2626   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2627   Assembler::movdqu(dst, src);
 2628 }
 2629 
 2630 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2631   assert(rscratch != noreg || always_reachable(src), "missing");
 2632 
 2633   if (reachable(src)) {
 2634     movdqu(dst, as_Address(src));
 2635   } else {
 2636     lea(rscratch, src);
 2637     movdqu(dst, Address(rscratch, 0));
 2638   }
 2639 }
 2640 
 2641 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
 2642   assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2643   Assembler::vmovdqu(dst, src);
 2644 }
 2645 
 2646 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
 2647   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2648   Assembler::vmovdqu(dst, src);
 2649 }
 2650 
 2651 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
 2652   assert(((dst->encoding() < 16  && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 2653   Assembler::vmovdqu(dst, src);
 2654 }
 2655 
 2656 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2657   assert(rscratch != noreg || always_reachable(src), "missing");
 2658 
 2659   if (reachable(src)) {
 2660     vmovdqu(dst, as_Address(src));
 2661   }
 2662   else {
 2663     lea(rscratch, src);
 2664     vmovdqu(dst, Address(rscratch, 0));
 2665   }
 2666 }
 2667 
 2668 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2669   assert(rscratch != noreg || always_reachable(src), "missing");
 2670 
 2671   if (vector_len == AVX_512bit) {
 2672     evmovdquq(dst, src, AVX_512bit, rscratch);
 2673   } else if (vector_len == AVX_256bit) {
 2674     vmovdqu(dst, src, rscratch);
 2675   } else {
 2676     movdqu(dst, src, rscratch);
 2677   }
 2678 }
 2679 
 2680 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src, int vector_len) {
 2681   if (vector_len == AVX_512bit) {
 2682     evmovdquq(dst, src, AVX_512bit);
 2683   } else if (vector_len == AVX_256bit) {
 2684     vmovdqu(dst, src);
 2685   } else {
 2686     movdqu(dst, src);
 2687   }
 2688 }
 2689 
 2690 void MacroAssembler::vmovdqu(Address dst, XMMRegister src, int vector_len) {
 2691   if (vector_len == AVX_512bit) {
 2692     evmovdquq(dst, src, AVX_512bit);
 2693   } else if (vector_len == AVX_256bit) {
 2694     vmovdqu(dst, src);
 2695   } else {
 2696     movdqu(dst, src);
 2697   }
 2698 }
 2699 
 2700 void MacroAssembler::vmovdqu(XMMRegister dst, Address src, int vector_len) {
 2701   if (vector_len == AVX_512bit) {
 2702     evmovdquq(dst, src, AVX_512bit);
 2703   } else if (vector_len == AVX_256bit) {
 2704     vmovdqu(dst, src);
 2705   } else {
 2706     movdqu(dst, src);
 2707   }
 2708 }
 2709 
 2710 void MacroAssembler::vmovdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2711   assert(rscratch != noreg || always_reachable(src), "missing");
 2712 
 2713   if (reachable(src)) {
 2714     vmovdqa(dst, as_Address(src));
 2715   }
 2716   else {
 2717     lea(rscratch, src);
 2718     vmovdqa(dst, Address(rscratch, 0));
 2719   }
 2720 }
 2721 
 2722 void MacroAssembler::vmovdqa(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2723   assert(rscratch != noreg || always_reachable(src), "missing");
 2724 
 2725   if (vector_len == AVX_512bit) {
 2726     evmovdqaq(dst, src, AVX_512bit, rscratch);
 2727   } else if (vector_len == AVX_256bit) {
 2728     vmovdqa(dst, src, rscratch);
 2729   } else {
 2730     movdqa(dst, src, rscratch);
 2731   }
 2732 }
 2733 
 2734 void MacroAssembler::kmov(KRegister dst, Address src) {
 2735   if (VM_Version::supports_avx512bw()) {
 2736     kmovql(dst, src);
 2737   } else {
 2738     assert(VM_Version::supports_evex(), "");
 2739     kmovwl(dst, src);
 2740   }
 2741 }
 2742 
 2743 void MacroAssembler::kmov(Address dst, KRegister src) {
 2744   if (VM_Version::supports_avx512bw()) {
 2745     kmovql(dst, src);
 2746   } else {
 2747     assert(VM_Version::supports_evex(), "");
 2748     kmovwl(dst, src);
 2749   }
 2750 }
 2751 
 2752 void MacroAssembler::kmov(KRegister dst, KRegister src) {
 2753   if (VM_Version::supports_avx512bw()) {
 2754     kmovql(dst, src);
 2755   } else {
 2756     assert(VM_Version::supports_evex(), "");
 2757     kmovwl(dst, src);
 2758   }
 2759 }
 2760 
 2761 void MacroAssembler::kmov(Register dst, KRegister src) {
 2762   if (VM_Version::supports_avx512bw()) {
 2763     kmovql(dst, src);
 2764   } else {
 2765     assert(VM_Version::supports_evex(), "");
 2766     kmovwl(dst, src);
 2767   }
 2768 }
 2769 
 2770 void MacroAssembler::kmov(KRegister dst, Register src) {
 2771   if (VM_Version::supports_avx512bw()) {
 2772     kmovql(dst, src);
 2773   } else {
 2774     assert(VM_Version::supports_evex(), "");
 2775     kmovwl(dst, src);
 2776   }
 2777 }
 2778 
 2779 void MacroAssembler::kmovql(KRegister dst, AddressLiteral src, Register rscratch) {
 2780   assert(rscratch != noreg || always_reachable(src), "missing");
 2781 
 2782   if (reachable(src)) {
 2783     kmovql(dst, as_Address(src));
 2784   } else {
 2785     lea(rscratch, src);
 2786     kmovql(dst, Address(rscratch, 0));
 2787   }
 2788 }
 2789 
 2790 void MacroAssembler::kmovwl(KRegister dst, AddressLiteral src, Register rscratch) {
 2791   assert(rscratch != noreg || always_reachable(src), "missing");
 2792 
 2793   if (reachable(src)) {
 2794     kmovwl(dst, as_Address(src));
 2795   } else {
 2796     lea(rscratch, src);
 2797     kmovwl(dst, Address(rscratch, 0));
 2798   }
 2799 }
 2800 
 2801 void MacroAssembler::evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2802                                int vector_len, Register rscratch) {
 2803   assert(rscratch != noreg || always_reachable(src), "missing");
 2804 
 2805   if (reachable(src)) {
 2806     Assembler::evmovdqub(dst, mask, as_Address(src), merge, vector_len);
 2807   } else {
 2808     lea(rscratch, src);
 2809     Assembler::evmovdqub(dst, mask, Address(rscratch, 0), merge, vector_len);
 2810   }
 2811 }
 2812 
 2813 void MacroAssembler::evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge,
 2814                                int vector_len, Register rscratch) {
 2815   assert(rscratch != noreg || always_reachable(src), "missing");
 2816 
 2817   if (reachable(src)) {
 2818     Assembler::evmovdquw(dst, mask, as_Address(src), merge, vector_len);
 2819   } else {
 2820     lea(rscratch, src);
 2821     Assembler::evmovdquw(dst, mask, Address(rscratch, 0), merge, vector_len);
 2822   }
 2823 }
 2824 
 2825 void MacroAssembler::evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2826   assert(rscratch != noreg || always_reachable(src), "missing");
 2827 
 2828   if (reachable(src)) {
 2829     Assembler::evmovdqul(dst, mask, as_Address(src), merge, vector_len);
 2830   } else {
 2831     lea(rscratch, src);
 2832     Assembler::evmovdqul(dst, mask, Address(rscratch, 0), merge, vector_len);
 2833   }
 2834 }
 2835 
 2836 void MacroAssembler::evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2837   assert(rscratch != noreg || always_reachable(src), "missing");
 2838 
 2839   if (reachable(src)) {
 2840     Assembler::evmovdquq(dst, mask, as_Address(src), merge, vector_len);
 2841   } else {
 2842     lea(rscratch, src);
 2843     Assembler::evmovdquq(dst, mask, Address(rscratch, 0), merge, vector_len);
 2844   }
 2845 }
 2846 
 2847 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2848   assert(rscratch != noreg || always_reachable(src), "missing");
 2849 
 2850   if (reachable(src)) {
 2851     Assembler::evmovdquq(dst, as_Address(src), vector_len);
 2852   } else {
 2853     lea(rscratch, src);
 2854     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
 2855   }
 2856 }
 2857 
 2858 void MacroAssembler::evmovdqaq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
 2859   assert(rscratch != noreg || always_reachable(src), "missing");
 2860 
 2861   if (reachable(src)) {
 2862     Assembler::evmovdqaq(dst, mask, as_Address(src), merge, vector_len);
 2863   } else {
 2864     lea(rscratch, src);
 2865     Assembler::evmovdqaq(dst, mask, Address(rscratch, 0), merge, vector_len);
 2866   }
 2867 }
 2868 
 2869 void MacroAssembler::evmovdqaq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2870   assert(rscratch != noreg || always_reachable(src), "missing");
 2871 
 2872   if (reachable(src)) {
 2873     Assembler::evmovdqaq(dst, as_Address(src), vector_len);
 2874   } else {
 2875     lea(rscratch, src);
 2876     Assembler::evmovdqaq(dst, Address(rscratch, 0), vector_len);
 2877   }
 2878 }
 2879 
 2880 
 2881 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2882   assert(rscratch != noreg || always_reachable(src), "missing");
 2883 
 2884   if (reachable(src)) {
 2885     Assembler::movdqa(dst, as_Address(src));
 2886   } else {
 2887     lea(rscratch, src);
 2888     Assembler::movdqa(dst, Address(rscratch, 0));
 2889   }
 2890 }
 2891 
 2892 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2893   assert(rscratch != noreg || always_reachable(src), "missing");
 2894 
 2895   if (reachable(src)) {
 2896     Assembler::movsd(dst, as_Address(src));
 2897   } else {
 2898     lea(rscratch, src);
 2899     Assembler::movsd(dst, Address(rscratch, 0));
 2900   }
 2901 }
 2902 
 2903 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2904   assert(rscratch != noreg || always_reachable(src), "missing");
 2905 
 2906   if (reachable(src)) {
 2907     Assembler::movss(dst, as_Address(src));
 2908   } else {
 2909     lea(rscratch, src);
 2910     Assembler::movss(dst, Address(rscratch, 0));
 2911   }
 2912 }
 2913 
 2914 void MacroAssembler::movddup(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2915   assert(rscratch != noreg || always_reachable(src), "missing");
 2916 
 2917   if (reachable(src)) {
 2918     Assembler::movddup(dst, as_Address(src));
 2919   } else {
 2920     lea(rscratch, src);
 2921     Assembler::movddup(dst, Address(rscratch, 0));
 2922   }
 2923 }
 2924 
 2925 void MacroAssembler::vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 2926   assert(rscratch != noreg || always_reachable(src), "missing");
 2927 
 2928   if (reachable(src)) {
 2929     Assembler::vmovddup(dst, as_Address(src), vector_len);
 2930   } else {
 2931     lea(rscratch, src);
 2932     Assembler::vmovddup(dst, Address(rscratch, 0), vector_len);
 2933   }
 2934 }
 2935 
 2936 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2937   assert(rscratch != noreg || always_reachable(src), "missing");
 2938 
 2939   if (reachable(src)) {
 2940     Assembler::mulsd(dst, as_Address(src));
 2941   } else {
 2942     lea(rscratch, src);
 2943     Assembler::mulsd(dst, Address(rscratch, 0));
 2944   }
 2945 }
 2946 
 2947 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 2948   assert(rscratch != noreg || always_reachable(src), "missing");
 2949 
 2950   if (reachable(src)) {
 2951     Assembler::mulss(dst, as_Address(src));
 2952   } else {
 2953     lea(rscratch, src);
 2954     Assembler::mulss(dst, Address(rscratch, 0));
 2955   }
 2956 }
 2957 
 2958 void MacroAssembler::null_check(Register reg, int offset) {
 2959   if (needs_explicit_null_check(offset)) {
 2960     // provoke OS null exception if reg is null by
 2961     // accessing M[reg] w/o changing any (non-CC) registers
 2962     // NOTE: cmpl is plenty here to provoke a segv
 2963     cmpptr(rax, Address(reg, 0));
 2964     // Note: should probably use testl(rax, Address(reg, 0));
 2965     //       may be shorter code (however, this version of
 2966     //       testl needs to be implemented first)
 2967   } else {
 2968     // nothing to do, (later) access of M[reg + offset]
 2969     // will provoke OS null exception if reg is null
 2970   }
 2971 }
 2972 
 2973 void MacroAssembler::os_breakpoint() {
 2974   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 2975   // (e.g., MSVC can't call ps() otherwise)
 2976   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 2977 }
 2978 
 2979 void MacroAssembler::unimplemented(const char* what) {
 2980   const char* buf = nullptr;
 2981   {
 2982     ResourceMark rm;
 2983     stringStream ss;
 2984     ss.print("unimplemented: %s", what);
 2985     buf = code_string(ss.as_string());
 2986   }
 2987   stop(buf);
 2988 }
 2989 
 2990 #ifdef _LP64
 2991 #define XSTATE_BV 0x200
 2992 #endif
 2993 
 2994 void MacroAssembler::pop_CPU_state() {
 2995   pop_FPU_state();
 2996   pop_IU_state();
 2997 }
 2998 
 2999 void MacroAssembler::pop_FPU_state() {
 3000 #ifndef _LP64
 3001   frstor(Address(rsp, 0));
 3002 #else
 3003   fxrstor(Address(rsp, 0));
 3004 #endif
 3005   addptr(rsp, FPUStateSizeInWords * wordSize);
 3006 }
 3007 
 3008 void MacroAssembler::pop_IU_state() {
 3009   popa();
 3010   LP64_ONLY(addq(rsp, 8));
 3011   popf();
 3012 }
 3013 
 3014 // Save Integer and Float state
 3015 // Warning: Stack must be 16 byte aligned (64bit)
 3016 void MacroAssembler::push_CPU_state() {
 3017   push_IU_state();
 3018   push_FPU_state();
 3019 }
 3020 
 3021 void MacroAssembler::push_FPU_state() {
 3022   subptr(rsp, FPUStateSizeInWords * wordSize);
 3023 #ifndef _LP64
 3024   fnsave(Address(rsp, 0));
 3025   fwait();
 3026 #else
 3027   fxsave(Address(rsp, 0));
 3028 #endif // LP64
 3029 }
 3030 
 3031 void MacroAssembler::push_IU_state() {
 3032   // Push flags first because pusha kills them
 3033   pushf();
 3034   // Make sure rsp stays 16-byte aligned
 3035   LP64_ONLY(subq(rsp, 8));
 3036   pusha();
 3037 }
 3038 
 3039 void MacroAssembler::push_cont_fastpath() {
 3040   if (!Continuations::enabled()) return;
 3041 
 3042 #ifndef _LP64
 3043   Register rthread = rax;
 3044   Register rrealsp = rbx;
 3045   push(rthread);
 3046   push(rrealsp);
 3047 
 3048   get_thread(rthread);
 3049 
 3050   // The code below wants the original RSP.
 3051   // Move it back after the pushes above.
 3052   movptr(rrealsp, rsp);
 3053   addptr(rrealsp, 2*wordSize);
 3054 #else
 3055   Register rthread = r15_thread;
 3056   Register rrealsp = rsp;
 3057 #endif
 3058 
 3059   Label done;
 3060   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3061   jccb(Assembler::belowEqual, done);
 3062   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), rrealsp);
 3063   bind(done);
 3064 
 3065 #ifndef _LP64
 3066   pop(rrealsp);
 3067   pop(rthread);
 3068 #endif
 3069 }
 3070 
 3071 void MacroAssembler::pop_cont_fastpath() {
 3072   if (!Continuations::enabled()) return;
 3073 
 3074 #ifndef _LP64
 3075   Register rthread = rax;
 3076   Register rrealsp = rbx;
 3077   push(rthread);
 3078   push(rrealsp);
 3079 
 3080   get_thread(rthread);
 3081 
 3082   // The code below wants the original RSP.
 3083   // Move it back after the pushes above.
 3084   movptr(rrealsp, rsp);
 3085   addptr(rrealsp, 2*wordSize);
 3086 #else
 3087   Register rthread = r15_thread;
 3088   Register rrealsp = rsp;
 3089 #endif
 3090 
 3091   Label done;
 3092   cmpptr(rrealsp, Address(rthread, JavaThread::cont_fastpath_offset()));
 3093   jccb(Assembler::below, done);
 3094   movptr(Address(rthread, JavaThread::cont_fastpath_offset()), 0);
 3095   bind(done);
 3096 
 3097 #ifndef _LP64
 3098   pop(rrealsp);
 3099   pop(rthread);
 3100 #endif
 3101 }
 3102 
 3103 void MacroAssembler::inc_held_monitor_count() {
 3104 #ifdef _LP64
 3105   incrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3106 #endif
 3107 }
 3108 
 3109 void MacroAssembler::dec_held_monitor_count() {
 3110 #ifdef _LP64
 3111   decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
 3112 #endif
 3113 }
 3114 
 3115 #ifdef ASSERT
 3116 void MacroAssembler::stop_if_in_cont(Register cont, const char* name) {
 3117 #ifdef _LP64
 3118   Label no_cont;
 3119   movptr(cont, Address(r15_thread, JavaThread::cont_entry_offset()));
 3120   testl(cont, cont);
 3121   jcc(Assembler::zero, no_cont);
 3122   stop(name);
 3123   bind(no_cont);
 3124 #else
 3125   Unimplemented();
 3126 #endif
 3127 }
 3128 #endif
 3129 
 3130 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { // determine java_thread register
 3131   // we must set sp to zero to clear frame
 3132   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 3133   // must clear fp, so that compiled frames are not confused; it is
 3134   // possible that we need it only for debugging
 3135   if (clear_fp) {
 3136     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 3137   }
 3138   // Always clear the pc because it could have been set by make_walkable()
 3139   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 3140   vzeroupper();
 3141 }
 3142 
 3143 void MacroAssembler::restore_rax(Register tmp) {
 3144   if (tmp == noreg) pop(rax);
 3145   else if (tmp != rax) mov(rax, tmp);
 3146 }
 3147 
 3148 void MacroAssembler::round_to(Register reg, int modulus) {
 3149   addptr(reg, modulus - 1);
 3150   andptr(reg, -modulus);
 3151 }
 3152 
 3153 void MacroAssembler::save_rax(Register tmp) {
 3154   if (tmp == noreg) push(rax);
 3155   else if (tmp != rax) mov(tmp, rax);
 3156 }
 3157 
 3158 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool in_nmethod) {
 3159   if (at_return) {
 3160     // Note that when in_nmethod is set, the stack pointer is incremented before the poll. Therefore,
 3161     // we may safely use rsp instead to perform the stack watermark check.
 3162     cmpptr(in_nmethod ? rsp : rbp, Address(r15_thread, JavaThread::polling_word_offset()));
 3163     jcc(Assembler::above, slow_path);
 3164     return;
 3165   }
 3166   testb(Address(r15_thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
 3167   jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
 3168 }
 3169 
 3170 // Calls to C land
 3171 //
 3172 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
 3173 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 3174 // has to be reset to 0. This is required to allow proper stack traversal.
 3175 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 3176                                          Register last_java_fp,
 3177                                          address  last_java_pc,
 3178                                          Register rscratch) {
 3179   vzeroupper();
 3180   // determine last_java_sp register
 3181   if (!last_java_sp->is_valid()) {
 3182     last_java_sp = rsp;
 3183   }
 3184   // last_java_fp is optional
 3185   if (last_java_fp->is_valid()) {
 3186     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
 3187   }
 3188   // last_java_pc is optional
 3189   if (last_java_pc != nullptr) {
 3190     Address java_pc(r15_thread,
 3191                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 3192     lea(java_pc, InternalAddress(last_java_pc), rscratch);
 3193   }
 3194   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 3195 }
 3196 
 3197 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 3198                                          Register last_java_fp,
 3199                                          Label &L,
 3200                                          Register scratch) {
 3201   lea(scratch, L);
 3202   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), scratch);
 3203   set_last_Java_frame(last_java_sp, last_java_fp, nullptr, scratch);
 3204 }
 3205 
 3206 void MacroAssembler::shlptr(Register dst, int imm8) {
 3207   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
 3208 }
 3209 
 3210 void MacroAssembler::shrptr(Register dst, int imm8) {
 3211   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
 3212 }
 3213 
 3214 void MacroAssembler::sign_extend_byte(Register reg) {
 3215   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
 3216     movsbl(reg, reg); // movsxb
 3217   } else {
 3218     shll(reg, 24);
 3219     sarl(reg, 24);
 3220   }
 3221 }
 3222 
 3223 void MacroAssembler::sign_extend_short(Register reg) {
 3224   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
 3225     movswl(reg, reg); // movsxw
 3226   } else {
 3227     shll(reg, 16);
 3228     sarl(reg, 16);
 3229   }
 3230 }
 3231 
 3232 void MacroAssembler::testl(Address dst, int32_t imm32) {
 3233   if (imm32 >= 0 && is8bit(imm32)) {
 3234     testb(dst, imm32);
 3235   } else {
 3236     Assembler::testl(dst, imm32);
 3237   }
 3238 }
 3239 
 3240 void MacroAssembler::testl(Register dst, int32_t imm32) {
 3241   if (imm32 >= 0 && is8bit(imm32) && dst->has_byte_register()) {
 3242     testb(dst, imm32);
 3243   } else {
 3244     Assembler::testl(dst, imm32);
 3245   }
 3246 }
 3247 
 3248 void MacroAssembler::testl(Register dst, AddressLiteral src) {
 3249   assert(always_reachable(src), "Address should be reachable");
 3250   testl(dst, as_Address(src));
 3251 }
 3252 
 3253 #ifdef _LP64
 3254 
 3255 void MacroAssembler::testq(Address dst, int32_t imm32) {
 3256   if (imm32 >= 0) {
 3257     testl(dst, imm32);
 3258   } else {
 3259     Assembler::testq(dst, imm32);
 3260   }
 3261 }
 3262 
 3263 void MacroAssembler::testq(Register dst, int32_t imm32) {
 3264   if (imm32 >= 0) {
 3265     testl(dst, imm32);
 3266   } else {
 3267     Assembler::testq(dst, imm32);
 3268   }
 3269 }
 3270 
 3271 #endif
 3272 
 3273 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
 3274   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3275   Assembler::pcmpeqb(dst, src);
 3276 }
 3277 
 3278 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
 3279   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3280   Assembler::pcmpeqw(dst, src);
 3281 }
 3282 
 3283 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 3284   assert((dst->encoding() < 16),"XMM register should be 0-15");
 3285   Assembler::pcmpestri(dst, src, imm8);
 3286 }
 3287 
 3288 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 3289   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3290   Assembler::pcmpestri(dst, src, imm8);
 3291 }
 3292 
 3293 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 3294   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3295   Assembler::pmovzxbw(dst, src);
 3296 }
 3297 
 3298 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
 3299   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3300   Assembler::pmovzxbw(dst, src);
 3301 }
 3302 
 3303 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
 3304   assert((src->encoding() < 16),"XMM register should be 0-15");
 3305   Assembler::pmovmskb(dst, src);
 3306 }
 3307 
 3308 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
 3309   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3310   Assembler::ptest(dst, src);
 3311 }
 3312 
 3313 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3314   assert(rscratch != noreg || always_reachable(src), "missing");
 3315 
 3316   if (reachable(src)) {
 3317     Assembler::sqrtss(dst, as_Address(src));
 3318   } else {
 3319     lea(rscratch, src);
 3320     Assembler::sqrtss(dst, Address(rscratch, 0));
 3321   }
 3322 }
 3323 
 3324 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3325   assert(rscratch != noreg || always_reachable(src), "missing");
 3326 
 3327   if (reachable(src)) {
 3328     Assembler::subsd(dst, as_Address(src));
 3329   } else {
 3330     lea(rscratch, src);
 3331     Assembler::subsd(dst, Address(rscratch, 0));
 3332   }
 3333 }
 3334 
 3335 void MacroAssembler::roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch) {
 3336   assert(rscratch != noreg || always_reachable(src), "missing");
 3337 
 3338   if (reachable(src)) {
 3339     Assembler::roundsd(dst, as_Address(src), rmode);
 3340   } else {
 3341     lea(rscratch, src);
 3342     Assembler::roundsd(dst, Address(rscratch, 0), rmode);
 3343   }
 3344 }
 3345 
 3346 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3347   assert(rscratch != noreg || always_reachable(src), "missing");
 3348 
 3349   if (reachable(src)) {
 3350     Assembler::subss(dst, as_Address(src));
 3351   } else {
 3352     lea(rscratch, src);
 3353     Assembler::subss(dst, Address(rscratch, 0));
 3354   }
 3355 }
 3356 
 3357 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3358   assert(rscratch != noreg || always_reachable(src), "missing");
 3359 
 3360   if (reachable(src)) {
 3361     Assembler::ucomisd(dst, as_Address(src));
 3362   } else {
 3363     lea(rscratch, src);
 3364     Assembler::ucomisd(dst, Address(rscratch, 0));
 3365   }
 3366 }
 3367 
 3368 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3369   assert(rscratch != noreg || always_reachable(src), "missing");
 3370 
 3371   if (reachable(src)) {
 3372     Assembler::ucomiss(dst, as_Address(src));
 3373   } else {
 3374     lea(rscratch, src);
 3375     Assembler::ucomiss(dst, Address(rscratch, 0));
 3376   }
 3377 }
 3378 
 3379 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3380   assert(rscratch != noreg || always_reachable(src), "missing");
 3381 
 3382   // Used in sign-bit flipping with aligned address.
 3383   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3384 
 3385   if (UseAVX > 2 &&
 3386       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3387       (dst->encoding() >= 16)) {
 3388     vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch);
 3389   } else if (reachable(src)) {
 3390     Assembler::xorpd(dst, as_Address(src));
 3391   } else {
 3392     lea(rscratch, src);
 3393     Assembler::xorpd(dst, Address(rscratch, 0));
 3394   }
 3395 }
 3396 
 3397 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
 3398   if (UseAVX > 2 &&
 3399       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3400       ((dst->encoding() >= 16) || (src->encoding() >= 16))) {
 3401     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3402   } else {
 3403     Assembler::xorpd(dst, src);
 3404   }
 3405 }
 3406 
 3407 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
 3408   if (UseAVX > 2 &&
 3409       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3410       ((dst->encoding() >= 16) || (src->encoding() >= 16))) {
 3411     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
 3412   } else {
 3413     Assembler::xorps(dst, src);
 3414   }
 3415 }
 3416 
 3417 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3418   assert(rscratch != noreg || always_reachable(src), "missing");
 3419 
 3420   // Used in sign-bit flipping with aligned address.
 3421   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 3422 
 3423   if (UseAVX > 2 &&
 3424       (!VM_Version::supports_avx512dq() || !VM_Version::supports_avx512vl()) &&
 3425       (dst->encoding() >= 16)) {
 3426     vpxor(dst, dst, src, Assembler::AVX_512bit, rscratch);
 3427   } else if (reachable(src)) {
 3428     Assembler::xorps(dst, as_Address(src));
 3429   } else {
 3430     lea(rscratch, src);
 3431     Assembler::xorps(dst, Address(rscratch, 0));
 3432   }
 3433 }
 3434 
 3435 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src, Register rscratch) {
 3436   assert(rscratch != noreg || always_reachable(src), "missing");
 3437 
 3438   // Used in sign-bit flipping with aligned address.
 3439   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
 3440   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
 3441   if (reachable(src)) {
 3442     Assembler::pshufb(dst, as_Address(src));
 3443   } else {
 3444     lea(rscratch, src);
 3445     Assembler::pshufb(dst, Address(rscratch, 0));
 3446   }
 3447 }
 3448 
 3449 // AVX 3-operands instructions
 3450 
 3451 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3452   assert(rscratch != noreg || always_reachable(src), "missing");
 3453 
 3454   if (reachable(src)) {
 3455     vaddsd(dst, nds, as_Address(src));
 3456   } else {
 3457     lea(rscratch, src);
 3458     vaddsd(dst, nds, Address(rscratch, 0));
 3459   }
 3460 }
 3461 
 3462 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3463   assert(rscratch != noreg || always_reachable(src), "missing");
 3464 
 3465   if (reachable(src)) {
 3466     vaddss(dst, nds, as_Address(src));
 3467   } else {
 3468     lea(rscratch, src);
 3469     vaddss(dst, nds, Address(rscratch, 0));
 3470   }
 3471 }
 3472 
 3473 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3474   assert(UseAVX > 0, "requires some form of AVX");
 3475   assert(rscratch != noreg || always_reachable(src), "missing");
 3476 
 3477   if (reachable(src)) {
 3478     Assembler::vpaddb(dst, nds, as_Address(src), vector_len);
 3479   } else {
 3480     lea(rscratch, src);
 3481     Assembler::vpaddb(dst, nds, Address(rscratch, 0), vector_len);
 3482   }
 3483 }
 3484 
 3485 void MacroAssembler::vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3486   assert(UseAVX > 0, "requires some form of AVX");
 3487   assert(rscratch != noreg || always_reachable(src), "missing");
 3488 
 3489   if (reachable(src)) {
 3490     Assembler::vpaddd(dst, nds, as_Address(src), vector_len);
 3491   } else {
 3492     lea(rscratch, src);
 3493     Assembler::vpaddd(dst, nds, Address(rscratch, 0), vector_len);
 3494   }
 3495 }
 3496 
 3497 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3498   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3499   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3500 
 3501   vandps(dst, nds, negate_field, vector_len, rscratch);
 3502 }
 3503 
 3504 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch) {
 3505   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3506   assert(rscratch != noreg || always_reachable(negate_field), "missing");
 3507 
 3508   vandpd(dst, nds, negate_field, vector_len, rscratch);
 3509 }
 3510 
 3511 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3512   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3513   Assembler::vpaddb(dst, nds, src, vector_len);
 3514 }
 3515 
 3516 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3517   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3518   Assembler::vpaddb(dst, nds, src, vector_len);
 3519 }
 3520 
 3521 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3522   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3523   Assembler::vpaddw(dst, nds, src, vector_len);
 3524 }
 3525 
 3526 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3527   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3528   Assembler::vpaddw(dst, nds, src, vector_len);
 3529 }
 3530 
 3531 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3532   assert(rscratch != noreg || always_reachable(src), "missing");
 3533 
 3534   if (reachable(src)) {
 3535     Assembler::vpand(dst, nds, as_Address(src), vector_len);
 3536   } else {
 3537     lea(rscratch, src);
 3538     Assembler::vpand(dst, nds, Address(rscratch, 0), vector_len);
 3539   }
 3540 }
 3541 
 3542 void MacroAssembler::vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3543   assert(rscratch != noreg || always_reachable(src), "missing");
 3544 
 3545   if (reachable(src)) {
 3546     Assembler::vpbroadcastd(dst, as_Address(src), vector_len);
 3547   } else {
 3548     lea(rscratch, src);
 3549     Assembler::vpbroadcastd(dst, Address(rscratch, 0), vector_len);
 3550   }
 3551 }
 3552 
 3553 void MacroAssembler::vbroadcasti128(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3554   assert(rscratch != noreg || always_reachable(src), "missing");
 3555 
 3556   if (reachable(src)) {
 3557     Assembler::vbroadcasti128(dst, as_Address(src), vector_len);
 3558   } else {
 3559     lea(rscratch, src);
 3560     Assembler::vbroadcasti128(dst, Address(rscratch, 0), vector_len);
 3561   }
 3562 }
 3563 
 3564 void MacroAssembler::vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3565   assert(rscratch != noreg || always_reachable(src), "missing");
 3566 
 3567   if (reachable(src)) {
 3568     Assembler::vpbroadcastq(dst, as_Address(src), vector_len);
 3569   } else {
 3570     lea(rscratch, src);
 3571     Assembler::vpbroadcastq(dst, Address(rscratch, 0), vector_len);
 3572   }
 3573 }
 3574 
 3575 void MacroAssembler::vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3576   assert(rscratch != noreg || always_reachable(src), "missing");
 3577 
 3578   if (reachable(src)) {
 3579     Assembler::vbroadcastsd(dst, as_Address(src), vector_len);
 3580   } else {
 3581     lea(rscratch, src);
 3582     Assembler::vbroadcastsd(dst, Address(rscratch, 0), vector_len);
 3583   }
 3584 }
 3585 
 3586 void MacroAssembler::vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
 3587   assert(rscratch != noreg || always_reachable(src), "missing");
 3588 
 3589   if (reachable(src)) {
 3590     Assembler::vbroadcastss(dst, as_Address(src), vector_len);
 3591   } else {
 3592     lea(rscratch, src);
 3593     Assembler::vbroadcastss(dst, Address(rscratch, 0), vector_len);
 3594   }
 3595 }
 3596 
 3597 // Vector float blend
 3598 // vblendvps(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3599 void MacroAssembler::vblendvps(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3600   // WARN: Allow dst == (src1|src2), mask == scratch
 3601   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3602   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst;
 3603   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3604   if (blend_emulation && scratch_available && dst_available) {
 3605     if (compute_mask) {
 3606       vpsrad(scratch, mask, 32, vector_len);
 3607       mask = scratch;
 3608     }
 3609     if (dst == src1) {
 3610       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src1
 3611       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3612     } else {
 3613       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3614       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src1
 3615     }
 3616     vpor(dst, dst, scratch, vector_len);
 3617   } else {
 3618     Assembler::vblendvps(dst, src1, src2, mask, vector_len);
 3619   }
 3620 }
 3621 
 3622 // vblendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister mask, int vector_len, bool compute_mask = true, XMMRegister scratch = xnoreg)
 3623 void MacroAssembler::vblendvpd(XMMRegister dst, XMMRegister src1, XMMRegister src2, XMMRegister mask, int vector_len, bool compute_mask, XMMRegister scratch) {
 3624   // WARN: Allow dst == (src1|src2), mask == scratch
 3625   bool blend_emulation = EnableX86ECoreOpts && UseAVX > 1;
 3626   bool scratch_available = scratch != xnoreg && scratch != src1 && scratch != src2 && scratch != dst && (!compute_mask || scratch != mask);
 3627   bool dst_available = dst != mask && (dst != src1 || dst != src2);
 3628   if (blend_emulation && scratch_available && dst_available) {
 3629     if (compute_mask) {
 3630       vpxor(scratch, scratch, scratch, vector_len);
 3631       vpcmpgtq(scratch, scratch, mask, vector_len);
 3632       mask = scratch;
 3633     }
 3634     if (dst == src1) {
 3635       vpandn(dst,     mask, src1, vector_len); // if mask == 0, src
 3636       vpand (scratch, mask, src2, vector_len); // if mask == 1, src2
 3637     } else {
 3638       vpand (dst,     mask, src2, vector_len); // if mask == 1, src2
 3639       vpandn(scratch, mask, src1, vector_len); // if mask == 0, src
 3640     }
 3641     vpor(dst, dst, scratch, vector_len);
 3642   } else {
 3643     Assembler::vblendvpd(dst, src1, src2, mask, vector_len);
 3644   }
 3645 }
 3646 
 3647 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3648   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3649   Assembler::vpcmpeqb(dst, nds, src, vector_len);
 3650 }
 3651 
 3652 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
 3653   assert(((dst->encoding() < 16 && src1->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3654   Assembler::vpcmpeqb(dst, src1, src2, vector_len);
 3655 }
 3656 
 3657 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3658   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3659   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3660 }
 3661 
 3662 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3663   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3664   Assembler::vpcmpeqw(dst, nds, src, vector_len);
 3665 }
 3666 
 3667 void MacroAssembler::evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3668   assert(rscratch != noreg || always_reachable(src), "missing");
 3669 
 3670   if (reachable(src)) {
 3671     Assembler::evpcmpeqd(kdst, mask, nds, as_Address(src), vector_len);
 3672   } else {
 3673     lea(rscratch, src);
 3674     Assembler::evpcmpeqd(kdst, mask, nds, Address(rscratch, 0), vector_len);
 3675   }
 3676 }
 3677 
 3678 void MacroAssembler::evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3679                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3680   assert(rscratch != noreg || always_reachable(src), "missing");
 3681 
 3682   if (reachable(src)) {
 3683     Assembler::evpcmpd(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3684   } else {
 3685     lea(rscratch, src);
 3686     Assembler::evpcmpd(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3687   }
 3688 }
 3689 
 3690 void MacroAssembler::evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3691                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3692   assert(rscratch != noreg || always_reachable(src), "missing");
 3693 
 3694   if (reachable(src)) {
 3695     Assembler::evpcmpq(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3696   } else {
 3697     lea(rscratch, src);
 3698     Assembler::evpcmpq(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3699   }
 3700 }
 3701 
 3702 void MacroAssembler::evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3703                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3704   assert(rscratch != noreg || always_reachable(src), "missing");
 3705 
 3706   if (reachable(src)) {
 3707     Assembler::evpcmpb(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3708   } else {
 3709     lea(rscratch, src);
 3710     Assembler::evpcmpb(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3711   }
 3712 }
 3713 
 3714 void MacroAssembler::evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3715                              int comparison, bool is_signed, int vector_len, Register rscratch) {
 3716   assert(rscratch != noreg || always_reachable(src), "missing");
 3717 
 3718   if (reachable(src)) {
 3719     Assembler::evpcmpw(kdst, mask, nds, as_Address(src), comparison, is_signed, vector_len);
 3720   } else {
 3721     lea(rscratch, src);
 3722     Assembler::evpcmpw(kdst, mask, nds, Address(rscratch, 0), comparison, is_signed, vector_len);
 3723   }
 3724 }
 3725 
 3726 void MacroAssembler::vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len) {
 3727   if (width == Assembler::Q) {
 3728     Assembler::vpcmpCCq(dst, nds, src, cond_encoding, vector_len);
 3729   } else {
 3730     Assembler::vpcmpCCbwd(dst, nds, src, cond_encoding, vector_len);
 3731   }
 3732 }
 3733 
 3734 void MacroAssembler::vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len) {
 3735   int eq_cond_enc = 0x29;
 3736   int gt_cond_enc = 0x37;
 3737   if (width != Assembler::Q) {
 3738     eq_cond_enc = 0x74 + width;
 3739     gt_cond_enc = 0x64 + width;
 3740   }
 3741   switch (cond) {
 3742   case eq:
 3743     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3744     break;
 3745   case neq:
 3746     vpcmpCC(dst, nds, src, eq_cond_enc, width, vector_len);
 3747     vallones(xtmp, vector_len);
 3748     vpxor(dst, xtmp, dst, vector_len);
 3749     break;
 3750   case le:
 3751     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3752     vallones(xtmp, vector_len);
 3753     vpxor(dst, xtmp, dst, vector_len);
 3754     break;
 3755   case nlt:
 3756     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3757     vallones(xtmp, vector_len);
 3758     vpxor(dst, xtmp, dst, vector_len);
 3759     break;
 3760   case lt:
 3761     vpcmpCC(dst, src, nds, gt_cond_enc, width, vector_len);
 3762     break;
 3763   case nle:
 3764     vpcmpCC(dst, nds, src, gt_cond_enc, width, vector_len);
 3765     break;
 3766   default:
 3767     assert(false, "Should not reach here");
 3768   }
 3769 }
 3770 
 3771 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
 3772   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3773   Assembler::vpmovzxbw(dst, src, vector_len);
 3774 }
 3775 
 3776 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src, int vector_len) {
 3777   assert((src->encoding() < 16),"XMM register should be 0-15");
 3778   Assembler::vpmovmskb(dst, src, vector_len);
 3779 }
 3780 
 3781 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3782   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3783   Assembler::vpmullw(dst, nds, src, vector_len);
 3784 }
 3785 
 3786 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3787   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3788   Assembler::vpmullw(dst, nds, src, vector_len);
 3789 }
 3790 
 3791 void MacroAssembler::vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3792   assert((UseAVX > 0), "AVX support is needed");
 3793   assert(rscratch != noreg || always_reachable(src), "missing");
 3794 
 3795   if (reachable(src)) {
 3796     Assembler::vpmulld(dst, nds, as_Address(src), vector_len);
 3797   } else {
 3798     lea(rscratch, src);
 3799     Assembler::vpmulld(dst, nds, Address(rscratch, 0), vector_len);
 3800   }
 3801 }
 3802 
 3803 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3804   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3805   Assembler::vpsubb(dst, nds, src, vector_len);
 3806 }
 3807 
 3808 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3809   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3810   Assembler::vpsubb(dst, nds, src, vector_len);
 3811 }
 3812 
 3813 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
 3814   assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3815   Assembler::vpsubw(dst, nds, src, vector_len);
 3816 }
 3817 
 3818 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
 3819   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3820   Assembler::vpsubw(dst, nds, src, vector_len);
 3821 }
 3822 
 3823 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3824   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3825   Assembler::vpsraw(dst, nds, shift, vector_len);
 3826 }
 3827 
 3828 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3829   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3830   Assembler::vpsraw(dst, nds, shift, vector_len);
 3831 }
 3832 
 3833 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3834   assert(UseAVX > 2,"");
 3835   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3836      vector_len = 2;
 3837   }
 3838   Assembler::evpsraq(dst, nds, shift, vector_len);
 3839 }
 3840 
 3841 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3842   assert(UseAVX > 2,"");
 3843   if (!VM_Version::supports_avx512vl() && vector_len < 2) {
 3844      vector_len = 2;
 3845   }
 3846   Assembler::evpsraq(dst, nds, shift, vector_len);
 3847 }
 3848 
 3849 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3850   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3851   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3852 }
 3853 
 3854 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3855   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3856   Assembler::vpsrlw(dst, nds, shift, vector_len);
 3857 }
 3858 
 3859 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
 3860   assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3861   Assembler::vpsllw(dst, nds, shift, vector_len);
 3862 }
 3863 
 3864 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
 3865   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3866   Assembler::vpsllw(dst, nds, shift, vector_len);
 3867 }
 3868 
 3869 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
 3870   assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
 3871   Assembler::vptest(dst, src);
 3872 }
 3873 
 3874 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 3875   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3876   Assembler::punpcklbw(dst, src);
 3877 }
 3878 
 3879 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
 3880   assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
 3881   Assembler::pshufd(dst, src, mode);
 3882 }
 3883 
 3884 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 3885   assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
 3886   Assembler::pshuflw(dst, src, mode);
 3887 }
 3888 
 3889 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3890   assert(rscratch != noreg || always_reachable(src), "missing");
 3891 
 3892   if (reachable(src)) {
 3893     vandpd(dst, nds, as_Address(src), vector_len);
 3894   } else {
 3895     lea(rscratch, src);
 3896     vandpd(dst, nds, Address(rscratch, 0), vector_len);
 3897   }
 3898 }
 3899 
 3900 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 3901   assert(rscratch != noreg || always_reachable(src), "missing");
 3902 
 3903   if (reachable(src)) {
 3904     vandps(dst, nds, as_Address(src), vector_len);
 3905   } else {
 3906     lea(rscratch, src);
 3907     vandps(dst, nds, Address(rscratch, 0), vector_len);
 3908   }
 3909 }
 3910 
 3911 void MacroAssembler::evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src,
 3912                             bool merge, int vector_len, Register rscratch) {
 3913   assert(rscratch != noreg || always_reachable(src), "missing");
 3914 
 3915   if (reachable(src)) {
 3916     Assembler::evpord(dst, mask, nds, as_Address(src), merge, vector_len);
 3917   } else {
 3918     lea(rscratch, src);
 3919     Assembler::evpord(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
 3920   }
 3921 }
 3922 
 3923 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3924   assert(rscratch != noreg || always_reachable(src), "missing");
 3925 
 3926   if (reachable(src)) {
 3927     vdivsd(dst, nds, as_Address(src));
 3928   } else {
 3929     lea(rscratch, src);
 3930     vdivsd(dst, nds, Address(rscratch, 0));
 3931   }
 3932 }
 3933 
 3934 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3935   assert(rscratch != noreg || always_reachable(src), "missing");
 3936 
 3937   if (reachable(src)) {
 3938     vdivss(dst, nds, as_Address(src));
 3939   } else {
 3940     lea(rscratch, src);
 3941     vdivss(dst, nds, Address(rscratch, 0));
 3942   }
 3943 }
 3944 
 3945 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3946   assert(rscratch != noreg || always_reachable(src), "missing");
 3947 
 3948   if (reachable(src)) {
 3949     vmulsd(dst, nds, as_Address(src));
 3950   } else {
 3951     lea(rscratch, src);
 3952     vmulsd(dst, nds, Address(rscratch, 0));
 3953   }
 3954 }
 3955 
 3956 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3957   assert(rscratch != noreg || always_reachable(src), "missing");
 3958 
 3959   if (reachable(src)) {
 3960     vmulss(dst, nds, as_Address(src));
 3961   } else {
 3962     lea(rscratch, src);
 3963     vmulss(dst, nds, Address(rscratch, 0));
 3964   }
 3965 }
 3966 
 3967 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3968   assert(rscratch != noreg || always_reachable(src), "missing");
 3969 
 3970   if (reachable(src)) {
 3971     vsubsd(dst, nds, as_Address(src));
 3972   } else {
 3973     lea(rscratch, src);
 3974     vsubsd(dst, nds, Address(rscratch, 0));
 3975   }
 3976 }
 3977 
 3978 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3979   assert(rscratch != noreg || always_reachable(src), "missing");
 3980 
 3981   if (reachable(src)) {
 3982     vsubss(dst, nds, as_Address(src));
 3983   } else {
 3984     lea(rscratch, src);
 3985     vsubss(dst, nds, Address(rscratch, 0));
 3986   }
 3987 }
 3988 
 3989 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3990   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3991   assert(rscratch != noreg || always_reachable(src), "missing");
 3992 
 3993   vxorps(dst, nds, src, Assembler::AVX_128bit, rscratch);
 3994 }
 3995 
 3996 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch) {
 3997   assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
 3998   assert(rscratch != noreg || always_reachable(src), "missing");
 3999 
 4000   vxorpd(dst, nds, src, Assembler::AVX_128bit, rscratch);
 4001 }
 4002 
 4003 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4004   assert(rscratch != noreg || always_reachable(src), "missing");
 4005 
 4006   if (reachable(src)) {
 4007     vxorpd(dst, nds, as_Address(src), vector_len);
 4008   } else {
 4009     lea(rscratch, src);
 4010     vxorpd(dst, nds, Address(rscratch, 0), vector_len);
 4011   }
 4012 }
 4013 
 4014 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4015   assert(rscratch != noreg || always_reachable(src), "missing");
 4016 
 4017   if (reachable(src)) {
 4018     vxorps(dst, nds, as_Address(src), vector_len);
 4019   } else {
 4020     lea(rscratch, src);
 4021     vxorps(dst, nds, Address(rscratch, 0), vector_len);
 4022   }
 4023 }
 4024 
 4025 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4026   assert(rscratch != noreg || always_reachable(src), "missing");
 4027 
 4028   if (UseAVX > 1 || (vector_len < 1)) {
 4029     if (reachable(src)) {
 4030       Assembler::vpxor(dst, nds, as_Address(src), vector_len);
 4031     } else {
 4032       lea(rscratch, src);
 4033       Assembler::vpxor(dst, nds, Address(rscratch, 0), vector_len);
 4034     }
 4035   } else {
 4036     MacroAssembler::vxorpd(dst, nds, src, vector_len, rscratch);
 4037   }
 4038 }
 4039 
 4040 void MacroAssembler::vpermd(XMMRegister dst,  XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
 4041   assert(rscratch != noreg || always_reachable(src), "missing");
 4042 
 4043   if (reachable(src)) {
 4044     Assembler::vpermd(dst, nds, as_Address(src), vector_len);
 4045   } else {
 4046     lea(rscratch, src);
 4047     Assembler::vpermd(dst, nds, Address(rscratch, 0), vector_len);
 4048   }
 4049 }
 4050 
 4051 void MacroAssembler::clear_jobject_tag(Register possibly_non_local) {
 4052   const int32_t inverted_mask = ~static_cast<int32_t>(JNIHandles::tag_mask);
 4053   STATIC_ASSERT(inverted_mask == -4); // otherwise check this code
 4054   // The inverted mask is sign-extended
 4055   andptr(possibly_non_local, inverted_mask);
 4056 }
 4057 
 4058 void MacroAssembler::resolve_jobject(Register value,
 4059                                      Register tmp) {
 4060   Register thread = r15_thread;
 4061   assert_different_registers(value, thread, tmp);
 4062   Label done, tagged, weak_tagged;
 4063   testptr(value, value);
 4064   jcc(Assembler::zero, done);           // Use null as-is.
 4065   testptr(value, JNIHandles::tag_mask); // Test for tag.
 4066   jcc(Assembler::notZero, tagged);
 4067 
 4068   // Resolve local handle
 4069   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp, thread);
 4070   verify_oop(value);
 4071   jmp(done);
 4072 
 4073   bind(tagged);
 4074   testptr(value, JNIHandles::TypeTag::weak_global); // Test for weak tag.
 4075   jcc(Assembler::notZero, weak_tagged);
 4076 
 4077   // Resolve global handle
 4078   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4079   verify_oop(value);
 4080   jmp(done);
 4081 
 4082   bind(weak_tagged);
 4083   // Resolve jweak.
 4084   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 4085                  value, Address(value, -JNIHandles::TypeTag::weak_global), tmp, thread);
 4086   verify_oop(value);
 4087 
 4088   bind(done);
 4089 }
 4090 
 4091 void MacroAssembler::resolve_global_jobject(Register value,
 4092                                             Register tmp) {
 4093   Register thread = r15_thread;
 4094   assert_different_registers(value, thread, tmp);
 4095   Label done;
 4096 
 4097   testptr(value, value);
 4098   jcc(Assembler::zero, done);           // Use null as-is.
 4099 
 4100 #ifdef ASSERT
 4101   {
 4102     Label valid_global_tag;
 4103     testptr(value, JNIHandles::TypeTag::global); // Test for global tag.
 4104     jcc(Assembler::notZero, valid_global_tag);
 4105     stop("non global jobject using resolve_global_jobject");
 4106     bind(valid_global_tag);
 4107   }
 4108 #endif
 4109 
 4110   // Resolve global handle
 4111   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, -JNIHandles::TypeTag::global), tmp, thread);
 4112   verify_oop(value);
 4113 
 4114   bind(done);
 4115 }
 4116 
 4117 void MacroAssembler::subptr(Register dst, int32_t imm32) {
 4118   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
 4119 }
 4120 
 4121 // Force generation of a 4 byte immediate value even if it fits into 8bit
 4122 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
 4123   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
 4124 }
 4125 
 4126 void MacroAssembler::subptr(Register dst, Register src) {
 4127   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
 4128 }
 4129 
 4130 // C++ bool manipulation
 4131 void MacroAssembler::testbool(Register dst) {
 4132   if(sizeof(bool) == 1)
 4133     testb(dst, 0xff);
 4134   else if(sizeof(bool) == 2) {
 4135     // testw implementation needed for two byte bools
 4136     ShouldNotReachHere();
 4137   } else if(sizeof(bool) == 4)
 4138     testl(dst, dst);
 4139   else
 4140     // unsupported
 4141     ShouldNotReachHere();
 4142 }
 4143 
 4144 void MacroAssembler::testptr(Register dst, Register src) {
 4145   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 4146 }
 4147 
 4148 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 4149 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 4150                                    Register var_size_in_bytes,
 4151                                    int con_size_in_bytes,
 4152                                    Register t1,
 4153                                    Register t2,
 4154                                    Label& slow_case) {
 4155   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4156   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 4157 }
 4158 
 4159 RegSet MacroAssembler::call_clobbered_gp_registers() {
 4160   RegSet regs;
 4161 #ifdef _LP64
 4162   regs += RegSet::of(rax, rcx, rdx);
 4163 #ifndef _WINDOWS
 4164   regs += RegSet::of(rsi, rdi);
 4165 #endif
 4166   regs += RegSet::range(r8, r11);
 4167 #else
 4168   regs += RegSet::of(rax, rcx, rdx);
 4169 #endif
 4170 #ifdef _LP64
 4171   if (UseAPX) {
 4172     regs += RegSet::range(r16, as_Register(Register::number_of_registers - 1));
 4173   }
 4174 #endif
 4175   return regs;
 4176 }
 4177 
 4178 XMMRegSet MacroAssembler::call_clobbered_xmm_registers() {
 4179   int num_xmm_registers = XMMRegister::available_xmm_registers();
 4180 #if defined(_WINDOWS)
 4181   XMMRegSet result = XMMRegSet::range(xmm0, xmm5);
 4182   if (num_xmm_registers > 16) {
 4183      result += XMMRegSet::range(xmm16, as_XMMRegister(num_xmm_registers - 1));
 4184   }
 4185   return result;
 4186 #else
 4187   return XMMRegSet::range(xmm0, as_XMMRegister(num_xmm_registers - 1));
 4188 #endif
 4189 }
 4190 
 4191 static int FPUSaveAreaSize = align_up(108, StackAlignmentInBytes); // 108 bytes needed for FPU state by fsave/frstor
 4192 
 4193 #ifndef _LP64
 4194 static bool use_x87_registers() { return UseSSE < 2; }
 4195 #endif
 4196 static bool use_xmm_registers() { return UseSSE >= 1; }
 4197 
 4198 // C1 only ever uses the first double/float of the XMM register.
 4199 static int xmm_save_size() { return UseSSE >= 2 ? sizeof(double) : sizeof(float); }
 4200 
 4201 static void save_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4202   if (UseSSE == 1) {
 4203     masm->movflt(Address(rsp, offset), reg);
 4204   } else {
 4205     masm->movdbl(Address(rsp, offset), reg);
 4206   }
 4207 }
 4208 
 4209 static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister reg) {
 4210   if (UseSSE == 1) {
 4211     masm->movflt(reg, Address(rsp, offset));
 4212   } else {
 4213     masm->movdbl(reg, Address(rsp, offset));
 4214   }
 4215 }
 4216 
 4217 static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers,
 4218                                   bool save_fpu, int& gp_area_size,
 4219                                   int& fp_area_size, int& xmm_area_size) {
 4220 
 4221   gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
 4222                          StackAlignmentInBytes);
 4223 #ifdef _LP64
 4224   fp_area_size = 0;
 4225 #else
 4226   fp_area_size = (save_fpu && use_x87_registers()) ? FPUSaveAreaSize : 0;
 4227 #endif
 4228   xmm_area_size = (save_fpu && use_xmm_registers()) ? xmm_registers.size() * xmm_save_size() : 0;
 4229 
 4230   return gp_area_size + fp_area_size + xmm_area_size;
 4231 }
 4232 
 4233 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude, bool save_fpu) {
 4234   block_comment("push_call_clobbered_registers start");
 4235   // Regular registers
 4236   RegSet gp_registers_to_push = call_clobbered_gp_registers() - exclude;
 4237 
 4238   int gp_area_size;
 4239   int fp_area_size;
 4240   int xmm_area_size;
 4241   int total_save_size = register_section_sizes(gp_registers_to_push, call_clobbered_xmm_registers(), save_fpu,
 4242                                                gp_area_size, fp_area_size, xmm_area_size);
 4243   subptr(rsp, total_save_size);
 4244 
 4245   push_set(gp_registers_to_push, 0);
 4246 
 4247 #ifndef _LP64
 4248   if (save_fpu && use_x87_registers()) {
 4249     fnsave(Address(rsp, gp_area_size));
 4250     fwait();
 4251   }
 4252 #endif
 4253   if (save_fpu && use_xmm_registers()) {
 4254     push_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4255   }
 4256 
 4257   block_comment("push_call_clobbered_registers end");
 4258 }
 4259 
 4260 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude, bool restore_fpu) {
 4261   block_comment("pop_call_clobbered_registers start");
 4262 
 4263   RegSet gp_registers_to_pop = call_clobbered_gp_registers() - exclude;
 4264 
 4265   int gp_area_size;
 4266   int fp_area_size;
 4267   int xmm_area_size;
 4268   int total_save_size = register_section_sizes(gp_registers_to_pop, call_clobbered_xmm_registers(), restore_fpu,
 4269                                                gp_area_size, fp_area_size, xmm_area_size);
 4270 
 4271   if (restore_fpu && use_xmm_registers()) {
 4272     pop_set(call_clobbered_xmm_registers(), gp_area_size + fp_area_size);
 4273   }
 4274 #ifndef _LP64
 4275   if (restore_fpu && use_x87_registers()) {
 4276     frstor(Address(rsp, gp_area_size));
 4277   }
 4278 #endif
 4279 
 4280   pop_set(gp_registers_to_pop, 0);
 4281 
 4282   addptr(rsp, total_save_size);
 4283 
 4284   vzeroupper();
 4285 
 4286   block_comment("pop_call_clobbered_registers end");
 4287 }
 4288 
 4289 void MacroAssembler::push_set(XMMRegSet set, int offset) {
 4290   assert(is_aligned(set.size() * xmm_save_size(), StackAlignmentInBytes), "must be");
 4291   int spill_offset = offset;
 4292 
 4293   for (RegSetIterator<XMMRegister> it = set.begin(); *it != xnoreg; ++it) {
 4294     save_xmm_register(this, spill_offset, *it);
 4295     spill_offset += xmm_save_size();
 4296   }
 4297 }
 4298 
 4299 void MacroAssembler::pop_set(XMMRegSet set, int offset) {
 4300   int restore_size = set.size() * xmm_save_size();
 4301   assert(is_aligned(restore_size, StackAlignmentInBytes), "must be");
 4302 
 4303   int restore_offset = offset + restore_size - xmm_save_size();
 4304 
 4305   for (ReverseRegSetIterator<XMMRegister> it = set.rbegin(); *it != xnoreg; ++it) {
 4306     restore_xmm_register(this, restore_offset, *it);
 4307     restore_offset -= xmm_save_size();
 4308   }
 4309 }
 4310 
 4311 void MacroAssembler::push_set(RegSet set, int offset) {
 4312   int spill_offset;
 4313   if (offset == -1) {
 4314     int register_push_size = set.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4315     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 4316     subptr(rsp, aligned_size);
 4317     spill_offset = 0;
 4318   } else {
 4319     spill_offset = offset;
 4320   }
 4321 
 4322   for (RegSetIterator<Register> it = set.begin(); *it != noreg; ++it) {
 4323     movptr(Address(rsp, spill_offset), *it);
 4324     spill_offset += Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4325   }
 4326 }
 4327 
 4328 void MacroAssembler::pop_set(RegSet set, int offset) {
 4329 
 4330   int gp_reg_size = Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4331   int restore_size = set.size() * gp_reg_size;
 4332   int aligned_size = align_up(restore_size, StackAlignmentInBytes);
 4333 
 4334   int restore_offset;
 4335   if (offset == -1) {
 4336     restore_offset = restore_size - gp_reg_size;
 4337   } else {
 4338     restore_offset = offset + restore_size - gp_reg_size;
 4339   }
 4340   for (ReverseRegSetIterator<Register> it = set.rbegin(); *it != noreg; ++it) {
 4341     movptr(*it, Address(rsp, restore_offset));
 4342     restore_offset -= gp_reg_size;
 4343   }
 4344 
 4345   if (offset == -1) {
 4346     addptr(rsp, aligned_size);
 4347   }
 4348 }
 4349 
 4350 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
 4351 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
 4352   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
 4353   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
 4354   Label done;
 4355 
 4356   testptr(length_in_bytes, length_in_bytes);
 4357   jcc(Assembler::zero, done);
 4358 
 4359   // initialize topmost word, divide index by 2, check if odd and test if zero
 4360   // note: for the remaining code to work, index must be a multiple of BytesPerWord
 4361 #ifdef ASSERT
 4362   {
 4363     Label L;
 4364     testptr(length_in_bytes, BytesPerWord - 1);
 4365     jcc(Assembler::zero, L);
 4366     stop("length must be a multiple of BytesPerWord");
 4367     bind(L);
 4368   }
 4369 #endif
 4370   Register index = length_in_bytes;
 4371   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
 4372   if (UseIncDec) {
 4373     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
 4374   } else {
 4375     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
 4376     shrptr(index, 1);
 4377   }
 4378 #ifndef _LP64
 4379   // index could have not been a multiple of 8 (i.e., bit 2 was set)
 4380   {
 4381     Label even;
 4382     // note: if index was a multiple of 8, then it cannot
 4383     //       be 0 now otherwise it must have been 0 before
 4384     //       => if it is even, we don't need to check for 0 again
 4385     jcc(Assembler::carryClear, even);
 4386     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4387     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4388     // index could be 0 now, must check again
 4389     jcc(Assembler::zero, done);
 4390     bind(even);
 4391   }
 4392 #endif // !_LP64
 4393   // initialize remaining object fields: index is a multiple of 2 now
 4394   {
 4395     Label loop;
 4396     bind(loop);
 4397     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4398     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4399     decrement(index);
 4400     jcc(Assembler::notZero, loop);
 4401   }
 4402 
 4403   bind(done);
 4404 }
 4405 
 4406 // Look up the method for a megamorphic invokeinterface call.
 4407 // The target method is determined by <intf_klass, itable_index>.
 4408 // The receiver klass is in recv_klass.
 4409 // On success, the result will be in method_result, and execution falls through.
 4410 // On failure, execution transfers to the given label.
 4411 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4412                                              Register intf_klass,
 4413                                              RegisterOrConstant itable_index,
 4414                                              Register method_result,
 4415                                              Register scan_temp,
 4416                                              Label& L_no_such_interface,
 4417                                              bool return_method) {
 4418   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4419   assert_different_registers(method_result, intf_klass, scan_temp);
 4420   assert(recv_klass != method_result || !return_method,
 4421          "recv_klass can be destroyed when method isn't needed");
 4422 
 4423   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4424          "caller must use same register for non-constant itable index as for method");
 4425 
 4426   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
 4427   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4428   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4429   int scan_step   = itableOffsetEntry::size() * wordSize;
 4430   int vte_size    = vtableEntry::size_in_bytes();
 4431   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4432   assert(vte_size == wordSize, "else adjust times_vte_scale");
 4433 
 4434   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4435 
 4436   // Could store the aligned, prescaled offset in the klass.
 4437   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
 4438 
 4439   if (return_method) {
 4440     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
 4441     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4442     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
 4443   }
 4444 
 4445   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
 4446   //   if (scan->interface() == intf) {
 4447   //     result = (klass + scan->offset() + itable_index);
 4448   //   }
 4449   // }
 4450   Label search, found_method;
 4451 
 4452   for (int peel = 1; peel >= 0; peel--) {
 4453     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset()));
 4454     cmpptr(intf_klass, method_result);
 4455 
 4456     if (peel) {
 4457       jccb(Assembler::equal, found_method);
 4458     } else {
 4459       jccb(Assembler::notEqual, search);
 4460       // (invert the test to fall through to found_method...)
 4461     }
 4462 
 4463     if (!peel)  break;
 4464 
 4465     bind(search);
 4466 
 4467     // Check that the previous entry is non-null.  A null entry means that
 4468     // the receiver class doesn't implement the interface, and wasn't the
 4469     // same as when the caller was compiled.
 4470     testptr(method_result, method_result);
 4471     jcc(Assembler::zero, L_no_such_interface);
 4472     addptr(scan_temp, scan_step);
 4473   }
 4474 
 4475   bind(found_method);
 4476 
 4477   if (return_method) {
 4478     // Got a hit.
 4479     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset()));
 4480     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
 4481   }
 4482 }
 4483 
 4484 // Look up the method for a megamorphic invokeinterface call in a single pass over itable:
 4485 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
 4486 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
 4487 // The target method is determined by <holder_klass, itable_index>.
 4488 // The receiver klass is in recv_klass.
 4489 // On success, the result will be in method_result, and execution falls through.
 4490 // On failure, execution transfers to the given label.
 4491 void MacroAssembler::lookup_interface_method_stub(Register recv_klass,
 4492                                                   Register holder_klass,
 4493                                                   Register resolved_klass,
 4494                                                   Register method_result,
 4495                                                   Register scan_temp,
 4496                                                   Register temp_reg2,
 4497                                                   Register receiver,
 4498                                                   int itable_index,
 4499                                                   Label& L_no_such_interface) {
 4500   assert_different_registers(recv_klass, method_result, holder_klass, resolved_klass, scan_temp, temp_reg2, receiver);
 4501   Register temp_itbl_klass = method_result;
 4502   Register temp_reg = (temp_reg2 == noreg ? recv_klass : temp_reg2); // reuse recv_klass register on 32-bit x86 impl
 4503 
 4504   int vtable_base = in_bytes(Klass::vtable_start_offset());
 4505   int itentry_off = in_bytes(itableMethodEntry::method_offset());
 4506   int scan_step = itableOffsetEntry::size() * wordSize;
 4507   int vte_size = vtableEntry::size_in_bytes();
 4508   int ioffset = in_bytes(itableOffsetEntry::interface_offset());
 4509   int ooffset = in_bytes(itableOffsetEntry::offset_offset());
 4510   Address::ScaleFactor times_vte_scale = Address::times_ptr;
 4511   assert(vte_size == wordSize, "adjust times_vte_scale");
 4512 
 4513   Label L_loop_scan_resolved_entry, L_resolved_found, L_holder_found;
 4514 
 4515   // temp_itbl_klass = recv_klass.itable[0]
 4516   // scan_temp = &recv_klass.itable[0] + step
 4517   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 4518   movptr(temp_itbl_klass, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset));
 4519   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base + ioffset + scan_step));
 4520   xorptr(temp_reg, temp_reg);
 4521 
 4522   // Initial checks:
 4523   //   - if (holder_klass != resolved_klass), go to "scan for resolved"
 4524   //   - if (itable[0] == 0), no such interface
 4525   //   - if (itable[0] == holder_klass), shortcut to "holder found"
 4526   cmpptr(holder_klass, resolved_klass);
 4527   jccb(Assembler::notEqual, L_loop_scan_resolved_entry);
 4528   testptr(temp_itbl_klass, temp_itbl_klass);
 4529   jccb(Assembler::zero, L_no_such_interface);
 4530   cmpptr(holder_klass, temp_itbl_klass);
 4531   jccb(Assembler::equal, L_holder_found);
 4532 
 4533   // Loop: Look for holder_klass record in itable
 4534   //   do {
 4535   //     tmp = itable[index];
 4536   //     index += step;
 4537   //     if (tmp == holder_klass) {
 4538   //       goto L_holder_found; // Found!
 4539   //     }
 4540   //   } while (tmp != 0);
 4541   //   goto L_no_such_interface // Not found.
 4542   Label L_scan_holder;
 4543   bind(L_scan_holder);
 4544     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4545     addptr(scan_temp, scan_step);
 4546     cmpptr(holder_klass, temp_itbl_klass);
 4547     jccb(Assembler::equal, L_holder_found);
 4548     testptr(temp_itbl_klass, temp_itbl_klass);
 4549     jccb(Assembler::notZero, L_scan_holder);
 4550 
 4551   jmpb(L_no_such_interface);
 4552 
 4553   // Loop: Look for resolved_class record in itable
 4554   //   do {
 4555   //     tmp = itable[index];
 4556   //     index += step;
 4557   //     if (tmp == holder_klass) {
 4558   //        // Also check if we have met a holder klass
 4559   //        holder_tmp = itable[index-step-ioffset];
 4560   //     }
 4561   //     if (tmp == resolved_klass) {
 4562   //        goto L_resolved_found;  // Found!
 4563   //     }
 4564   //   } while (tmp != 0);
 4565   //   goto L_no_such_interface // Not found.
 4566   //
 4567   Label L_loop_scan_resolved;
 4568   bind(L_loop_scan_resolved);
 4569     movptr(temp_itbl_klass, Address(scan_temp, 0));
 4570     addptr(scan_temp, scan_step);
 4571     bind(L_loop_scan_resolved_entry);
 4572     cmpptr(holder_klass, temp_itbl_klass);
 4573     cmovl(Assembler::equal, temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4574     cmpptr(resolved_klass, temp_itbl_klass);
 4575     jccb(Assembler::equal, L_resolved_found);
 4576     testptr(temp_itbl_klass, temp_itbl_klass);
 4577     jccb(Assembler::notZero, L_loop_scan_resolved);
 4578 
 4579   jmpb(L_no_such_interface);
 4580 
 4581   Label L_ready;
 4582 
 4583   // See if we already have a holder klass. If not, go and scan for it.
 4584   bind(L_resolved_found);
 4585   testptr(temp_reg, temp_reg);
 4586   jccb(Assembler::zero, L_scan_holder);
 4587   jmpb(L_ready);
 4588 
 4589   bind(L_holder_found);
 4590   movl(temp_reg, Address(scan_temp, ooffset - ioffset - scan_step));
 4591 
 4592   // Finally, temp_reg contains holder_klass vtable offset
 4593   bind(L_ready);
 4594   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 4595   if (temp_reg2 == noreg) { // recv_klass register is clobbered for 32-bit x86 impl
 4596     load_klass(scan_temp, receiver, noreg);
 4597     movptr(method_result, Address(scan_temp, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4598   } else {
 4599     movptr(method_result, Address(recv_klass, temp_reg, Address::times_1, itable_index * wordSize + itentry_off));
 4600   }
 4601 }
 4602 
 4603 
 4604 // virtual method calling
 4605 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 4606                                            RegisterOrConstant vtable_index,
 4607                                            Register method_result) {
 4608   const ByteSize base = Klass::vtable_start_offset();
 4609   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
 4610   Address vtable_entry_addr(recv_klass,
 4611                             vtable_index, Address::times_ptr,
 4612                             base + vtableEntry::method_offset());
 4613   movptr(method_result, vtable_entry_addr);
 4614 }
 4615 
 4616 
 4617 void MacroAssembler::check_klass_subtype(Register sub_klass,
 4618                            Register super_klass,
 4619                            Register temp_reg,
 4620                            Label& L_success) {
 4621   Label L_failure;
 4622   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, nullptr);
 4623   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, nullptr);
 4624   bind(L_failure);
 4625 }
 4626 
 4627 
 4628 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
 4629                                                    Register super_klass,
 4630                                                    Register temp_reg,
 4631                                                    Label* L_success,
 4632                                                    Label* L_failure,
 4633                                                    Label* L_slow_path,
 4634                                         RegisterOrConstant super_check_offset) {
 4635   assert_different_registers(sub_klass, super_klass, temp_reg);
 4636   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
 4637   if (super_check_offset.is_register()) {
 4638     assert_different_registers(sub_klass, super_klass,
 4639                                super_check_offset.as_register());
 4640   } else if (must_load_sco) {
 4641     assert(temp_reg != noreg, "supply either a temp or a register offset");
 4642   }
 4643 
 4644   Label L_fallthrough;
 4645   int label_nulls = 0;
 4646   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4647   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4648   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
 4649   assert(label_nulls <= 1, "at most one null in the batch");
 4650 
 4651   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4652   int sco_offset = in_bytes(Klass::super_check_offset_offset());
 4653   Address super_check_offset_addr(super_klass, sco_offset);
 4654 
 4655   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
 4656   // range of a jccb.  If this routine grows larger, reconsider at
 4657   // least some of these.
 4658 #define local_jcc(assembler_cond, label)                                \
 4659   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
 4660   else                             jcc( assembler_cond, label) /*omit semi*/
 4661 
 4662   // Hacked jmp, which may only be used just before L_fallthrough.
 4663 #define final_jmp(label)                                                \
 4664   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
 4665   else                            jmp(label)                /*omit semi*/
 4666 
 4667   // If the pointers are equal, we are done (e.g., String[] elements).
 4668   // This self-check enables sharing of secondary supertype arrays among
 4669   // non-primary types such as array-of-interface.  Otherwise, each such
 4670   // type would need its own customized SSA.
 4671   // We move this check to the front of the fast path because many
 4672   // type checks are in fact trivially successful in this manner,
 4673   // so we get a nicely predicted branch right at the start of the check.
 4674   cmpptr(sub_klass, super_klass);
 4675   local_jcc(Assembler::equal, *L_success);
 4676 
 4677   // Check the supertype display:
 4678   if (must_load_sco) {
 4679     // Positive movl does right thing on LP64.
 4680     movl(temp_reg, super_check_offset_addr);
 4681     super_check_offset = RegisterOrConstant(temp_reg);
 4682   }
 4683   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
 4684   cmpptr(super_klass, super_check_addr); // load displayed supertype
 4685 
 4686   // This check has worked decisively for primary supers.
 4687   // Secondary supers are sought in the super_cache ('super_cache_addr').
 4688   // (Secondary supers are interfaces and very deeply nested subtypes.)
 4689   // This works in the same check above because of a tricky aliasing
 4690   // between the super_cache and the primary super display elements.
 4691   // (The 'super_check_addr' can address either, as the case requires.)
 4692   // Note that the cache is updated below if it does not help us find
 4693   // what we need immediately.
 4694   // So if it was a primary super, we can just fail immediately.
 4695   // Otherwise, it's the slow path for us (no success at this point).
 4696 
 4697   if (super_check_offset.is_register()) {
 4698     local_jcc(Assembler::equal, *L_success);
 4699     cmpl(super_check_offset.as_register(), sc_offset);
 4700     if (L_failure == &L_fallthrough) {
 4701       local_jcc(Assembler::equal, *L_slow_path);
 4702     } else {
 4703       local_jcc(Assembler::notEqual, *L_failure);
 4704       final_jmp(*L_slow_path);
 4705     }
 4706   } else if (super_check_offset.as_constant() == sc_offset) {
 4707     // Need a slow path; fast failure is impossible.
 4708     if (L_slow_path == &L_fallthrough) {
 4709       local_jcc(Assembler::equal, *L_success);
 4710     } else {
 4711       local_jcc(Assembler::notEqual, *L_slow_path);
 4712       final_jmp(*L_success);
 4713     }
 4714   } else {
 4715     // No slow path; it's a fast decision.
 4716     if (L_failure == &L_fallthrough) {
 4717       local_jcc(Assembler::equal, *L_success);
 4718     } else {
 4719       local_jcc(Assembler::notEqual, *L_failure);
 4720       final_jmp(*L_success);
 4721     }
 4722   }
 4723 
 4724   bind(L_fallthrough);
 4725 
 4726 #undef local_jcc
 4727 #undef final_jmp
 4728 }
 4729 
 4730 
 4731 void MacroAssembler::check_klass_subtype_slow_path_linear(Register sub_klass,
 4732                                                           Register super_klass,
 4733                                                           Register temp_reg,
 4734                                                           Register temp2_reg,
 4735                                                           Label* L_success,
 4736                                                           Label* L_failure,
 4737                                                           bool set_cond_codes) {
 4738   assert_different_registers(sub_klass, super_klass, temp_reg);
 4739   if (temp2_reg != noreg)
 4740     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
 4741 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
 4742 
 4743   Label L_fallthrough;
 4744   int label_nulls = 0;
 4745   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4746   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4747   assert(label_nulls <= 1, "at most one null in the batch");
 4748 
 4749   // a couple of useful fields in sub_klass:
 4750   int ss_offset = in_bytes(Klass::secondary_supers_offset());
 4751   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
 4752   Address secondary_supers_addr(sub_klass, ss_offset);
 4753   Address super_cache_addr(     sub_klass, sc_offset);
 4754 
 4755   // Do a linear scan of the secondary super-klass chain.
 4756   // This code is rarely used, so simplicity is a virtue here.
 4757   // The repne_scan instruction uses fixed registers, which we must spill.
 4758   // Don't worry too much about pre-existing connections with the input regs.
 4759 
 4760   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
 4761   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
 4762 
 4763   // Get super_klass value into rax (even if it was in rdi or rcx).
 4764   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
 4765   if (super_klass != rax) {
 4766     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
 4767     mov(rax, super_klass);
 4768   }
 4769   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
 4770   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
 4771 
 4772 #ifndef PRODUCT
 4773   uint* pst_counter = &SharedRuntime::_partial_subtype_ctr;
 4774   ExternalAddress pst_counter_addr((address) pst_counter);
 4775   NOT_LP64(  incrementl(pst_counter_addr) );
 4776   LP64_ONLY( lea(rcx, pst_counter_addr) );
 4777   LP64_ONLY( incrementl(Address(rcx, 0)) );
 4778 #endif //PRODUCT
 4779 
 4780   // We will consult the secondary-super array.
 4781   movptr(rdi, secondary_supers_addr);
 4782   // Load the array length.  (Positive movl does right thing on LP64.)
 4783   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
 4784   // Skip to start of data.
 4785   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
 4786 
 4787   // Scan RCX words at [RDI] for an occurrence of RAX.
 4788   // Set NZ/Z based on last compare.
 4789   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
 4790   // not change flags (only scas instruction which is repeated sets flags).
 4791   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
 4792 
 4793     testptr(rax,rax); // Set Z = 0
 4794     repne_scan();
 4795 
 4796   // Unspill the temp. registers:
 4797   if (pushed_rdi)  pop(rdi);
 4798   if (pushed_rcx)  pop(rcx);
 4799   if (pushed_rax)  pop(rax);
 4800 
 4801   if (set_cond_codes) {
 4802     // Special hack for the AD files:  rdi is guaranteed non-zero.
 4803     assert(!pushed_rdi, "rdi must be left non-null");
 4804     // Also, the condition codes are properly set Z/NZ on succeed/failure.
 4805   }
 4806 
 4807   if (L_failure == &L_fallthrough)
 4808         jccb(Assembler::notEqual, *L_failure);
 4809   else  jcc(Assembler::notEqual, *L_failure);
 4810 
 4811   // Success.  Cache the super we found and proceed in triumph.
 4812   movptr(super_cache_addr, super_klass);
 4813 
 4814   if (L_success != &L_fallthrough) {
 4815     jmp(*L_success);
 4816   }
 4817 
 4818 #undef IS_A_TEMP
 4819 
 4820   bind(L_fallthrough);
 4821 }
 4822 
 4823 #ifndef _LP64
 4824 
 4825 // 32-bit x86 only: always use the linear search.
 4826 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4827                                                    Register super_klass,
 4828                                                    Register temp_reg,
 4829                                                    Register temp2_reg,
 4830                                                    Label* L_success,
 4831                                                    Label* L_failure,
 4832                                                    bool set_cond_codes) {
 4833   check_klass_subtype_slow_path_linear
 4834     (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, set_cond_codes);
 4835 }
 4836 
 4837 #else // _LP64
 4838 
 4839 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4840                                                    Register super_klass,
 4841                                                    Register temp_reg,
 4842                                                    Register temp2_reg,
 4843                                                    Label* L_success,
 4844                                                    Label* L_failure,
 4845                                                    bool set_cond_codes) {
 4846   assert(set_cond_codes == false, "must be false on 64-bit x86");
 4847   check_klass_subtype_slow_path
 4848     (sub_klass, super_klass, temp_reg, temp2_reg, noreg, noreg,
 4849      L_success, L_failure);
 4850 }
 4851 
 4852 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
 4853                                                    Register super_klass,
 4854                                                    Register temp_reg,
 4855                                                    Register temp2_reg,
 4856                                                    Register temp3_reg,
 4857                                                    Register temp4_reg,
 4858                                                    Label* L_success,
 4859                                                    Label* L_failure) {
 4860   if (UseSecondarySupersTable) {
 4861     check_klass_subtype_slow_path_table
 4862       (sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, temp4_reg,
 4863        L_success, L_failure);
 4864   } else {
 4865     check_klass_subtype_slow_path_linear
 4866       (sub_klass, super_klass, temp_reg, temp2_reg, L_success, L_failure, /*set_cond_codes*/false);
 4867   }
 4868 }
 4869 
 4870 Register MacroAssembler::allocate_if_noreg(Register r,
 4871                                   RegSetIterator<Register> &available_regs,
 4872                                   RegSet &regs_to_push) {
 4873   if (!r->is_valid()) {
 4874     r = *available_regs++;
 4875     regs_to_push += r;
 4876   }
 4877   return r;
 4878 }
 4879 
 4880 void MacroAssembler::check_klass_subtype_slow_path_table(Register sub_klass,
 4881                                                          Register super_klass,
 4882                                                          Register temp_reg,
 4883                                                          Register temp2_reg,
 4884                                                          Register temp3_reg,
 4885                                                          Register result_reg,
 4886                                                          Label* L_success,
 4887                                                          Label* L_failure) {
 4888   // NB! Callers may assume that, when temp2_reg is a valid register,
 4889   // this code sets it to a nonzero value.
 4890   bool temp2_reg_was_valid = temp2_reg->is_valid();
 4891 
 4892   RegSet temps = RegSet::of(temp_reg, temp2_reg, temp3_reg);
 4893 
 4894   Label L_fallthrough;
 4895   int label_nulls = 0;
 4896   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 4897   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 4898   assert(label_nulls <= 1, "at most one null in the batch");
 4899 
 4900   BLOCK_COMMENT("check_klass_subtype_slow_path_table");
 4901 
 4902   RegSetIterator<Register> available_regs
 4903     = (RegSet::of(rax, rcx, rdx, r8) + r9 + r10 + r11 + r12 - temps - sub_klass - super_klass).begin();
 4904 
 4905   RegSet pushed_regs;
 4906 
 4907   temp_reg = allocate_if_noreg(temp_reg, available_regs, pushed_regs);
 4908   temp2_reg = allocate_if_noreg(temp2_reg, available_regs, pushed_regs);
 4909   temp3_reg = allocate_if_noreg(temp3_reg, available_regs, pushed_regs);
 4910   result_reg = allocate_if_noreg(result_reg, available_regs, pushed_regs);
 4911   Register temp4_reg = allocate_if_noreg(noreg, available_regs, pushed_regs);
 4912 
 4913   assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, temp3_reg, result_reg);
 4914 
 4915   {
 4916 
 4917     int register_push_size = pushed_regs.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size;
 4918     int aligned_size = align_up(register_push_size, StackAlignmentInBytes);
 4919     subptr(rsp, aligned_size);
 4920     push_set(pushed_regs, 0);
 4921 
 4922     lookup_secondary_supers_table_var(sub_klass,
 4923                                       super_klass,
 4924                                       temp_reg, temp2_reg, temp3_reg, temp4_reg, result_reg);
 4925     cmpq(result_reg, 0);
 4926 
 4927     // Unspill the temp. registers:
 4928     pop_set(pushed_regs, 0);
 4929     // Increment SP but do not clobber flags.
 4930     lea(rsp, Address(rsp, aligned_size));
 4931   }
 4932 
 4933   if (temp2_reg_was_valid) {
 4934     movq(temp2_reg, 1);
 4935   }
 4936 
 4937   jcc(Assembler::notEqual, *L_failure);
 4938 
 4939   if (L_success != &L_fallthrough) {
 4940     jmp(*L_success);
 4941   }
 4942 
 4943   bind(L_fallthrough);
 4944 }
 4945 
 4946 // population_count variant for running without the POPCNT
 4947 // instruction, which was introduced with SSE4.2 in 2008.
 4948 void MacroAssembler::population_count(Register dst, Register src,
 4949                                       Register scratch1, Register scratch2) {
 4950   assert_different_registers(src, scratch1, scratch2);
 4951   if (UsePopCountInstruction) {
 4952     Assembler::popcntq(dst, src);
 4953   } else {
 4954     assert_different_registers(src, scratch1, scratch2);
 4955     assert_different_registers(dst, scratch1, scratch2);
 4956     Label loop, done;
 4957 
 4958     mov(scratch1, src);
 4959     // dst = 0;
 4960     // while(scratch1 != 0) {
 4961     //   dst++;
 4962     //   scratch1 &= (scratch1 - 1);
 4963     // }
 4964     xorl(dst, dst);
 4965     testq(scratch1, scratch1);
 4966     jccb(Assembler::equal, done);
 4967     {
 4968       bind(loop);
 4969       incq(dst);
 4970       movq(scratch2, scratch1);
 4971       decq(scratch2);
 4972       andq(scratch1, scratch2);
 4973       jccb(Assembler::notEqual, loop);
 4974     }
 4975     bind(done);
 4976   }
 4977 #ifdef ASSERT
 4978   mov64(scratch1, 0xCafeBabeDeadBeef);
 4979   movq(scratch2, scratch1);
 4980 #endif
 4981 }
 4982 
 4983 // Ensure that the inline code and the stub are using the same registers.
 4984 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS                      \
 4985 do {                                                                 \
 4986   assert(r_super_klass  == rax, "mismatch");                         \
 4987   assert(r_array_base   == rbx, "mismatch");                         \
 4988   assert(r_array_length == rcx, "mismatch");                         \
 4989   assert(r_array_index  == rdx, "mismatch");                         \
 4990   assert(r_sub_klass    == rsi || r_sub_klass == noreg, "mismatch"); \
 4991   assert(r_bitmap       == r11 || r_bitmap    == noreg, "mismatch"); \
 4992   assert(result         == rdi || result      == noreg, "mismatch"); \
 4993 } while(0)
 4994 
 4995 // Versions of salq and rorq that don't need count to be in rcx
 4996 
 4997 void MacroAssembler::salq(Register dest, Register count) {
 4998   if (count == rcx) {
 4999     Assembler::salq(dest);
 5000   } else {
 5001     assert_different_registers(rcx, dest);
 5002     xchgq(rcx, count);
 5003     Assembler::salq(dest);
 5004     xchgq(rcx, count);
 5005   }
 5006 }
 5007 
 5008 void MacroAssembler::rorq(Register dest, Register count) {
 5009   if (count == rcx) {
 5010     Assembler::rorq(dest);
 5011   } else {
 5012     assert_different_registers(rcx, dest);
 5013     xchgq(rcx, count);
 5014     Assembler::rorq(dest);
 5015     xchgq(rcx, count);
 5016   }
 5017 }
 5018 
 5019 // Return true: we succeeded in generating this code
 5020 //
 5021 // At runtime, return 0 in result if r_super_klass is a superclass of
 5022 // r_sub_klass, otherwise return nonzero. Use this if you know the
 5023 // super_klass_slot of the class you're looking for. This is always
 5024 // the case for instanceof and checkcast.
 5025 void MacroAssembler::lookup_secondary_supers_table_const(Register r_sub_klass,
 5026                                                          Register r_super_klass,
 5027                                                          Register temp1,
 5028                                                          Register temp2,
 5029                                                          Register temp3,
 5030                                                          Register temp4,
 5031                                                          Register result,
 5032                                                          u1 super_klass_slot) {
 5033   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
 5034 
 5035   Label L_fallthrough, L_success, L_failure;
 5036 
 5037   BLOCK_COMMENT("lookup_secondary_supers_table {");
 5038 
 5039   const Register
 5040     r_array_index  = temp1,
 5041     r_array_length = temp2,
 5042     r_array_base   = temp3,
 5043     r_bitmap       = temp4;
 5044 
 5045   LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS;
 5046 
 5047   xorq(result, result); // = 0
 5048 
 5049   movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
 5050   movq(r_array_index, r_bitmap);
 5051 
 5052   // First check the bitmap to see if super_klass might be present. If
 5053   // the bit is zero, we are certain that super_klass is not one of
 5054   // the secondary supers.
 5055   u1 bit = super_klass_slot;
 5056   {
 5057     // NB: If the count in a x86 shift instruction is 0, the flags are
 5058     // not affected, so we do a testq instead.
 5059     int shift_count = Klass::SECONDARY_SUPERS_TABLE_MASK - bit;
 5060     if (shift_count != 0) {
 5061       salq(r_array_index, shift_count);
 5062     } else {
 5063       testq(r_array_index, r_array_index);
 5064     }
 5065   }
 5066   // We test the MSB of r_array_index, i.e. its sign bit
 5067   jcc(Assembler::positive, L_failure);
 5068 
 5069   // Get the first array index that can contain super_klass into r_array_index.
 5070   if (bit != 0) {
 5071     population_count(r_array_index, r_array_index, temp2, temp3);
 5072   } else {
 5073     movl(r_array_index, 1);
 5074   }
 5075   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
 5076 
 5077   // We will consult the secondary-super array.
 5078   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5079 
 5080   // We're asserting that the first word in an Array<Klass*> is the
 5081   // length, and the second word is the first word of the data. If
 5082   // that ever changes, r_array_base will have to be adjusted here.
 5083   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
 5084   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
 5085 
 5086   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 5087   jccb(Assembler::equal, L_success);
 5088 
 5089   // Is there another entry to check? Consult the bitmap.
 5090   btq(r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK);
 5091   jccb(Assembler::carryClear, L_failure);
 5092 
 5093   // Linear probe. Rotate the bitmap so that the next bit to test is
 5094   // in Bit 1.
 5095   if (bit != 0) {
 5096     rorq(r_bitmap, bit);
 5097   }
 5098 
 5099   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
 5100   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
 5101   // Kills: r_array_length.
 5102   // Returns: result.
 5103   call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_slow_path_stub()));
 5104   // Result (0/1) is in rdi
 5105   jmpb(L_fallthrough);
 5106 
 5107   bind(L_failure);
 5108   incq(result); // 0 => 1
 5109 
 5110   bind(L_success);
 5111   // result = 0;
 5112 
 5113   bind(L_fallthrough);
 5114   BLOCK_COMMENT("} lookup_secondary_supers_table");
 5115 
 5116   if (VerifySecondarySupers) {
 5117     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
 5118                                   temp1, temp2, temp3);
 5119   }
 5120 }
 5121 
 5122 // At runtime, return 0 in result if r_super_klass is a superclass of
 5123 // r_sub_klass, otherwise return nonzero. Use this version of
 5124 // lookup_secondary_supers_table() if you don't know ahead of time
 5125 // which superclass will be searched for. Used by interpreter and
 5126 // runtime stubs. It is larger and has somewhat greater latency than
 5127 // the version above, which takes a constant super_klass_slot.
 5128 void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass,
 5129                                                        Register r_super_klass,
 5130                                                        Register temp1,
 5131                                                        Register temp2,
 5132                                                        Register temp3,
 5133                                                        Register temp4,
 5134                                                        Register result) {
 5135   assert_different_registers(r_sub_klass, r_super_klass, temp1, temp2, temp3, temp4, result);
 5136   assert_different_registers(r_sub_klass, r_super_klass, rcx);
 5137   RegSet temps = RegSet::of(temp1, temp2, temp3, temp4);
 5138 
 5139   Label L_fallthrough, L_success, L_failure;
 5140 
 5141   BLOCK_COMMENT("lookup_secondary_supers_table {");
 5142 
 5143   RegSetIterator<Register> available_regs = (temps - rcx).begin();
 5144 
 5145   // FIXME. Once we are sure that all paths reaching this point really
 5146   // do pass rcx as one of our temps we can get rid of the following
 5147   // workaround.
 5148   assert(temps.contains(rcx), "fix this code");
 5149 
 5150   // We prefer to have our shift count in rcx. If rcx is one of our
 5151   // temps, use it for slot. If not, pick any of our temps.
 5152   Register slot;
 5153   if (!temps.contains(rcx)) {
 5154     slot = *available_regs++;
 5155   } else {
 5156     slot = rcx;
 5157   }
 5158 
 5159   const Register r_array_index = *available_regs++;
 5160   const Register r_bitmap      = *available_regs++;
 5161 
 5162   // The logic above guarantees this property, but we state it here.
 5163   assert_different_registers(r_array_index, r_bitmap, rcx);
 5164 
 5165   movq(r_bitmap, Address(r_sub_klass, Klass::secondary_supers_bitmap_offset()));
 5166   movq(r_array_index, r_bitmap);
 5167 
 5168   // First check the bitmap to see if super_klass might be present. If
 5169   // the bit is zero, we are certain that super_klass is not one of
 5170   // the secondary supers.
 5171   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
 5172   xorl(slot, (u1)(Klass::SECONDARY_SUPERS_TABLE_SIZE - 1)); // slot ^ 63 === 63 - slot (mod 64)
 5173   salq(r_array_index, slot);
 5174 
 5175   testq(r_array_index, r_array_index);
 5176   // We test the MSB of r_array_index, i.e. its sign bit
 5177   jcc(Assembler::positive, L_failure);
 5178 
 5179   const Register r_array_base = *available_regs++;
 5180 
 5181   // Get the first array index that can contain super_klass into r_array_index.
 5182   // Note: Clobbers r_array_base and slot.
 5183   population_count(r_array_index, r_array_index, /*temp2*/r_array_base, /*temp3*/slot);
 5184 
 5185   // NB! r_array_index is off by 1. It is compensated by keeping r_array_base off by 1 word.
 5186 
 5187   // We will consult the secondary-super array.
 5188   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5189 
 5190   // We're asserting that the first word in an Array<Klass*> is the
 5191   // length, and the second word is the first word of the data. If
 5192   // that ever changes, r_array_base will have to be adjusted here.
 5193   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code");
 5194   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
 5195 
 5196   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 5197   jccb(Assembler::equal, L_success);
 5198 
 5199   // Restore slot to its true value
 5200   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
 5201 
 5202   // Linear probe. Rotate the bitmap so that the next bit to test is
 5203   // in Bit 1.
 5204   rorq(r_bitmap, slot);
 5205 
 5206   // Is there another entry to check? Consult the bitmap.
 5207   btq(r_bitmap, 1);
 5208   jccb(Assembler::carryClear, L_failure);
 5209 
 5210   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
 5211   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
 5212   // Kills: r_array_length.
 5213   // Returns: result.
 5214   lookup_secondary_supers_table_slow_path(r_super_klass,
 5215                                           r_array_base,
 5216                                           r_array_index,
 5217                                           r_bitmap,
 5218                                           /*temp1*/result,
 5219                                           /*temp2*/slot,
 5220                                           &L_success,
 5221                                           nullptr);
 5222 
 5223   bind(L_failure);
 5224   movq(result, 1);
 5225   jmpb(L_fallthrough);
 5226 
 5227   bind(L_success);
 5228   xorq(result, result); // = 0
 5229 
 5230   bind(L_fallthrough);
 5231   BLOCK_COMMENT("} lookup_secondary_supers_table");
 5232 
 5233   if (VerifySecondarySupers) {
 5234     verify_secondary_supers_table(r_sub_klass, r_super_klass, result,
 5235                                   temp1, temp2, temp3);
 5236   }
 5237 }
 5238 
 5239 void MacroAssembler::repne_scanq(Register addr, Register value, Register count, Register limit,
 5240                                  Label* L_success, Label* L_failure) {
 5241   Label L_loop, L_fallthrough;
 5242   {
 5243     int label_nulls = 0;
 5244     if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; }
 5245     if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; }
 5246     assert(label_nulls <= 1, "at most one null in the batch");
 5247   }
 5248   bind(L_loop);
 5249   cmpq(value, Address(addr, count, Address::times_8));
 5250   jcc(Assembler::equal, *L_success);
 5251   addl(count, 1);
 5252   cmpl(count, limit);
 5253   jcc(Assembler::less, L_loop);
 5254 
 5255   if (&L_fallthrough != L_failure) {
 5256     jmp(*L_failure);
 5257   }
 5258   bind(L_fallthrough);
 5259 }
 5260 
 5261 // Called by code generated by check_klass_subtype_slow_path
 5262 // above. This is called when there is a collision in the hashed
 5263 // lookup in the secondary supers array.
 5264 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass,
 5265                                                              Register r_array_base,
 5266                                                              Register r_array_index,
 5267                                                              Register r_bitmap,
 5268                                                              Register temp1,
 5269                                                              Register temp2,
 5270                                                              Label* L_success,
 5271                                                              Label* L_failure) {
 5272   assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, temp1, temp2);
 5273 
 5274   const Register
 5275     r_array_length = temp1,
 5276     r_sub_klass    = noreg,
 5277     result         = noreg;
 5278 
 5279   Label L_fallthrough;
 5280   int label_nulls = 0;
 5281   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
 5282   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
 5283   assert(label_nulls <= 1, "at most one null in the batch");
 5284 
 5285   // Load the array length.
 5286   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
 5287   // And adjust the array base to point to the data.
 5288   // NB! Effectively increments current slot index by 1.
 5289   assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "");
 5290   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
 5291 
 5292   // Linear probe
 5293   Label L_huge;
 5294 
 5295   // The bitmap is full to bursting.
 5296   // Implicit invariant: BITMAP_FULL implies (length > 0)
 5297   cmpl(r_array_length, (int32_t)Klass::SECONDARY_SUPERS_TABLE_SIZE - 2);
 5298   jcc(Assembler::greater, L_huge);
 5299 
 5300   // NB! Our caller has checked bits 0 and 1 in the bitmap. The
 5301   // current slot (at secondary_supers[r_array_index]) has not yet
 5302   // been inspected, and r_array_index may be out of bounds if we
 5303   // wrapped around the end of the array.
 5304 
 5305   { // This is conventional linear probing, but instead of terminating
 5306     // when a null entry is found in the table, we maintain a bitmap
 5307     // in which a 0 indicates missing entries.
 5308     // The check above guarantees there are 0s in the bitmap, so the loop
 5309     // eventually terminates.
 5310 
 5311     xorl(temp2, temp2); // = 0;
 5312 
 5313     Label L_again;
 5314     bind(L_again);
 5315 
 5316     // Check for array wraparound.
 5317     cmpl(r_array_index, r_array_length);
 5318     cmovl(Assembler::greaterEqual, r_array_index, temp2);
 5319 
 5320     cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
 5321     jcc(Assembler::equal, *L_success);
 5322 
 5323     // If the next bit in bitmap is zero, we're done.
 5324     btq(r_bitmap, 2); // look-ahead check (Bit 2); Bits 0 and 1 are tested by now
 5325     jcc(Assembler::carryClear, *L_failure);
 5326 
 5327     rorq(r_bitmap, 1); // Bits 1/2 => 0/1
 5328     addl(r_array_index, 1);
 5329 
 5330     jmp(L_again);
 5331   }
 5332 
 5333   { // Degenerate case: more than 64 secondary supers.
 5334     // FIXME: We could do something smarter here, maybe a vectorized
 5335     // comparison or a binary search, but is that worth any added
 5336     // complexity?
 5337     bind(L_huge);
 5338     xorl(r_array_index, r_array_index); // = 0
 5339     repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length,
 5340                 L_success,
 5341                 (&L_fallthrough != L_failure ? L_failure : nullptr));
 5342 
 5343     bind(L_fallthrough);
 5344   }
 5345 }
 5346 
 5347 struct VerifyHelperArguments {
 5348   Klass* _super;
 5349   Klass* _sub;
 5350   intptr_t _linear_result;
 5351   intptr_t _table_result;
 5352 };
 5353 
 5354 static void verify_secondary_supers_table_helper(const char* msg, VerifyHelperArguments* args) {
 5355   Klass::on_secondary_supers_verification_failure(args->_super,
 5356                                                   args->_sub,
 5357                                                   args->_linear_result,
 5358                                                   args->_table_result,
 5359                                                   msg);
 5360 }
 5361 
 5362 // Make sure that the hashed lookup and a linear scan agree.
 5363 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
 5364                                                    Register r_super_klass,
 5365                                                    Register result,
 5366                                                    Register temp1,
 5367                                                    Register temp2,
 5368                                                    Register temp3) {
 5369   const Register
 5370       r_array_index  = temp1,
 5371       r_array_length = temp2,
 5372       r_array_base   = temp3,
 5373       r_bitmap       = noreg;
 5374 
 5375   BLOCK_COMMENT("verify_secondary_supers_table {");
 5376 
 5377   Label L_success, L_failure, L_check, L_done;
 5378 
 5379   movptr(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset())));
 5380   movl(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes()));
 5381   // And adjust the array base to point to the data.
 5382   addptr(r_array_base, Array<Klass*>::base_offset_in_bytes());
 5383 
 5384   testl(r_array_length, r_array_length); // array_length == 0?
 5385   jcc(Assembler::zero, L_failure);
 5386 
 5387   movl(r_array_index, 0);
 5388   repne_scanq(r_array_base, r_super_klass, r_array_index, r_array_length, &L_success);
 5389   // fall through to L_failure
 5390 
 5391   const Register linear_result = r_array_index; // reuse temp1
 5392 
 5393   bind(L_failure); // not present
 5394   movl(linear_result, 1);
 5395   jmp(L_check);
 5396 
 5397   bind(L_success); // present
 5398   movl(linear_result, 0);
 5399 
 5400   bind(L_check);
 5401   cmpl(linear_result, result);
 5402   jcc(Assembler::equal, L_done);
 5403 
 5404   { // To avoid calling convention issues, build a record on the stack
 5405     // and pass the pointer to that instead.
 5406     push(result);
 5407     push(linear_result);
 5408     push(r_sub_klass);
 5409     push(r_super_klass);
 5410     movptr(c_rarg1, rsp);
 5411     movptr(c_rarg0, (uintptr_t) "mismatch");
 5412     call(RuntimeAddress(CAST_FROM_FN_PTR(address, verify_secondary_supers_table_helper)));
 5413     should_not_reach_here();
 5414   }
 5415   bind(L_done);
 5416 
 5417   BLOCK_COMMENT("} verify_secondary_supers_table");
 5418 }
 5419 
 5420 #undef LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS
 5421 
 5422 #endif // LP64
 5423 
 5424 void MacroAssembler::clinit_barrier(Register klass, Label* L_fast_path, Label* L_slow_path) {
 5425   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
 5426 
 5427   Label L_fallthrough;
 5428   if (L_fast_path == nullptr) {
 5429     L_fast_path = &L_fallthrough;
 5430   } else if (L_slow_path == nullptr) {
 5431     L_slow_path = &L_fallthrough;
 5432   }
 5433 
 5434   // Fast path check: class is fully initialized.
 5435   // init_state needs acquire, but x86 is TSO, and so we are already good.
 5436   cmpb(Address(klass, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
 5437   jcc(Assembler::equal, *L_fast_path);
 5438 
 5439   // Fast path check: current thread is initializer thread
 5440   cmpptr(r15_thread, Address(klass, InstanceKlass::init_thread_offset()));
 5441   if (L_slow_path == &L_fallthrough) {
 5442     jcc(Assembler::equal, *L_fast_path);
 5443     bind(*L_slow_path);
 5444   } else if (L_fast_path == &L_fallthrough) {
 5445     jcc(Assembler::notEqual, *L_slow_path);
 5446     bind(*L_fast_path);
 5447   } else {
 5448     Unimplemented();
 5449   }
 5450 }
 5451 
 5452 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
 5453   if (VM_Version::supports_cmov()) {
 5454     cmovl(cc, dst, src);
 5455   } else {
 5456     Label L;
 5457     jccb(negate_condition(cc), L);
 5458     movl(dst, src);
 5459     bind(L);
 5460   }
 5461 }
 5462 
 5463 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 5464   if (VM_Version::supports_cmov()) {
 5465     cmovl(cc, dst, src);
 5466   } else {
 5467     Label L;
 5468     jccb(negate_condition(cc), L);
 5469     movl(dst, src);
 5470     bind(L);
 5471   }
 5472 }
 5473 
 5474 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 5475   if (!VerifyOops) return;
 5476 
 5477   BLOCK_COMMENT("verify_oop {");
 5478 #ifdef _LP64
 5479   push(rscratch1);
 5480 #endif
 5481   push(rax);                          // save rax
 5482   push(reg);                          // pass register argument
 5483 
 5484   // Pass register number to verify_oop_subroutine
 5485   const char* b = nullptr;
 5486   {
 5487     ResourceMark rm;
 5488     stringStream ss;
 5489     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 5490     b = code_string(ss.as_string());
 5491   }
 5492   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
 5493   pushptr(buffer.addr(), rscratch1);
 5494 
 5495   // call indirectly to solve generation ordering problem
 5496   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5497   call(rax);
 5498   // Caller pops the arguments (oop, message) and restores rax, r10
 5499   BLOCK_COMMENT("} verify_oop");
 5500 }
 5501 
 5502 void MacroAssembler::vallones(XMMRegister dst, int vector_len) {
 5503   if (UseAVX > 2 && (vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl())) {
 5504     // Only pcmpeq has dependency breaking treatment (i.e the execution can begin without
 5505     // waiting for the previous result on dst), not vpcmpeqd, so just use vpternlog
 5506     vpternlogd(dst, 0xFF, dst, dst, vector_len);
 5507   } else if (VM_Version::supports_avx()) {
 5508     vpcmpeqd(dst, dst, dst, vector_len);
 5509   } else {
 5510     pcmpeqd(dst, dst);
 5511   }
 5512 }
 5513 
 5514 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
 5515                                          int extra_slot_offset) {
 5516   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 5517   int stackElementSize = Interpreter::stackElementSize;
 5518   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 5519 #ifdef ASSERT
 5520   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 5521   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 5522 #endif
 5523   Register             scale_reg    = noreg;
 5524   Address::ScaleFactor scale_factor = Address::no_scale;
 5525   if (arg_slot.is_constant()) {
 5526     offset += arg_slot.as_constant() * stackElementSize;
 5527   } else {
 5528     scale_reg    = arg_slot.as_register();
 5529     scale_factor = Address::times(stackElementSize);
 5530   }
 5531   offset += wordSize;           // return PC is on stack
 5532   return Address(rsp, scale_reg, scale_factor, offset);
 5533 }
 5534 
 5535 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 5536   if (!VerifyOops) return;
 5537 
 5538 #ifdef _LP64
 5539   push(rscratch1);
 5540 #endif
 5541   push(rax); // save rax,
 5542   // addr may contain rsp so we will have to adjust it based on the push
 5543   // we just did (and on 64 bit we do two pushes)
 5544   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 5545   // stores rax into addr which is backwards of what was intended.
 5546   if (addr.uses(rsp)) {
 5547     lea(rax, addr);
 5548     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
 5549   } else {
 5550     pushptr(addr);
 5551   }
 5552 
 5553   // Pass register number to verify_oop_subroutine
 5554   const char* b = nullptr;
 5555   {
 5556     ResourceMark rm;
 5557     stringStream ss;
 5558     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 5559     b = code_string(ss.as_string());
 5560   }
 5561   AddressLiteral buffer((address) b, external_word_Relocation::spec_for_immediate());
 5562   pushptr(buffer.addr(), rscratch1);
 5563 
 5564   // call indirectly to solve generation ordering problem
 5565   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
 5566   call(rax);
 5567   // Caller pops the arguments (addr, message) and restores rax, r10.
 5568 }
 5569 
 5570 void MacroAssembler::verify_tlab() {
 5571 #ifdef ASSERT
 5572   if (UseTLAB && VerifyOops) {
 5573     Label next, ok;
 5574     Register t1 = rsi;
 5575     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
 5576 
 5577     push(t1);
 5578     NOT_LP64(push(thread_reg));
 5579     NOT_LP64(get_thread(thread_reg));
 5580 
 5581     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5582     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
 5583     jcc(Assembler::aboveEqual, next);
 5584     STOP("assert(top >= start)");
 5585     should_not_reach_here();
 5586 
 5587     bind(next);
 5588     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
 5589     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
 5590     jcc(Assembler::aboveEqual, ok);
 5591     STOP("assert(top <= end)");
 5592     should_not_reach_here();
 5593 
 5594     bind(ok);
 5595     NOT_LP64(pop(thread_reg));
 5596     pop(t1);
 5597   }
 5598 #endif
 5599 }
 5600 
 5601 class ControlWord {
 5602  public:
 5603   int32_t _value;
 5604 
 5605   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
 5606   int  precision_control() const       { return  (_value >>  8) & 3      ; }
 5607   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5608   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5609   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5610   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5611   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5612   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5613 
 5614   void print() const {
 5615     // rounding control
 5616     const char* rc;
 5617     switch (rounding_control()) {
 5618       case 0: rc = "round near"; break;
 5619       case 1: rc = "round down"; break;
 5620       case 2: rc = "round up  "; break;
 5621       case 3: rc = "chop      "; break;
 5622       default:
 5623         rc = nullptr; // silence compiler warnings
 5624         fatal("Unknown rounding control: %d", rounding_control());
 5625     };
 5626     // precision control
 5627     const char* pc;
 5628     switch (precision_control()) {
 5629       case 0: pc = "24 bits "; break;
 5630       case 1: pc = "reserved"; break;
 5631       case 2: pc = "53 bits "; break;
 5632       case 3: pc = "64 bits "; break;
 5633       default:
 5634         pc = nullptr; // silence compiler warnings
 5635         fatal("Unknown precision control: %d", precision_control());
 5636     };
 5637     // flags
 5638     char f[9];
 5639     f[0] = ' ';
 5640     f[1] = ' ';
 5641     f[2] = (precision   ()) ? 'P' : 'p';
 5642     f[3] = (underflow   ()) ? 'U' : 'u';
 5643     f[4] = (overflow    ()) ? 'O' : 'o';
 5644     f[5] = (zero_divide ()) ? 'Z' : 'z';
 5645     f[6] = (denormalized()) ? 'D' : 'd';
 5646     f[7] = (invalid     ()) ? 'I' : 'i';
 5647     f[8] = '\x0';
 5648     // output
 5649     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
 5650   }
 5651 
 5652 };
 5653 
 5654 class StatusWord {
 5655  public:
 5656   int32_t _value;
 5657 
 5658   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
 5659   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
 5660   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
 5661   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
 5662   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
 5663   int  top() const                     { return  (_value >> 11) & 7      ; }
 5664   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
 5665   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
 5666   bool precision() const               { return ((_value >>  5) & 1) != 0; }
 5667   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
 5668   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
 5669   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
 5670   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
 5671   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
 5672 
 5673   void print() const {
 5674     // condition codes
 5675     char c[5];
 5676     c[0] = (C3()) ? '3' : '-';
 5677     c[1] = (C2()) ? '2' : '-';
 5678     c[2] = (C1()) ? '1' : '-';
 5679     c[3] = (C0()) ? '0' : '-';
 5680     c[4] = '\x0';
 5681     // flags
 5682     char f[9];
 5683     f[0] = (error_status()) ? 'E' : '-';
 5684     f[1] = (stack_fault ()) ? 'S' : '-';
 5685     f[2] = (precision   ()) ? 'P' : '-';
 5686     f[3] = (underflow   ()) ? 'U' : '-';
 5687     f[4] = (overflow    ()) ? 'O' : '-';
 5688     f[5] = (zero_divide ()) ? 'Z' : '-';
 5689     f[6] = (denormalized()) ? 'D' : '-';
 5690     f[7] = (invalid     ()) ? 'I' : '-';
 5691     f[8] = '\x0';
 5692     // output
 5693     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
 5694   }
 5695 
 5696 };
 5697 
 5698 class TagWord {
 5699  public:
 5700   int32_t _value;
 5701 
 5702   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
 5703 
 5704   void print() const {
 5705     printf("%04x", _value & 0xFFFF);
 5706   }
 5707 
 5708 };
 5709 
 5710 class FPU_Register {
 5711  public:
 5712   int32_t _m0;
 5713   int32_t _m1;
 5714   int16_t _ex;
 5715 
 5716   bool is_indefinite() const           {
 5717     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
 5718   }
 5719 
 5720   void print() const {
 5721     char  sign = (_ex < 0) ? '-' : '+';
 5722     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
 5723     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
 5724   };
 5725 
 5726 };
 5727 
 5728 class FPU_State {
 5729  public:
 5730   enum {
 5731     register_size       = 10,
 5732     number_of_registers =  8,
 5733     register_mask       =  7
 5734   };
 5735 
 5736   ControlWord  _control_word;
 5737   StatusWord   _status_word;
 5738   TagWord      _tag_word;
 5739   int32_t      _error_offset;
 5740   int32_t      _error_selector;
 5741   int32_t      _data_offset;
 5742   int32_t      _data_selector;
 5743   int8_t       _register[register_size * number_of_registers];
 5744 
 5745   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
 5746   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
 5747 
 5748   const char* tag_as_string(int tag) const {
 5749     switch (tag) {
 5750       case 0: return "valid";
 5751       case 1: return "zero";
 5752       case 2: return "special";
 5753       case 3: return "empty";
 5754     }
 5755     ShouldNotReachHere();
 5756     return nullptr;
 5757   }
 5758 
 5759   void print() const {
 5760     // print computation registers
 5761     { int t = _status_word.top();
 5762       for (int i = 0; i < number_of_registers; i++) {
 5763         int j = (i - t) & register_mask;
 5764         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
 5765         st(j)->print();
 5766         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
 5767       }
 5768     }
 5769     printf("\n");
 5770     // print control registers
 5771     printf("ctrl = "); _control_word.print(); printf("\n");
 5772     printf("stat = "); _status_word .print(); printf("\n");
 5773     printf("tags = "); _tag_word    .print(); printf("\n");
 5774   }
 5775 
 5776 };
 5777 
 5778 class Flag_Register {
 5779  public:
 5780   int32_t _value;
 5781 
 5782   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
 5783   bool direction() const               { return ((_value >> 10) & 1) != 0; }
 5784   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
 5785   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
 5786   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
 5787   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
 5788   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
 5789 
 5790   void print() const {
 5791     // flags
 5792     char f[8];
 5793     f[0] = (overflow       ()) ? 'O' : '-';
 5794     f[1] = (direction      ()) ? 'D' : '-';
 5795     f[2] = (sign           ()) ? 'S' : '-';
 5796     f[3] = (zero           ()) ? 'Z' : '-';
 5797     f[4] = (auxiliary_carry()) ? 'A' : '-';
 5798     f[5] = (parity         ()) ? 'P' : '-';
 5799     f[6] = (carry          ()) ? 'C' : '-';
 5800     f[7] = '\x0';
 5801     // output
 5802     printf("%08x  flags = %s", _value, f);
 5803   }
 5804 
 5805 };
 5806 
 5807 class IU_Register {
 5808  public:
 5809   int32_t _value;
 5810 
 5811   void print() const {
 5812     printf("%08x  %11d", _value, _value);
 5813   }
 5814 
 5815 };
 5816 
 5817 class IU_State {
 5818  public:
 5819   Flag_Register _eflags;
 5820   IU_Register   _rdi;
 5821   IU_Register   _rsi;
 5822   IU_Register   _rbp;
 5823   IU_Register   _rsp;
 5824   IU_Register   _rbx;
 5825   IU_Register   _rdx;
 5826   IU_Register   _rcx;
 5827   IU_Register   _rax;
 5828 
 5829   void print() const {
 5830     // computation registers
 5831     printf("rax,  = "); _rax.print(); printf("\n");
 5832     printf("rbx,  = "); _rbx.print(); printf("\n");
 5833     printf("rcx  = "); _rcx.print(); printf("\n");
 5834     printf("rdx  = "); _rdx.print(); printf("\n");
 5835     printf("rdi  = "); _rdi.print(); printf("\n");
 5836     printf("rsi  = "); _rsi.print(); printf("\n");
 5837     printf("rbp,  = "); _rbp.print(); printf("\n");
 5838     printf("rsp  = "); _rsp.print(); printf("\n");
 5839     printf("\n");
 5840     // control registers
 5841     printf("flgs = "); _eflags.print(); printf("\n");
 5842   }
 5843 };
 5844 
 5845 
 5846 class CPU_State {
 5847  public:
 5848   FPU_State _fpu_state;
 5849   IU_State  _iu_state;
 5850 
 5851   void print() const {
 5852     printf("--------------------------------------------------\n");
 5853     _iu_state .print();
 5854     printf("\n");
 5855     _fpu_state.print();
 5856     printf("--------------------------------------------------\n");
 5857   }
 5858 
 5859 };
 5860 
 5861 
 5862 static void _print_CPU_state(CPU_State* state) {
 5863   state->print();
 5864 };
 5865 
 5866 
 5867 void MacroAssembler::print_CPU_state() {
 5868   push_CPU_state();
 5869   push(rsp);                // pass CPU state
 5870   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
 5871   addptr(rsp, wordSize);       // discard argument
 5872   pop_CPU_state();
 5873 }
 5874 
 5875 
 5876 #ifndef _LP64
 5877 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
 5878   static int counter = 0;
 5879   FPU_State* fs = &state->_fpu_state;
 5880   counter++;
 5881   // For leaf calls, only verify that the top few elements remain empty.
 5882   // We only need 1 empty at the top for C2 code.
 5883   if( stack_depth < 0 ) {
 5884     if( fs->tag_for_st(7) != 3 ) {
 5885       printf("FPR7 not empty\n");
 5886       state->print();
 5887       assert(false, "error");
 5888       return false;
 5889     }
 5890     return true;                // All other stack states do not matter
 5891   }
 5892 
 5893   assert((fs->_control_word._value & 0xffff) == StubRoutines::x86::fpu_cntrl_wrd_std(),
 5894          "bad FPU control word");
 5895 
 5896   // compute stack depth
 5897   int i = 0;
 5898   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
 5899   int d = i;
 5900   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
 5901   // verify findings
 5902   if (i != FPU_State::number_of_registers) {
 5903     // stack not contiguous
 5904     printf("%s: stack not contiguous at ST%d\n", s, i);
 5905     state->print();
 5906     assert(false, "error");
 5907     return false;
 5908   }
 5909   // check if computed stack depth corresponds to expected stack depth
 5910   if (stack_depth < 0) {
 5911     // expected stack depth is -stack_depth or less
 5912     if (d > -stack_depth) {
 5913       // too many elements on the stack
 5914       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
 5915       state->print();
 5916       assert(false, "error");
 5917       return false;
 5918     }
 5919   } else {
 5920     // expected stack depth is stack_depth
 5921     if (d != stack_depth) {
 5922       // wrong stack depth
 5923       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
 5924       state->print();
 5925       assert(false, "error");
 5926       return false;
 5927     }
 5928   }
 5929   // everything is cool
 5930   return true;
 5931 }
 5932 
 5933 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
 5934   if (!VerifyFPU) return;
 5935   push_CPU_state();
 5936   push(rsp);                // pass CPU state
 5937   ExternalAddress msg((address) s);
 5938   // pass message string s
 5939   pushptr(msg.addr(), noreg);
 5940   push(stack_depth);        // pass stack depth
 5941   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
 5942   addptr(rsp, 3 * wordSize);   // discard arguments
 5943   // check for error
 5944   { Label L;
 5945     testl(rax, rax);
 5946     jcc(Assembler::notZero, L);
 5947     int3();                  // break if error condition
 5948     bind(L);
 5949   }
 5950   pop_CPU_state();
 5951 }
 5952 #endif // _LP64
 5953 
 5954 void MacroAssembler::restore_cpu_control_state_after_jni(Register rscratch) {
 5955   // Either restore the MXCSR register after returning from the JNI Call
 5956   // or verify that it wasn't changed (with -Xcheck:jni flag).
 5957   if (VM_Version::supports_sse()) {
 5958     if (RestoreMXCSROnJNICalls) {
 5959       ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), rscratch);
 5960     } else if (CheckJNICalls) {
 5961       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
 5962     }
 5963   }
 5964   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
 5965   vzeroupper();
 5966 
 5967 #ifndef _LP64
 5968   // Either restore the x87 floating pointer control word after returning
 5969   // from the JNI call or verify that it wasn't changed.
 5970   if (CheckJNICalls) {
 5971     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
 5972   }
 5973 #endif // _LP64
 5974 }
 5975 
 5976 // ((OopHandle)result).resolve();
 5977 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
 5978   assert_different_registers(result, tmp);
 5979 
 5980   // Only 64 bit platforms support GCs that require a tmp register
 5981   // Only IN_HEAP loads require a thread_tmp register
 5982   // OopHandle::resolve is an indirection like jobject.
 5983   access_load_at(T_OBJECT, IN_NATIVE,
 5984                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
 5985 }
 5986 
 5987 // ((WeakHandle)result).resolve();
 5988 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
 5989   assert_different_registers(rresult, rtmp);
 5990   Label resolved;
 5991 
 5992   // A null weak handle resolves to null.
 5993   cmpptr(rresult, 0);
 5994   jcc(Assembler::equal, resolved);
 5995 
 5996   // Only 64 bit platforms support GCs that require a tmp register
 5997   // Only IN_HEAP loads require a thread_tmp register
 5998   // WeakHandle::resolve is an indirection like jweak.
 5999   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
 6000                  rresult, Address(rresult, 0), rtmp, /*tmp_thread*/noreg);
 6001   bind(resolved);
 6002 }
 6003 
 6004 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 6005   // get mirror
 6006   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 6007   load_method_holder(mirror, method);
 6008   movptr(mirror, Address(mirror, mirror_offset));
 6009   resolve_oop_handle(mirror, tmp);
 6010 }
 6011 
 6012 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 6013   load_method_holder(rresult, rmethod);
 6014   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 6015 }
 6016 
 6017 void MacroAssembler::load_method_holder(Register holder, Register method) {
 6018   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 6019   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 6020   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
 6021 }
 6022 
 6023 #ifdef _LP64
 6024 void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
 6025   assert(UseCompactObjectHeaders, "expect compact object headers");
 6026   movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
 6027   shrq(dst, markWord::klass_shift);
 6028 }
 6029 #endif
 6030 
 6031 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 6032   assert_different_registers(src, tmp);
 6033   assert_different_registers(dst, tmp);
 6034 #ifdef _LP64
 6035   if (UseCompactObjectHeaders) {
 6036     load_narrow_klass_compact(dst, src);
 6037     decode_klass_not_null(dst, tmp);
 6038   } else if (UseCompressedClassPointers) {
 6039     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 6040     decode_klass_not_null(dst, tmp);
 6041   } else
 6042 #endif
 6043   {
 6044     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 6045   }
 6046 }
 6047 
 6048 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 6049   assert(!UseCompactObjectHeaders, "not with compact headers");
 6050   assert_different_registers(src, tmp);
 6051   assert_different_registers(dst, tmp);
 6052 #ifdef _LP64
 6053   if (UseCompressedClassPointers) {
 6054     encode_klass_not_null(src, tmp);
 6055     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 6056   } else
 6057 #endif
 6058     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 6059 }
 6060 
 6061 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
 6062 #ifdef _LP64
 6063   if (UseCompactObjectHeaders) {
 6064     assert(tmp != noreg, "need tmp");
 6065     assert_different_registers(klass, obj, tmp);
 6066     load_narrow_klass_compact(tmp, obj);
 6067     cmpl(klass, tmp);
 6068   } else if (UseCompressedClassPointers) {
 6069     cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
 6070   } else
 6071 #endif
 6072   {
 6073     cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
 6074   }
 6075 }
 6076 
 6077 void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) {
 6078 #ifdef _LP64
 6079   if (UseCompactObjectHeaders) {
 6080     assert(tmp2 != noreg, "need tmp2");
 6081     assert_different_registers(obj1, obj2, tmp1, tmp2);
 6082     load_narrow_klass_compact(tmp1, obj1);
 6083     load_narrow_klass_compact(tmp2, obj2);
 6084     cmpl(tmp1, tmp2);
 6085   } else if (UseCompressedClassPointers) {
 6086     movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
 6087     cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
 6088   } else
 6089 #endif
 6090   {
 6091     movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
 6092     cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
 6093   }
 6094 }
 6095 
 6096 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 6097                                     Register tmp1, Register thread_tmp) {
 6098   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 6099   decorators = AccessInternal::decorator_fixup(decorators, type);
 6100   bool as_raw = (decorators & AS_RAW) != 0;
 6101   if (as_raw) {
 6102     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 6103   } else {
 6104     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 6105   }
 6106 }
 6107 
 6108 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 6109                                      Register tmp1, Register tmp2, Register tmp3) {
 6110   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 6111   decorators = AccessInternal::decorator_fixup(decorators, type);
 6112   bool as_raw = (decorators & AS_RAW) != 0;
 6113   if (as_raw) {
 6114     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 6115   } else {
 6116     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 6117   }
 6118 }
 6119 
 6120 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 6121                                    Register thread_tmp, DecoratorSet decorators) {
 6122   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 6123 }
 6124 
 6125 // Doesn't do verification, generates fixed size code
 6126 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 6127                                             Register thread_tmp, DecoratorSet decorators) {
 6128   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 6129 }
 6130 
 6131 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
 6132                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 6133   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
 6134 }
 6135 
 6136 // Used for storing nulls.
 6137 void MacroAssembler::store_heap_oop_null(Address dst) {
 6138   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 6139 }
 6140 
 6141 #ifdef _LP64
 6142 void MacroAssembler::store_klass_gap(Register dst, Register src) {
 6143   assert(!UseCompactObjectHeaders, "Don't use with compact headers");
 6144   if (UseCompressedClassPointers) {
 6145     // Store to klass gap in destination
 6146     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
 6147   }
 6148 }
 6149 
 6150 #ifdef ASSERT
 6151 void MacroAssembler::verify_heapbase(const char* msg) {
 6152   assert (UseCompressedOops, "should be compressed");
 6153   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6154   if (CheckCompressedOops) {
 6155     Label ok;
 6156     ExternalAddress src2(CompressedOops::base_addr());
 6157     const bool is_src2_reachable = reachable(src2);
 6158     if (!is_src2_reachable) {
 6159       push(rscratch1);  // cmpptr trashes rscratch1
 6160     }
 6161     cmpptr(r12_heapbase, src2, rscratch1);
 6162     jcc(Assembler::equal, ok);
 6163     STOP(msg);
 6164     bind(ok);
 6165     if (!is_src2_reachable) {
 6166       pop(rscratch1);
 6167     }
 6168   }
 6169 }
 6170 #endif
 6171 
 6172 // Algorithm must match oop.inline.hpp encode_heap_oop.
 6173 void MacroAssembler::encode_heap_oop(Register r) {
 6174 #ifdef ASSERT
 6175   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
 6176 #endif
 6177   verify_oop_msg(r, "broken oop in encode_heap_oop");
 6178   if (CompressedOops::base() == nullptr) {
 6179     if (CompressedOops::shift() != 0) {
 6180       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6181       shrq(r, LogMinObjAlignmentInBytes);
 6182     }
 6183     return;
 6184   }
 6185   testq(r, r);
 6186   cmovq(Assembler::equal, r, r12_heapbase);
 6187   subq(r, r12_heapbase);
 6188   shrq(r, LogMinObjAlignmentInBytes);
 6189 }
 6190 
 6191 void MacroAssembler::encode_heap_oop_not_null(Register r) {
 6192 #ifdef ASSERT
 6193   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
 6194   if (CheckCompressedOops) {
 6195     Label ok;
 6196     testq(r, r);
 6197     jcc(Assembler::notEqual, ok);
 6198     STOP("null oop passed to encode_heap_oop_not_null");
 6199     bind(ok);
 6200   }
 6201 #endif
 6202   verify_oop_msg(r, "broken oop in encode_heap_oop_not_null");
 6203   if (CompressedOops::base() != nullptr) {
 6204     subq(r, r12_heapbase);
 6205   }
 6206   if (CompressedOops::shift() != 0) {
 6207     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6208     shrq(r, LogMinObjAlignmentInBytes);
 6209   }
 6210 }
 6211 
 6212 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
 6213 #ifdef ASSERT
 6214   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
 6215   if (CheckCompressedOops) {
 6216     Label ok;
 6217     testq(src, src);
 6218     jcc(Assembler::notEqual, ok);
 6219     STOP("null oop passed to encode_heap_oop_not_null2");
 6220     bind(ok);
 6221   }
 6222 #endif
 6223   verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2");
 6224   if (dst != src) {
 6225     movq(dst, src);
 6226   }
 6227   if (CompressedOops::base() != nullptr) {
 6228     subq(dst, r12_heapbase);
 6229   }
 6230   if (CompressedOops::shift() != 0) {
 6231     assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6232     shrq(dst, LogMinObjAlignmentInBytes);
 6233   }
 6234 }
 6235 
 6236 void  MacroAssembler::decode_heap_oop(Register r) {
 6237 #ifdef ASSERT
 6238   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
 6239 #endif
 6240   if (CompressedOops::base() == nullptr) {
 6241     if (CompressedOops::shift() != 0) {
 6242       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6243       shlq(r, LogMinObjAlignmentInBytes);
 6244     }
 6245   } else {
 6246     Label done;
 6247     shlq(r, LogMinObjAlignmentInBytes);
 6248     jccb(Assembler::equal, done);
 6249     addq(r, r12_heapbase);
 6250     bind(done);
 6251   }
 6252   verify_oop_msg(r, "broken oop in decode_heap_oop");
 6253 }
 6254 
 6255 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
 6256   // Note: it will change flags
 6257   assert (UseCompressedOops, "should only be used for compressed headers");
 6258   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6259   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6260   // vtableStubs also counts instructions in pd_code_size_limit.
 6261   // Also do not verify_oop as this is called by verify_oop.
 6262   if (CompressedOops::shift() != 0) {
 6263     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6264     shlq(r, LogMinObjAlignmentInBytes);
 6265     if (CompressedOops::base() != nullptr) {
 6266       addq(r, r12_heapbase);
 6267     }
 6268   } else {
 6269     assert (CompressedOops::base() == nullptr, "sanity");
 6270   }
 6271 }
 6272 
 6273 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
 6274   // Note: it will change flags
 6275   assert (UseCompressedOops, "should only be used for compressed headers");
 6276   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6277   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6278   // vtableStubs also counts instructions in pd_code_size_limit.
 6279   // Also do not verify_oop as this is called by verify_oop.
 6280   if (CompressedOops::shift() != 0) {
 6281     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
 6282     if (LogMinObjAlignmentInBytes == Address::times_8) {
 6283       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
 6284     } else {
 6285       if (dst != src) {
 6286         movq(dst, src);
 6287       }
 6288       shlq(dst, LogMinObjAlignmentInBytes);
 6289       if (CompressedOops::base() != nullptr) {
 6290         addq(dst, r12_heapbase);
 6291       }
 6292     }
 6293   } else {
 6294     assert (CompressedOops::base() == nullptr, "sanity");
 6295     if (dst != src) {
 6296       movq(dst, src);
 6297     }
 6298   }
 6299 }
 6300 
 6301 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
 6302   assert_different_registers(r, tmp);
 6303   if (CompressedKlassPointers::base() != nullptr) {
 6304     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 6305     subq(r, tmp);
 6306   }
 6307   if (CompressedKlassPointers::shift() != 0) {
 6308     shrq(r, CompressedKlassPointers::shift());
 6309   }
 6310 }
 6311 
 6312 void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src) {
 6313   assert_different_registers(src, dst);
 6314   if (CompressedKlassPointers::base() != nullptr) {
 6315     mov64(dst, -(int64_t)CompressedKlassPointers::base());
 6316     addq(dst, src);
 6317   } else {
 6318     movptr(dst, src);
 6319   }
 6320   if (CompressedKlassPointers::shift() != 0) {
 6321     shrq(dst, CompressedKlassPointers::shift());
 6322   }
 6323 }
 6324 
 6325 void  MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
 6326   assert_different_registers(r, tmp);
 6327   // Note: it will change flags
 6328   assert(UseCompressedClassPointers, "should only be used for compressed headers");
 6329   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6330   // vtableStubs also counts instructions in pd_code_size_limit.
 6331   // Also do not verify_oop as this is called by verify_oop.
 6332   if (CompressedKlassPointers::shift() != 0) {
 6333     shlq(r, CompressedKlassPointers::shift());
 6334   }
 6335   if (CompressedKlassPointers::base() != nullptr) {
 6336     mov64(tmp, (int64_t)CompressedKlassPointers::base());
 6337     addq(r, tmp);
 6338   }
 6339 }
 6340 
 6341 void  MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src) {
 6342   assert_different_registers(src, dst);
 6343   // Note: it will change flags
 6344   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6345   // Cannot assert, unverified entry point counts instructions (see .ad file)
 6346   // vtableStubs also counts instructions in pd_code_size_limit.
 6347   // Also do not verify_oop as this is called by verify_oop.
 6348 
 6349   if (CompressedKlassPointers::base() == nullptr &&
 6350       CompressedKlassPointers::shift() == 0) {
 6351     // The best case scenario is that there is no base or shift. Then it is already
 6352     // a pointer that needs nothing but a register rename.
 6353     movl(dst, src);
 6354   } else {
 6355     if (CompressedKlassPointers::shift() <= Address::times_8) {
 6356       if (CompressedKlassPointers::base() != nullptr) {
 6357         mov64(dst, (int64_t)CompressedKlassPointers::base());
 6358       } else {
 6359         xorq(dst, dst);
 6360       }
 6361       if (CompressedKlassPointers::shift() != 0) {
 6362         assert(CompressedKlassPointers::shift() == Address::times_8, "klass not aligned on 64bits?");
 6363         leaq(dst, Address(dst, src, Address::times_8, 0));
 6364       } else {
 6365         addq(dst, src);
 6366       }
 6367     } else {
 6368       if (CompressedKlassPointers::base() != nullptr) {
 6369         const uint64_t base_right_shifted =
 6370             (uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
 6371         mov64(dst, base_right_shifted);
 6372       } else {
 6373         xorq(dst, dst);
 6374       }
 6375       addq(dst, src);
 6376       shlq(dst, CompressedKlassPointers::shift());
 6377     }
 6378   }
 6379 }
 6380 
 6381 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
 6382   assert (UseCompressedOops, "should only be used for compressed headers");
 6383   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6384   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6385   int oop_index = oop_recorder()->find_index(obj);
 6386   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6387   mov_narrow_oop(dst, oop_index, rspec);
 6388 }
 6389 
 6390 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
 6391   assert (UseCompressedOops, "should only be used for compressed headers");
 6392   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6393   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6394   int oop_index = oop_recorder()->find_index(obj);
 6395   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6396   mov_narrow_oop(dst, oop_index, rspec);
 6397 }
 6398 
 6399 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
 6400   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6401   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6402   int klass_index = oop_recorder()->find_index(k);
 6403   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6404   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6405 }
 6406 
 6407 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
 6408   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6409   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6410   int klass_index = oop_recorder()->find_index(k);
 6411   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6412   mov_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6413 }
 6414 
 6415 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
 6416   assert (UseCompressedOops, "should only be used for compressed headers");
 6417   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6418   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6419   int oop_index = oop_recorder()->find_index(obj);
 6420   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6421   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6422 }
 6423 
 6424 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
 6425   assert (UseCompressedOops, "should only be used for compressed headers");
 6426   assert (Universe::heap() != nullptr, "java heap should be initialized");
 6427   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6428   int oop_index = oop_recorder()->find_index(obj);
 6429   RelocationHolder rspec = oop_Relocation::spec(oop_index);
 6430   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 6431 }
 6432 
 6433 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
 6434   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6435   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6436   int klass_index = oop_recorder()->find_index(k);
 6437   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6438   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6439 }
 6440 
 6441 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
 6442   assert (UseCompressedClassPointers, "should only be used for compressed headers");
 6443   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 6444   int klass_index = oop_recorder()->find_index(k);
 6445   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 6446   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 6447 }
 6448 
 6449 void MacroAssembler::reinit_heapbase() {
 6450   if (UseCompressedOops) {
 6451     if (Universe::heap() != nullptr) {
 6452       if (CompressedOops::base() == nullptr) {
 6453         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 6454       } else {
 6455         mov64(r12_heapbase, (int64_t)CompressedOops::base());
 6456       }
 6457     } else {
 6458       movptr(r12_heapbase, ExternalAddress(CompressedOops::base_addr()));
 6459     }
 6460   }
 6461 }
 6462 
 6463 #endif // _LP64
 6464 
 6465 #if COMPILER2_OR_JVMCI
 6466 
 6467 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 6468 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6469   // cnt - number of qwords (8-byte words).
 6470   // base - start address, qword aligned.
 6471   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 6472   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 6473   if (use64byteVector) {
 6474     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
 6475   } else if (MaxVectorSize >= 32) {
 6476     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
 6477   } else {
 6478     pxor(xtmp, xtmp);
 6479   }
 6480   jmp(L_zero_64_bytes);
 6481 
 6482   BIND(L_loop);
 6483   if (MaxVectorSize >= 32) {
 6484     fill64(base, 0, xtmp, use64byteVector);
 6485   } else {
 6486     movdqu(Address(base,  0), xtmp);
 6487     movdqu(Address(base, 16), xtmp);
 6488     movdqu(Address(base, 32), xtmp);
 6489     movdqu(Address(base, 48), xtmp);
 6490   }
 6491   addptr(base, 64);
 6492 
 6493   BIND(L_zero_64_bytes);
 6494   subptr(cnt, 8);
 6495   jccb(Assembler::greaterEqual, L_loop);
 6496 
 6497   // Copy trailing 64 bytes
 6498   if (use64byteVector) {
 6499     addptr(cnt, 8);
 6500     jccb(Assembler::equal, L_end);
 6501     fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true);
 6502     jmp(L_end);
 6503   } else {
 6504     addptr(cnt, 4);
 6505     jccb(Assembler::less, L_tail);
 6506     if (MaxVectorSize >= 32) {
 6507       vmovdqu(Address(base, 0), xtmp);
 6508     } else {
 6509       movdqu(Address(base,  0), xtmp);
 6510       movdqu(Address(base, 16), xtmp);
 6511     }
 6512   }
 6513   addptr(base, 32);
 6514   subptr(cnt, 4);
 6515 
 6516   BIND(L_tail);
 6517   addptr(cnt, 4);
 6518   jccb(Assembler::lessEqual, L_end);
 6519   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 6520     fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp);
 6521   } else {
 6522     decrement(cnt);
 6523 
 6524     BIND(L_sloop);
 6525     movq(Address(base, 0), xtmp);
 6526     addptr(base, 8);
 6527     decrement(cnt);
 6528     jccb(Assembler::greaterEqual, L_sloop);
 6529   }
 6530   BIND(L_end);
 6531 }
 6532 
 6533 // Clearing constant sized memory using YMM/ZMM registers.
 6534 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6535   assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
 6536   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 6537 
 6538   int vector64_count = (cnt & (~0x7)) >> 3;
 6539   cnt = cnt & 0x7;
 6540   const int fill64_per_loop = 4;
 6541   const int max_unrolled_fill64 = 8;
 6542 
 6543   // 64 byte initialization loop.
 6544   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 6545   int start64 = 0;
 6546   if (vector64_count > max_unrolled_fill64) {
 6547     Label LOOP;
 6548     Register index = rtmp;
 6549 
 6550     start64 = vector64_count - (vector64_count % fill64_per_loop);
 6551 
 6552     movl(index, 0);
 6553     BIND(LOOP);
 6554     for (int i = 0; i < fill64_per_loop; i++) {
 6555       fill64(Address(base, index, Address::times_1, i * 64), xtmp, use64byteVector);
 6556     }
 6557     addl(index, fill64_per_loop * 64);
 6558     cmpl(index, start64 * 64);
 6559     jccb(Assembler::less, LOOP);
 6560   }
 6561   for (int i = start64; i < vector64_count; i++) {
 6562     fill64(base, i * 64, xtmp, use64byteVector);
 6563   }
 6564 
 6565   // Clear remaining 64 byte tail.
 6566   int disp = vector64_count * 64;
 6567   if (cnt) {
 6568     switch (cnt) {
 6569       case 1:
 6570         movq(Address(base, disp), xtmp);
 6571         break;
 6572       case 2:
 6573         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_128bit);
 6574         break;
 6575       case 3:
 6576         movl(rtmp, 0x7);
 6577         kmovwl(mask, rtmp);
 6578         evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_256bit);
 6579         break;
 6580       case 4:
 6581         evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6582         break;
 6583       case 5:
 6584         if (use64byteVector) {
 6585           movl(rtmp, 0x1F);
 6586           kmovwl(mask, rtmp);
 6587           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6588         } else {
 6589           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6590           movq(Address(base, disp + 32), xtmp);
 6591         }
 6592         break;
 6593       case 6:
 6594         if (use64byteVector) {
 6595           movl(rtmp, 0x3F);
 6596           kmovwl(mask, rtmp);
 6597           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6598         } else {
 6599           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6600           evmovdqu(T_LONG, k0, Address(base, disp + 32), xtmp, false, Assembler::AVX_128bit);
 6601         }
 6602         break;
 6603       case 7:
 6604         if (use64byteVector) {
 6605           movl(rtmp, 0x7F);
 6606           kmovwl(mask, rtmp);
 6607           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6608         } else {
 6609           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6610           movl(rtmp, 0x7);
 6611           kmovwl(mask, rtmp);
 6612           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
 6613         }
 6614         break;
 6615       default:
 6616         fatal("Unexpected length : %d\n",cnt);
 6617         break;
 6618     }
 6619   }
 6620 }
 6621 
 6622 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
 6623                                bool is_large, KRegister mask) {
 6624   // cnt      - number of qwords (8-byte words).
 6625   // base     - start address, qword aligned.
 6626   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 6627   assert(base==rdi, "base register must be edi for rep stos");
 6628   assert(tmp==rax,   "tmp register must be eax for rep stos");
 6629   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 6630   assert(InitArrayShortSize % BytesPerLong == 0,
 6631     "InitArrayShortSize should be the multiple of BytesPerLong");
 6632 
 6633   Label DONE;
 6634   if (!is_large || !UseXMMForObjInit) {
 6635     xorptr(tmp, tmp);
 6636   }
 6637 
 6638   if (!is_large) {
 6639     Label LOOP, LONG;
 6640     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 6641     jccb(Assembler::greater, LONG);
 6642 
 6643     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6644 
 6645     decrement(cnt);
 6646     jccb(Assembler::negative, DONE); // Zero length
 6647 
 6648     // Use individual pointer-sized stores for small counts:
 6649     BIND(LOOP);
 6650     movptr(Address(base, cnt, Address::times_ptr), tmp);
 6651     decrement(cnt);
 6652     jccb(Assembler::greaterEqual, LOOP);
 6653     jmpb(DONE);
 6654 
 6655     BIND(LONG);
 6656   }
 6657 
 6658   // Use longer rep-prefixed ops for non-small counts:
 6659   if (UseFastStosb) {
 6660     shlptr(cnt, 3); // convert to number of bytes
 6661     rep_stosb();
 6662   } else if (UseXMMForObjInit) {
 6663     xmm_clear_mem(base, cnt, tmp, xtmp, mask);
 6664   } else {
 6665     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6666     rep_stos();
 6667   }
 6668 
 6669   BIND(DONE);
 6670 }
 6671 
 6672 #endif //COMPILER2_OR_JVMCI
 6673 
 6674 
 6675 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 6676                                    Register to, Register value, Register count,
 6677                                    Register rtmp, XMMRegister xtmp) {
 6678   ShortBranchVerifier sbv(this);
 6679   assert_different_registers(to, value, count, rtmp);
 6680   Label L_exit;
 6681   Label L_fill_2_bytes, L_fill_4_bytes;
 6682 
 6683 #if defined(COMPILER2) && defined(_LP64)
 6684   if(MaxVectorSize >=32 &&
 6685      VM_Version::supports_avx512vlbw() &&
 6686      VM_Version::supports_bmi2()) {
 6687     generate_fill_avx3(t, to, value, count, rtmp, xtmp);
 6688     return;
 6689   }
 6690 #endif
 6691 
 6692   int shift = -1;
 6693   switch (t) {
 6694     case T_BYTE:
 6695       shift = 2;
 6696       break;
 6697     case T_SHORT:
 6698       shift = 1;
 6699       break;
 6700     case T_INT:
 6701       shift = 0;
 6702       break;
 6703     default: ShouldNotReachHere();
 6704   }
 6705 
 6706   if (t == T_BYTE) {
 6707     andl(value, 0xff);
 6708     movl(rtmp, value);
 6709     shll(rtmp, 8);
 6710     orl(value, rtmp);
 6711   }
 6712   if (t == T_SHORT) {
 6713     andl(value, 0xffff);
 6714   }
 6715   if (t == T_BYTE || t == T_SHORT) {
 6716     movl(rtmp, value);
 6717     shll(rtmp, 16);
 6718     orl(value, rtmp);
 6719   }
 6720 
 6721   cmpptr(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
 6722   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
 6723   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 6724     Label L_skip_align2;
 6725     // align source address at 4 bytes address boundary
 6726     if (t == T_BYTE) {
 6727       Label L_skip_align1;
 6728       // One byte misalignment happens only for byte arrays
 6729       testptr(to, 1);
 6730       jccb(Assembler::zero, L_skip_align1);
 6731       movb(Address(to, 0), value);
 6732       increment(to);
 6733       decrement(count);
 6734       BIND(L_skip_align1);
 6735     }
 6736     // Two bytes misalignment happens only for byte and short (char) arrays
 6737     testptr(to, 2);
 6738     jccb(Assembler::zero, L_skip_align2);
 6739     movw(Address(to, 0), value);
 6740     addptr(to, 2);
 6741     subptr(count, 1<<(shift-1));
 6742     BIND(L_skip_align2);
 6743   }
 6744   if (UseSSE < 2) {
 6745     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6746     // Fill 32-byte chunks
 6747     subptr(count, 8 << shift);
 6748     jcc(Assembler::less, L_check_fill_8_bytes);
 6749     align(16);
 6750 
 6751     BIND(L_fill_32_bytes_loop);
 6752 
 6753     for (int i = 0; i < 32; i += 4) {
 6754       movl(Address(to, i), value);
 6755     }
 6756 
 6757     addptr(to, 32);
 6758     subptr(count, 8 << shift);
 6759     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6760     BIND(L_check_fill_8_bytes);
 6761     addptr(count, 8 << shift);
 6762     jccb(Assembler::zero, L_exit);
 6763     jmpb(L_fill_8_bytes);
 6764 
 6765     //
 6766     // length is too short, just fill qwords
 6767     //
 6768     BIND(L_fill_8_bytes_loop);
 6769     movl(Address(to, 0), value);
 6770     movl(Address(to, 4), value);
 6771     addptr(to, 8);
 6772     BIND(L_fill_8_bytes);
 6773     subptr(count, 1 << (shift + 1));
 6774     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6775     // fall through to fill 4 bytes
 6776   } else {
 6777     Label L_fill_32_bytes;
 6778     if (!UseUnalignedLoadStores) {
 6779       // align to 8 bytes, we know we are 4 byte aligned to start
 6780       testptr(to, 4);
 6781       jccb(Assembler::zero, L_fill_32_bytes);
 6782       movl(Address(to, 0), value);
 6783       addptr(to, 4);
 6784       subptr(count, 1<<shift);
 6785     }
 6786     BIND(L_fill_32_bytes);
 6787     {
 6788       assert( UseSSE >= 2, "supported cpu only" );
 6789       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
 6790       movdl(xtmp, value);
 6791       if (UseAVX >= 2 && UseUnalignedLoadStores) {
 6792         Label L_check_fill_32_bytes;
 6793         if (UseAVX > 2) {
 6794           // Fill 64-byte chunks
 6795           Label L_fill_64_bytes_loop_avx3, L_check_fill_64_bytes_avx2;
 6796 
 6797           // If number of bytes to fill < VM_Version::avx3_threshold(), perform fill using AVX2
 6798           cmpptr(count, VM_Version::avx3_threshold());
 6799           jccb(Assembler::below, L_check_fill_64_bytes_avx2);
 6800 
 6801           vpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
 6802 
 6803           subptr(count, 16 << shift);
 6804           jccb(Assembler::less, L_check_fill_32_bytes);
 6805           align(16);
 6806 
 6807           BIND(L_fill_64_bytes_loop_avx3);
 6808           evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
 6809           addptr(to, 64);
 6810           subptr(count, 16 << shift);
 6811           jcc(Assembler::greaterEqual, L_fill_64_bytes_loop_avx3);
 6812           jmpb(L_check_fill_32_bytes);
 6813 
 6814           BIND(L_check_fill_64_bytes_avx2);
 6815         }
 6816         // Fill 64-byte chunks
 6817         Label L_fill_64_bytes_loop;
 6818         vpbroadcastd(xtmp, xtmp, Assembler::AVX_256bit);
 6819 
 6820         subptr(count, 16 << shift);
 6821         jcc(Assembler::less, L_check_fill_32_bytes);
 6822         align(16);
 6823 
 6824         BIND(L_fill_64_bytes_loop);
 6825         vmovdqu(Address(to, 0), xtmp);
 6826         vmovdqu(Address(to, 32), xtmp);
 6827         addptr(to, 64);
 6828         subptr(count, 16 << shift);
 6829         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
 6830 
 6831         BIND(L_check_fill_32_bytes);
 6832         addptr(count, 8 << shift);
 6833         jccb(Assembler::less, L_check_fill_8_bytes);
 6834         vmovdqu(Address(to, 0), xtmp);
 6835         addptr(to, 32);
 6836         subptr(count, 8 << shift);
 6837 
 6838         BIND(L_check_fill_8_bytes);
 6839         // clean upper bits of YMM registers
 6840         movdl(xtmp, value);
 6841         pshufd(xtmp, xtmp, 0);
 6842       } else {
 6843         // Fill 32-byte chunks
 6844         pshufd(xtmp, xtmp, 0);
 6845 
 6846         subptr(count, 8 << shift);
 6847         jcc(Assembler::less, L_check_fill_8_bytes);
 6848         align(16);
 6849 
 6850         BIND(L_fill_32_bytes_loop);
 6851 
 6852         if (UseUnalignedLoadStores) {
 6853           movdqu(Address(to, 0), xtmp);
 6854           movdqu(Address(to, 16), xtmp);
 6855         } else {
 6856           movq(Address(to, 0), xtmp);
 6857           movq(Address(to, 8), xtmp);
 6858           movq(Address(to, 16), xtmp);
 6859           movq(Address(to, 24), xtmp);
 6860         }
 6861 
 6862         addptr(to, 32);
 6863         subptr(count, 8 << shift);
 6864         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
 6865 
 6866         BIND(L_check_fill_8_bytes);
 6867       }
 6868       addptr(count, 8 << shift);
 6869       jccb(Assembler::zero, L_exit);
 6870       jmpb(L_fill_8_bytes);
 6871 
 6872       //
 6873       // length is too short, just fill qwords
 6874       //
 6875       BIND(L_fill_8_bytes_loop);
 6876       movq(Address(to, 0), xtmp);
 6877       addptr(to, 8);
 6878       BIND(L_fill_8_bytes);
 6879       subptr(count, 1 << (shift + 1));
 6880       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
 6881     }
 6882   }
 6883   // fill trailing 4 bytes
 6884   BIND(L_fill_4_bytes);
 6885   testl(count, 1<<shift);
 6886   jccb(Assembler::zero, L_fill_2_bytes);
 6887   movl(Address(to, 0), value);
 6888   if (t == T_BYTE || t == T_SHORT) {
 6889     Label L_fill_byte;
 6890     addptr(to, 4);
 6891     BIND(L_fill_2_bytes);
 6892     // fill trailing 2 bytes
 6893     testl(count, 1<<(shift-1));
 6894     jccb(Assembler::zero, L_fill_byte);
 6895     movw(Address(to, 0), value);
 6896     if (t == T_BYTE) {
 6897       addptr(to, 2);
 6898       BIND(L_fill_byte);
 6899       // fill trailing byte
 6900       testl(count, 1);
 6901       jccb(Assembler::zero, L_exit);
 6902       movb(Address(to, 0), value);
 6903     } else {
 6904       BIND(L_fill_byte);
 6905     }
 6906   } else {
 6907     BIND(L_fill_2_bytes);
 6908   }
 6909   BIND(L_exit);
 6910 }
 6911 
 6912 void MacroAssembler::evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len) {
 6913   switch(type) {
 6914     case T_BYTE:
 6915     case T_BOOLEAN:
 6916       evpbroadcastb(dst, src, vector_len);
 6917       break;
 6918     case T_SHORT:
 6919     case T_CHAR:
 6920       evpbroadcastw(dst, src, vector_len);
 6921       break;
 6922     case T_INT:
 6923     case T_FLOAT:
 6924       evpbroadcastd(dst, src, vector_len);
 6925       break;
 6926     case T_LONG:
 6927     case T_DOUBLE:
 6928       evpbroadcastq(dst, src, vector_len);
 6929       break;
 6930     default:
 6931       fatal("Unhandled type : %s", type2name(type));
 6932       break;
 6933   }
 6934 }
 6935 
 6936 // encode char[] to byte[] in ISO_8859_1 or ASCII
 6937    //@IntrinsicCandidate
 6938    //private static int implEncodeISOArray(byte[] sa, int sp,
 6939    //byte[] da, int dp, int len) {
 6940    //  int i = 0;
 6941    //  for (; i < len; i++) {
 6942    //    char c = StringUTF16.getChar(sa, sp++);
 6943    //    if (c > '\u00FF')
 6944    //      break;
 6945    //    da[dp++] = (byte)c;
 6946    //  }
 6947    //  return i;
 6948    //}
 6949    //
 6950    //@IntrinsicCandidate
 6951    //private static int implEncodeAsciiArray(char[] sa, int sp,
 6952    //    byte[] da, int dp, int len) {
 6953    //  int i = 0;
 6954    //  for (; i < len; i++) {
 6955    //    char c = sa[sp++];
 6956    //    if (c >= '\u0080')
 6957    //      break;
 6958    //    da[dp++] = (byte)c;
 6959    //  }
 6960    //  return i;
 6961    //}
 6962 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
 6963   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 6964   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 6965   Register tmp5, Register result, bool ascii) {
 6966 
 6967   // rsi: src
 6968   // rdi: dst
 6969   // rdx: len
 6970   // rcx: tmp5
 6971   // rax: result
 6972   ShortBranchVerifier sbv(this);
 6973   assert_different_registers(src, dst, len, tmp5, result);
 6974   Label L_done, L_copy_1_char, L_copy_1_char_exit;
 6975 
 6976   int mask = ascii ? 0xff80ff80 : 0xff00ff00;
 6977   int short_mask = ascii ? 0xff80 : 0xff00;
 6978 
 6979   // set result
 6980   xorl(result, result);
 6981   // check for zero length
 6982   testl(len, len);
 6983   jcc(Assembler::zero, L_done);
 6984 
 6985   movl(result, len);
 6986 
 6987   // Setup pointers
 6988   lea(src, Address(src, len, Address::times_2)); // char[]
 6989   lea(dst, Address(dst, len, Address::times_1)); // byte[]
 6990   negptr(len);
 6991 
 6992   if (UseSSE42Intrinsics || UseAVX >= 2) {
 6993     Label L_copy_8_chars, L_copy_8_chars_exit;
 6994     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
 6995 
 6996     if (UseAVX >= 2) {
 6997       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
 6998       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 6999       movdl(tmp1Reg, tmp5);
 7000       vpbroadcastd(tmp1Reg, tmp1Reg, Assembler::AVX_256bit);
 7001       jmp(L_chars_32_check);
 7002 
 7003       bind(L_copy_32_chars);
 7004       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
 7005       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
 7006       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 7007       vptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 7008       jccb(Assembler::notZero, L_copy_32_chars_exit);
 7009       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
 7010       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
 7011       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
 7012 
 7013       bind(L_chars_32_check);
 7014       addptr(len, 32);
 7015       jcc(Assembler::lessEqual, L_copy_32_chars);
 7016 
 7017       bind(L_copy_32_chars_exit);
 7018       subptr(len, 16);
 7019       jccb(Assembler::greater, L_copy_16_chars_exit);
 7020 
 7021     } else if (UseSSE42Intrinsics) {
 7022       movl(tmp5, mask);   // create mask to test for Unicode or non-ASCII chars in vector
 7023       movdl(tmp1Reg, tmp5);
 7024       pshufd(tmp1Reg, tmp1Reg, 0);
 7025       jmpb(L_chars_16_check);
 7026     }
 7027 
 7028     bind(L_copy_16_chars);
 7029     if (UseAVX >= 2) {
 7030       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
 7031       vptest(tmp2Reg, tmp1Reg);
 7032       jcc(Assembler::notZero, L_copy_16_chars_exit);
 7033       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
 7034       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
 7035     } else {
 7036       if (UseAVX > 0) {
 7037         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 7038         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 7039         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
 7040       } else {
 7041         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
 7042         por(tmp2Reg, tmp3Reg);
 7043         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
 7044         por(tmp2Reg, tmp4Reg);
 7045       }
 7046       ptest(tmp2Reg, tmp1Reg);       // check for Unicode or non-ASCII chars in vector
 7047       jccb(Assembler::notZero, L_copy_16_chars_exit);
 7048       packuswb(tmp3Reg, tmp4Reg);
 7049     }
 7050     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
 7051 
 7052     bind(L_chars_16_check);
 7053     addptr(len, 16);
 7054     jcc(Assembler::lessEqual, L_copy_16_chars);
 7055 
 7056     bind(L_copy_16_chars_exit);
 7057     if (UseAVX >= 2) {
 7058       // clean upper bits of YMM registers
 7059       vpxor(tmp2Reg, tmp2Reg);
 7060       vpxor(tmp3Reg, tmp3Reg);
 7061       vpxor(tmp4Reg, tmp4Reg);
 7062       movdl(tmp1Reg, tmp5);
 7063       pshufd(tmp1Reg, tmp1Reg, 0);
 7064     }
 7065     subptr(len, 8);
 7066     jccb(Assembler::greater, L_copy_8_chars_exit);
 7067 
 7068     bind(L_copy_8_chars);
 7069     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
 7070     ptest(tmp3Reg, tmp1Reg);
 7071     jccb(Assembler::notZero, L_copy_8_chars_exit);
 7072     packuswb(tmp3Reg, tmp1Reg);
 7073     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
 7074     addptr(len, 8);
 7075     jccb(Assembler::lessEqual, L_copy_8_chars);
 7076 
 7077     bind(L_copy_8_chars_exit);
 7078     subptr(len, 8);
 7079     jccb(Assembler::zero, L_done);
 7080   }
 7081 
 7082   bind(L_copy_1_char);
 7083   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
 7084   testl(tmp5, short_mask);      // check if Unicode or non-ASCII char
 7085   jccb(Assembler::notZero, L_copy_1_char_exit);
 7086   movb(Address(dst, len, Address::times_1, 0), tmp5);
 7087   addptr(len, 1);
 7088   jccb(Assembler::less, L_copy_1_char);
 7089 
 7090   bind(L_copy_1_char_exit);
 7091   addptr(result, len); // len is negative count of not processed elements
 7092 
 7093   bind(L_done);
 7094 }
 7095 
 7096 #ifdef _LP64
 7097 /**
 7098  * Helper for multiply_to_len().
 7099  */
 7100 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
 7101   addq(dest_lo, src1);
 7102   adcq(dest_hi, 0);
 7103   addq(dest_lo, src2);
 7104   adcq(dest_hi, 0);
 7105 }
 7106 
 7107 /**
 7108  * Multiply 64 bit by 64 bit first loop.
 7109  */
 7110 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
 7111                                            Register y, Register y_idx, Register z,
 7112                                            Register carry, Register product,
 7113                                            Register idx, Register kdx) {
 7114   //
 7115   //  jlong carry, x[], y[], z[];
 7116   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7117   //    huge_128 product = y[idx] * x[xstart] + carry;
 7118   //    z[kdx] = (jlong)product;
 7119   //    carry  = (jlong)(product >>> 64);
 7120   //  }
 7121   //  z[xstart] = carry;
 7122   //
 7123 
 7124   Label L_first_loop, L_first_loop_exit;
 7125   Label L_one_x, L_one_y, L_multiply;
 7126 
 7127   decrementl(xstart);
 7128   jcc(Assembler::negative, L_one_x);
 7129 
 7130   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7131   rorq(x_xstart, 32); // convert big-endian to little-endian
 7132 
 7133   bind(L_first_loop);
 7134   decrementl(idx);
 7135   jcc(Assembler::negative, L_first_loop_exit);
 7136   decrementl(idx);
 7137   jcc(Assembler::negative, L_one_y);
 7138   movq(y_idx, Address(y, idx, Address::times_4,  0));
 7139   rorq(y_idx, 32); // convert big-endian to little-endian
 7140   bind(L_multiply);
 7141   movq(product, x_xstart);
 7142   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
 7143   addq(product, carry);
 7144   adcq(rdx, 0);
 7145   subl(kdx, 2);
 7146   movl(Address(z, kdx, Address::times_4,  4), product);
 7147   shrq(product, 32);
 7148   movl(Address(z, kdx, Address::times_4,  0), product);
 7149   movq(carry, rdx);
 7150   jmp(L_first_loop);
 7151 
 7152   bind(L_one_y);
 7153   movl(y_idx, Address(y,  0));
 7154   jmp(L_multiply);
 7155 
 7156   bind(L_one_x);
 7157   movl(x_xstart, Address(x,  0));
 7158   jmp(L_first_loop);
 7159 
 7160   bind(L_first_loop_exit);
 7161 }
 7162 
 7163 /**
 7164  * Multiply 64 bit by 64 bit and add 128 bit.
 7165  */
 7166 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
 7167                                             Register yz_idx, Register idx,
 7168                                             Register carry, Register product, int offset) {
 7169   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
 7170   //     z[kdx] = (jlong)product;
 7171 
 7172   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
 7173   rorq(yz_idx, 32); // convert big-endian to little-endian
 7174   movq(product, x_xstart);
 7175   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
 7176   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
 7177   rorq(yz_idx, 32); // convert big-endian to little-endian
 7178 
 7179   add2_with_carry(rdx, product, carry, yz_idx);
 7180 
 7181   movl(Address(z, idx, Address::times_4,  offset+4), product);
 7182   shrq(product, 32);
 7183   movl(Address(z, idx, Address::times_4,  offset), product);
 7184 
 7185 }
 7186 
 7187 /**
 7188  * Multiply 128 bit by 128 bit. Unrolled inner loop.
 7189  */
 7190 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
 7191                                              Register yz_idx, Register idx, Register jdx,
 7192                                              Register carry, Register product,
 7193                                              Register carry2) {
 7194   //   jlong carry, x[], y[], z[];
 7195   //   int kdx = ystart+1;
 7196   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 7197   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
 7198   //     z[kdx+idx+1] = (jlong)product;
 7199   //     jlong carry2  = (jlong)(product >>> 64);
 7200   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
 7201   //     z[kdx+idx] = (jlong)product;
 7202   //     carry  = (jlong)(product >>> 64);
 7203   //   }
 7204   //   idx += 2;
 7205   //   if (idx > 0) {
 7206   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
 7207   //     z[kdx+idx] = (jlong)product;
 7208   //     carry  = (jlong)(product >>> 64);
 7209   //   }
 7210   //
 7211 
 7212   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 7213 
 7214   movl(jdx, idx);
 7215   andl(jdx, 0xFFFFFFFC);
 7216   shrl(jdx, 2);
 7217 
 7218   bind(L_third_loop);
 7219   subl(jdx, 1);
 7220   jcc(Assembler::negative, L_third_loop_exit);
 7221   subl(idx, 4);
 7222 
 7223   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
 7224   movq(carry2, rdx);
 7225 
 7226   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
 7227   movq(carry, rdx);
 7228   jmp(L_third_loop);
 7229 
 7230   bind (L_third_loop_exit);
 7231 
 7232   andl (idx, 0x3);
 7233   jcc(Assembler::zero, L_post_third_loop_done);
 7234 
 7235   Label L_check_1;
 7236   subl(idx, 2);
 7237   jcc(Assembler::negative, L_check_1);
 7238 
 7239   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
 7240   movq(carry, rdx);
 7241 
 7242   bind (L_check_1);
 7243   addl (idx, 0x2);
 7244   andl (idx, 0x1);
 7245   subl(idx, 1);
 7246   jcc(Assembler::negative, L_post_third_loop_done);
 7247 
 7248   movl(yz_idx, Address(y, idx, Address::times_4,  0));
 7249   movq(product, x_xstart);
 7250   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
 7251   movl(yz_idx, Address(z, idx, Address::times_4,  0));
 7252 
 7253   add2_with_carry(rdx, product, yz_idx, carry);
 7254 
 7255   movl(Address(z, idx, Address::times_4,  0), product);
 7256   shrq(product, 32);
 7257 
 7258   shlq(rdx, 32);
 7259   orq(product, rdx);
 7260   movq(carry, product);
 7261 
 7262   bind(L_post_third_loop_done);
 7263 }
 7264 
 7265 /**
 7266  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
 7267  *
 7268  */
 7269 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
 7270                                                   Register carry, Register carry2,
 7271                                                   Register idx, Register jdx,
 7272                                                   Register yz_idx1, Register yz_idx2,
 7273                                                   Register tmp, Register tmp3, Register tmp4) {
 7274   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
 7275 
 7276   //   jlong carry, x[], y[], z[];
 7277   //   int kdx = ystart+1;
 7278   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
 7279   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
 7280   //     jlong carry2  = (jlong)(tmp3 >>> 64);
 7281   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
 7282   //     carry  = (jlong)(tmp4 >>> 64);
 7283   //     z[kdx+idx+1] = (jlong)tmp3;
 7284   //     z[kdx+idx] = (jlong)tmp4;
 7285   //   }
 7286   //   idx += 2;
 7287   //   if (idx > 0) {
 7288   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
 7289   //     z[kdx+idx] = (jlong)yz_idx1;
 7290   //     carry  = (jlong)(yz_idx1 >>> 64);
 7291   //   }
 7292   //
 7293 
 7294   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
 7295 
 7296   movl(jdx, idx);
 7297   andl(jdx, 0xFFFFFFFC);
 7298   shrl(jdx, 2);
 7299 
 7300   bind(L_third_loop);
 7301   subl(jdx, 1);
 7302   jcc(Assembler::negative, L_third_loop_exit);
 7303   subl(idx, 4);
 7304 
 7305   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
 7306   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
 7307   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
 7308   rorxq(yz_idx2, yz_idx2, 32);
 7309 
 7310   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
 7311   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
 7312 
 7313   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
 7314   rorxq(yz_idx1, yz_idx1, 32);
 7315   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 7316   rorxq(yz_idx2, yz_idx2, 32);
 7317 
 7318   if (VM_Version::supports_adx()) {
 7319     adcxq(tmp3, carry);
 7320     adoxq(tmp3, yz_idx1);
 7321 
 7322     adcxq(tmp4, tmp);
 7323     adoxq(tmp4, yz_idx2);
 7324 
 7325     movl(carry, 0); // does not affect flags
 7326     adcxq(carry2, carry);
 7327     adoxq(carry2, carry);
 7328   } else {
 7329     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
 7330     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
 7331   }
 7332   movq(carry, carry2);
 7333 
 7334   movl(Address(z, idx, Address::times_4, 12), tmp3);
 7335   shrq(tmp3, 32);
 7336   movl(Address(z, idx, Address::times_4,  8), tmp3);
 7337 
 7338   movl(Address(z, idx, Address::times_4,  4), tmp4);
 7339   shrq(tmp4, 32);
 7340   movl(Address(z, idx, Address::times_4,  0), tmp4);
 7341 
 7342   jmp(L_third_loop);
 7343 
 7344   bind (L_third_loop_exit);
 7345 
 7346   andl (idx, 0x3);
 7347   jcc(Assembler::zero, L_post_third_loop_done);
 7348 
 7349   Label L_check_1;
 7350   subl(idx, 2);
 7351   jcc(Assembler::negative, L_check_1);
 7352 
 7353   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
 7354   rorxq(yz_idx1, yz_idx1, 32);
 7355   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
 7356   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
 7357   rorxq(yz_idx2, yz_idx2, 32);
 7358 
 7359   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
 7360 
 7361   movl(Address(z, idx, Address::times_4,  4), tmp3);
 7362   shrq(tmp3, 32);
 7363   movl(Address(z, idx, Address::times_4,  0), tmp3);
 7364   movq(carry, tmp4);
 7365 
 7366   bind (L_check_1);
 7367   addl (idx, 0x2);
 7368   andl (idx, 0x1);
 7369   subl(idx, 1);
 7370   jcc(Assembler::negative, L_post_third_loop_done);
 7371   movl(tmp4, Address(y, idx, Address::times_4,  0));
 7372   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
 7373   movl(tmp4, Address(z, idx, Address::times_4,  0));
 7374 
 7375   add2_with_carry(carry2, tmp3, tmp4, carry);
 7376 
 7377   movl(Address(z, idx, Address::times_4,  0), tmp3);
 7378   shrq(tmp3, 32);
 7379 
 7380   shlq(carry2, 32);
 7381   orq(tmp3, carry2);
 7382   movq(carry, tmp3);
 7383 
 7384   bind(L_post_third_loop_done);
 7385 }
 7386 
 7387 /**
 7388  * Code for BigInteger::multiplyToLen() intrinsic.
 7389  *
 7390  * rdi: x
 7391  * rax: xlen
 7392  * rsi: y
 7393  * rcx: ylen
 7394  * r8:  z
 7395  * r11: tmp0
 7396  * r12: tmp1
 7397  * r13: tmp2
 7398  * r14: tmp3
 7399  * r15: tmp4
 7400  * rbx: tmp5
 7401  *
 7402  */
 7403 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register tmp0,
 7404                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
 7405   ShortBranchVerifier sbv(this);
 7406   assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
 7407 
 7408   push(tmp0);
 7409   push(tmp1);
 7410   push(tmp2);
 7411   push(tmp3);
 7412   push(tmp4);
 7413   push(tmp5);
 7414 
 7415   push(xlen);
 7416 
 7417   const Register idx = tmp1;
 7418   const Register kdx = tmp2;
 7419   const Register xstart = tmp3;
 7420 
 7421   const Register y_idx = tmp4;
 7422   const Register carry = tmp5;
 7423   const Register product  = xlen;
 7424   const Register x_xstart = tmp0;
 7425 
 7426   // First Loop.
 7427   //
 7428   //  final static long LONG_MASK = 0xffffffffL;
 7429   //  int xstart = xlen - 1;
 7430   //  int ystart = ylen - 1;
 7431   //  long carry = 0;
 7432   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
 7433   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
 7434   //    z[kdx] = (int)product;
 7435   //    carry = product >>> 32;
 7436   //  }
 7437   //  z[xstart] = (int)carry;
 7438   //
 7439 
 7440   movl(idx, ylen);               // idx = ylen;
 7441   lea(kdx, Address(xlen, ylen)); // kdx = xlen+ylen;
 7442   xorq(carry, carry);            // carry = 0;
 7443 
 7444   Label L_done;
 7445 
 7446   movl(xstart, xlen);
 7447   decrementl(xstart);
 7448   jcc(Assembler::negative, L_done);
 7449 
 7450   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
 7451 
 7452   Label L_second_loop;
 7453   testl(kdx, kdx);
 7454   jcc(Assembler::zero, L_second_loop);
 7455 
 7456   Label L_carry;
 7457   subl(kdx, 1);
 7458   jcc(Assembler::zero, L_carry);
 7459 
 7460   movl(Address(z, kdx, Address::times_4,  0), carry);
 7461   shrq(carry, 32);
 7462   subl(kdx, 1);
 7463 
 7464   bind(L_carry);
 7465   movl(Address(z, kdx, Address::times_4,  0), carry);
 7466 
 7467   // Second and third (nested) loops.
 7468   //
 7469   // for (int i = xstart-1; i >= 0; i--) { // Second loop
 7470   //   carry = 0;
 7471   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
 7472   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
 7473   //                    (z[k] & LONG_MASK) + carry;
 7474   //     z[k] = (int)product;
 7475   //     carry = product >>> 32;
 7476   //   }
 7477   //   z[i] = (int)carry;
 7478   // }
 7479   //
 7480   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
 7481 
 7482   const Register jdx = tmp1;
 7483 
 7484   bind(L_second_loop);
 7485   xorl(carry, carry);    // carry = 0;
 7486   movl(jdx, ylen);       // j = ystart+1
 7487 
 7488   subl(xstart, 1);       // i = xstart-1;
 7489   jcc(Assembler::negative, L_done);
 7490 
 7491   push (z);
 7492 
 7493   Label L_last_x;
 7494   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
 7495   subl(xstart, 1);       // i = xstart-1;
 7496   jcc(Assembler::negative, L_last_x);
 7497 
 7498   if (UseBMI2Instructions) {
 7499     movq(rdx,  Address(x, xstart, Address::times_4,  0));
 7500     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
 7501   } else {
 7502     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
 7503     rorq(x_xstart, 32);  // convert big-endian to little-endian
 7504   }
 7505 
 7506   Label L_third_loop_prologue;
 7507   bind(L_third_loop_prologue);
 7508 
 7509   push (x);
 7510   push (xstart);
 7511   push (ylen);
 7512 
 7513 
 7514   if (UseBMI2Instructions) {
 7515     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
 7516   } else { // !UseBMI2Instructions
 7517     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
 7518   }
 7519 
 7520   pop(ylen);
 7521   pop(xlen);
 7522   pop(x);
 7523   pop(z);
 7524 
 7525   movl(tmp3, xlen);
 7526   addl(tmp3, 1);
 7527   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7528   subl(tmp3, 1);
 7529   jccb(Assembler::negative, L_done);
 7530 
 7531   shrq(carry, 32);
 7532   movl(Address(z, tmp3, Address::times_4,  0), carry);
 7533   jmp(L_second_loop);
 7534 
 7535   // Next infrequent code is moved outside loops.
 7536   bind(L_last_x);
 7537   if (UseBMI2Instructions) {
 7538     movl(rdx, Address(x,  0));
 7539   } else {
 7540     movl(x_xstart, Address(x,  0));
 7541   }
 7542   jmp(L_third_loop_prologue);
 7543 
 7544   bind(L_done);
 7545 
 7546   pop(xlen);
 7547 
 7548   pop(tmp5);
 7549   pop(tmp4);
 7550   pop(tmp3);
 7551   pop(tmp2);
 7552   pop(tmp1);
 7553   pop(tmp0);
 7554 }
 7555 
 7556 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
 7557   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
 7558   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
 7559   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
 7560   Label VECTOR8_TAIL, VECTOR4_TAIL;
 7561   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
 7562   Label SAME_TILL_END, DONE;
 7563   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
 7564 
 7565   //scale is in rcx in both Win64 and Unix
 7566   ShortBranchVerifier sbv(this);
 7567 
 7568   shlq(length);
 7569   xorq(result, result);
 7570 
 7571   if ((AVX3Threshold == 0) && (UseAVX > 2) &&
 7572       VM_Version::supports_avx512vlbw()) {
 7573     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
 7574 
 7575     cmpq(length, 64);
 7576     jcc(Assembler::less, VECTOR32_TAIL);
 7577 
 7578     movq(tmp1, length);
 7579     andq(tmp1, 0x3F);      // tail count
 7580     andq(length, ~(0x3F)); //vector count
 7581 
 7582     bind(VECTOR64_LOOP);
 7583     // AVX512 code to compare 64 byte vectors.
 7584     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
 7585     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7586     kortestql(k7, k7);
 7587     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
 7588     addq(result, 64);
 7589     subq(length, 64);
 7590     jccb(Assembler::notZero, VECTOR64_LOOP);
 7591 
 7592     //bind(VECTOR64_TAIL);
 7593     testq(tmp1, tmp1);
 7594     jcc(Assembler::zero, SAME_TILL_END);
 7595 
 7596     //bind(VECTOR64_TAIL);
 7597     // AVX512 code to compare up to 63 byte vectors.
 7598     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
 7599     shlxq(tmp2, tmp2, tmp1);
 7600     notq(tmp2);
 7601     kmovql(k3, tmp2);
 7602 
 7603     evmovdqub(rymm0, k3, Address(obja, result), false, Assembler::AVX_512bit);
 7604     evpcmpeqb(k7, k3, rymm0, Address(objb, result), Assembler::AVX_512bit);
 7605 
 7606     ktestql(k7, k3);
 7607     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
 7608 
 7609     bind(VECTOR64_NOT_EQUAL);
 7610     kmovql(tmp1, k7);
 7611     notq(tmp1);
 7612     tzcntq(tmp1, tmp1);
 7613     addq(result, tmp1);
 7614     shrq(result);
 7615     jmp(DONE);
 7616     bind(VECTOR32_TAIL);
 7617   }
 7618 
 7619   cmpq(length, 8);
 7620   jcc(Assembler::equal, VECTOR8_LOOP);
 7621   jcc(Assembler::less, VECTOR4_TAIL);
 7622 
 7623   if (UseAVX >= 2) {
 7624     Label VECTOR16_TAIL, VECTOR32_LOOP;
 7625 
 7626     cmpq(length, 16);
 7627     jcc(Assembler::equal, VECTOR16_LOOP);
 7628     jcc(Assembler::less, VECTOR8_LOOP);
 7629 
 7630     cmpq(length, 32);
 7631     jccb(Assembler::less, VECTOR16_TAIL);
 7632 
 7633     subq(length, 32);
 7634     bind(VECTOR32_LOOP);
 7635     vmovdqu(rymm0, Address(obja, result));
 7636     vmovdqu(rymm1, Address(objb, result));
 7637     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
 7638     vptest(rymm2, rymm2);
 7639     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
 7640     addq(result, 32);
 7641     subq(length, 32);
 7642     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
 7643     addq(length, 32);
 7644     jcc(Assembler::equal, SAME_TILL_END);
 7645     //falling through if less than 32 bytes left //close the branch here.
 7646 
 7647     bind(VECTOR16_TAIL);
 7648     cmpq(length, 16);
 7649     jccb(Assembler::less, VECTOR8_TAIL);
 7650     bind(VECTOR16_LOOP);
 7651     movdqu(rymm0, Address(obja, result));
 7652     movdqu(rymm1, Address(objb, result));
 7653     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
 7654     ptest(rymm2, rymm2);
 7655     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7656     addq(result, 16);
 7657     subq(length, 16);
 7658     jcc(Assembler::equal, SAME_TILL_END);
 7659     //falling through if less than 16 bytes left
 7660   } else {//regular intrinsics
 7661 
 7662     cmpq(length, 16);
 7663     jccb(Assembler::less, VECTOR8_TAIL);
 7664 
 7665     subq(length, 16);
 7666     bind(VECTOR16_LOOP);
 7667     movdqu(rymm0, Address(obja, result));
 7668     movdqu(rymm1, Address(objb, result));
 7669     pxor(rymm0, rymm1);
 7670     ptest(rymm0, rymm0);
 7671     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
 7672     addq(result, 16);
 7673     subq(length, 16);
 7674     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
 7675     addq(length, 16);
 7676     jcc(Assembler::equal, SAME_TILL_END);
 7677     //falling through if less than 16 bytes left
 7678   }
 7679 
 7680   bind(VECTOR8_TAIL);
 7681   cmpq(length, 8);
 7682   jccb(Assembler::less, VECTOR4_TAIL);
 7683   bind(VECTOR8_LOOP);
 7684   movq(tmp1, Address(obja, result));
 7685   movq(tmp2, Address(objb, result));
 7686   xorq(tmp1, tmp2);
 7687   testq(tmp1, tmp1);
 7688   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
 7689   addq(result, 8);
 7690   subq(length, 8);
 7691   jcc(Assembler::equal, SAME_TILL_END);
 7692   //falling through if less than 8 bytes left
 7693 
 7694   bind(VECTOR4_TAIL);
 7695   cmpq(length, 4);
 7696   jccb(Assembler::less, BYTES_TAIL);
 7697   bind(VECTOR4_LOOP);
 7698   movl(tmp1, Address(obja, result));
 7699   xorl(tmp1, Address(objb, result));
 7700   testl(tmp1, tmp1);
 7701   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
 7702   addq(result, 4);
 7703   subq(length, 4);
 7704   jcc(Assembler::equal, SAME_TILL_END);
 7705   //falling through if less than 4 bytes left
 7706 
 7707   bind(BYTES_TAIL);
 7708   bind(BYTES_LOOP);
 7709   load_unsigned_byte(tmp1, Address(obja, result));
 7710   load_unsigned_byte(tmp2, Address(objb, result));
 7711   xorl(tmp1, tmp2);
 7712   testl(tmp1, tmp1);
 7713   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7714   decq(length);
 7715   jcc(Assembler::zero, SAME_TILL_END);
 7716   incq(result);
 7717   load_unsigned_byte(tmp1, Address(obja, result));
 7718   load_unsigned_byte(tmp2, Address(objb, result));
 7719   xorl(tmp1, tmp2);
 7720   testl(tmp1, tmp1);
 7721   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7722   decq(length);
 7723   jcc(Assembler::zero, SAME_TILL_END);
 7724   incq(result);
 7725   load_unsigned_byte(tmp1, Address(obja, result));
 7726   load_unsigned_byte(tmp2, Address(objb, result));
 7727   xorl(tmp1, tmp2);
 7728   testl(tmp1, tmp1);
 7729   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
 7730   jmp(SAME_TILL_END);
 7731 
 7732   if (UseAVX >= 2) {
 7733     bind(VECTOR32_NOT_EQUAL);
 7734     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
 7735     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
 7736     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
 7737     vpmovmskb(tmp1, rymm0);
 7738     bsfq(tmp1, tmp1);
 7739     addq(result, tmp1);
 7740     shrq(result);
 7741     jmp(DONE);
 7742   }
 7743 
 7744   bind(VECTOR16_NOT_EQUAL);
 7745   if (UseAVX >= 2) {
 7746     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
 7747     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
 7748     pxor(rymm0, rymm2);
 7749   } else {
 7750     pcmpeqb(rymm2, rymm2);
 7751     pxor(rymm0, rymm1);
 7752     pcmpeqb(rymm0, rymm1);
 7753     pxor(rymm0, rymm2);
 7754   }
 7755   pmovmskb(tmp1, rymm0);
 7756   bsfq(tmp1, tmp1);
 7757   addq(result, tmp1);
 7758   shrq(result);
 7759   jmpb(DONE);
 7760 
 7761   bind(VECTOR8_NOT_EQUAL);
 7762   bind(VECTOR4_NOT_EQUAL);
 7763   bsfq(tmp1, tmp1);
 7764   shrq(tmp1, 3);
 7765   addq(result, tmp1);
 7766   bind(BYTES_NOT_EQUAL);
 7767   shrq(result);
 7768   jmpb(DONE);
 7769 
 7770   bind(SAME_TILL_END);
 7771   mov64(result, -1);
 7772 
 7773   bind(DONE);
 7774 }
 7775 
 7776 //Helper functions for square_to_len()
 7777 
 7778 /**
 7779  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
 7780  * Preserves x and z and modifies rest of the registers.
 7781  */
 7782 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7783   // Perform square and right shift by 1
 7784   // Handle odd xlen case first, then for even xlen do the following
 7785   // jlong carry = 0;
 7786   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
 7787   //     huge_128 product = x[j:j+1] * x[j:j+1];
 7788   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
 7789   //     z[i+2:i+3] = (jlong)(product >>> 1);
 7790   //     carry = (jlong)product;
 7791   // }
 7792 
 7793   xorq(tmp5, tmp5);     // carry
 7794   xorq(rdxReg, rdxReg);
 7795   xorl(tmp1, tmp1);     // index for x
 7796   xorl(tmp4, tmp4);     // index for z
 7797 
 7798   Label L_first_loop, L_first_loop_exit;
 7799 
 7800   testl(xlen, 1);
 7801   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
 7802 
 7803   // Square and right shift by 1 the odd element using 32 bit multiply
 7804   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
 7805   imulq(raxReg, raxReg);
 7806   shrq(raxReg, 1);
 7807   adcq(tmp5, 0);
 7808   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
 7809   incrementl(tmp1);
 7810   addl(tmp4, 2);
 7811 
 7812   // Square and  right shift by 1 the rest using 64 bit multiply
 7813   bind(L_first_loop);
 7814   cmpptr(tmp1, xlen);
 7815   jccb(Assembler::equal, L_first_loop_exit);
 7816 
 7817   // Square
 7818   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
 7819   rorq(raxReg, 32);    // convert big-endian to little-endian
 7820   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
 7821 
 7822   // Right shift by 1 and save carry
 7823   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
 7824   rcrq(rdxReg, 1);
 7825   rcrq(raxReg, 1);
 7826   adcq(tmp5, 0);
 7827 
 7828   // Store result in z
 7829   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
 7830   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
 7831 
 7832   // Update indices for x and z
 7833   addl(tmp1, 2);
 7834   addl(tmp4, 4);
 7835   jmp(L_first_loop);
 7836 
 7837   bind(L_first_loop_exit);
 7838 }
 7839 
 7840 
 7841 /**
 7842  * Perform the following multiply add operation using BMI2 instructions
 7843  * carry:sum = sum + op1*op2 + carry
 7844  * op2 should be in rdx
 7845  * op2 is preserved, all other registers are modified
 7846  */
 7847 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
 7848   // assert op2 is rdx
 7849   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
 7850   addq(sum, carry);
 7851   adcq(tmp2, 0);
 7852   addq(sum, op1);
 7853   adcq(tmp2, 0);
 7854   movq(carry, tmp2);
 7855 }
 7856 
 7857 /**
 7858  * Perform the following multiply add operation:
 7859  * carry:sum = sum + op1*op2 + carry
 7860  * Preserves op1, op2 and modifies rest of registers
 7861  */
 7862 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
 7863   // rdx:rax = op1 * op2
 7864   movq(raxReg, op2);
 7865   mulq(op1);
 7866 
 7867   //  rdx:rax = sum + carry + rdx:rax
 7868   addq(sum, carry);
 7869   adcq(rdxReg, 0);
 7870   addq(sum, raxReg);
 7871   adcq(rdxReg, 0);
 7872 
 7873   // carry:sum = rdx:sum
 7874   movq(carry, rdxReg);
 7875 }
 7876 
 7877 /**
 7878  * Add 64 bit long carry into z[] with carry propagation.
 7879  * Preserves z and carry register values and modifies rest of registers.
 7880  *
 7881  */
 7882 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
 7883   Label L_fourth_loop, L_fourth_loop_exit;
 7884 
 7885   movl(tmp1, 1);
 7886   subl(zlen, 2);
 7887   addq(Address(z, zlen, Address::times_4, 0), carry);
 7888 
 7889   bind(L_fourth_loop);
 7890   jccb(Assembler::carryClear, L_fourth_loop_exit);
 7891   subl(zlen, 2);
 7892   jccb(Assembler::negative, L_fourth_loop_exit);
 7893   addq(Address(z, zlen, Address::times_4, 0), tmp1);
 7894   jmp(L_fourth_loop);
 7895   bind(L_fourth_loop_exit);
 7896 }
 7897 
 7898 /**
 7899  * Shift z[] left by 1 bit.
 7900  * Preserves x, len, z and zlen registers and modifies rest of the registers.
 7901  *
 7902  */
 7903 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
 7904 
 7905   Label L_fifth_loop, L_fifth_loop_exit;
 7906 
 7907   // Fifth loop
 7908   // Perform primitiveLeftShift(z, zlen, 1)
 7909 
 7910   const Register prev_carry = tmp1;
 7911   const Register new_carry = tmp4;
 7912   const Register value = tmp2;
 7913   const Register zidx = tmp3;
 7914 
 7915   // int zidx, carry;
 7916   // long value;
 7917   // carry = 0;
 7918   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
 7919   //    (carry:value)  = (z[i] << 1) | carry ;
 7920   //    z[i] = value;
 7921   // }
 7922 
 7923   movl(zidx, zlen);
 7924   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
 7925 
 7926   bind(L_fifth_loop);
 7927   decl(zidx);  // Use decl to preserve carry flag
 7928   decl(zidx);
 7929   jccb(Assembler::negative, L_fifth_loop_exit);
 7930 
 7931   if (UseBMI2Instructions) {
 7932      movq(value, Address(z, zidx, Address::times_4, 0));
 7933      rclq(value, 1);
 7934      rorxq(value, value, 32);
 7935      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7936   }
 7937   else {
 7938     // clear new_carry
 7939     xorl(new_carry, new_carry);
 7940 
 7941     // Shift z[i] by 1, or in previous carry and save new carry
 7942     movq(value, Address(z, zidx, Address::times_4, 0));
 7943     shlq(value, 1);
 7944     adcl(new_carry, 0);
 7945 
 7946     orq(value, prev_carry);
 7947     rorq(value, 0x20);
 7948     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
 7949 
 7950     // Set previous carry = new carry
 7951     movl(prev_carry, new_carry);
 7952   }
 7953   jmp(L_fifth_loop);
 7954 
 7955   bind(L_fifth_loop_exit);
 7956 }
 7957 
 7958 
 7959 /**
 7960  * Code for BigInteger::squareToLen() intrinsic
 7961  *
 7962  * rdi: x
 7963  * rsi: len
 7964  * r8:  z
 7965  * rcx: zlen
 7966  * r12: tmp1
 7967  * r13: tmp2
 7968  * r14: tmp3
 7969  * r15: tmp4
 7970  * rbx: tmp5
 7971  *
 7972  */
 7973 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 7974 
 7975   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
 7976   push(tmp1);
 7977   push(tmp2);
 7978   push(tmp3);
 7979   push(tmp4);
 7980   push(tmp5);
 7981 
 7982   // First loop
 7983   // Store the squares, right shifted one bit (i.e., divided by 2).
 7984   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
 7985 
 7986   // Add in off-diagonal sums.
 7987   //
 7988   // Second, third (nested) and fourth loops.
 7989   // zlen +=2;
 7990   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
 7991   //    carry = 0;
 7992   //    long op2 = x[xidx:xidx+1];
 7993   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
 7994   //       k -= 2;
 7995   //       long op1 = x[j:j+1];
 7996   //       long sum = z[k:k+1];
 7997   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
 7998   //       z[k:k+1] = sum;
 7999   //    }
 8000   //    add_one_64(z, k, carry, tmp_regs);
 8001   // }
 8002 
 8003   const Register carry = tmp5;
 8004   const Register sum = tmp3;
 8005   const Register op1 = tmp4;
 8006   Register op2 = tmp2;
 8007 
 8008   push(zlen);
 8009   push(len);
 8010   addl(zlen,2);
 8011   bind(L_second_loop);
 8012   xorq(carry, carry);
 8013   subl(zlen, 4);
 8014   subl(len, 2);
 8015   push(zlen);
 8016   push(len);
 8017   cmpl(len, 0);
 8018   jccb(Assembler::lessEqual, L_second_loop_exit);
 8019 
 8020   // Multiply an array by one 64 bit long.
 8021   if (UseBMI2Instructions) {
 8022     op2 = rdxReg;
 8023     movq(op2, Address(x, len, Address::times_4,  0));
 8024     rorxq(op2, op2, 32);
 8025   }
 8026   else {
 8027     movq(op2, Address(x, len, Address::times_4,  0));
 8028     rorq(op2, 32);
 8029   }
 8030 
 8031   bind(L_third_loop);
 8032   decrementl(len);
 8033   jccb(Assembler::negative, L_third_loop_exit);
 8034   decrementl(len);
 8035   jccb(Assembler::negative, L_last_x);
 8036 
 8037   movq(op1, Address(x, len, Address::times_4,  0));
 8038   rorq(op1, 32);
 8039 
 8040   bind(L_multiply);
 8041   subl(zlen, 2);
 8042   movq(sum, Address(z, zlen, Address::times_4,  0));
 8043 
 8044   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
 8045   if (UseBMI2Instructions) {
 8046     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
 8047   }
 8048   else {
 8049     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8050   }
 8051 
 8052   movq(Address(z, zlen, Address::times_4, 0), sum);
 8053 
 8054   jmp(L_third_loop);
 8055   bind(L_third_loop_exit);
 8056 
 8057   // Fourth loop
 8058   // Add 64 bit long carry into z with carry propagation.
 8059   // Uses offsetted zlen.
 8060   add_one_64(z, zlen, carry, tmp1);
 8061 
 8062   pop(len);
 8063   pop(zlen);
 8064   jmp(L_second_loop);
 8065 
 8066   // Next infrequent code is moved outside loops.
 8067   bind(L_last_x);
 8068   movl(op1, Address(x, 0));
 8069   jmp(L_multiply);
 8070 
 8071   bind(L_second_loop_exit);
 8072   pop(len);
 8073   pop(zlen);
 8074   pop(len);
 8075   pop(zlen);
 8076 
 8077   // Fifth loop
 8078   // Shift z left 1 bit.
 8079   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
 8080 
 8081   // z[zlen-1] |= x[len-1] & 1;
 8082   movl(tmp3, Address(x, len, Address::times_4, -4));
 8083   andl(tmp3, 1);
 8084   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
 8085 
 8086   pop(tmp5);
 8087   pop(tmp4);
 8088   pop(tmp3);
 8089   pop(tmp2);
 8090   pop(tmp1);
 8091 }
 8092 
 8093 /**
 8094  * Helper function for mul_add()
 8095  * Multiply the in[] by int k and add to out[] starting at offset offs using
 8096  * 128 bit by 32 bit multiply and return the carry in tmp5.
 8097  * Only quad int aligned length of in[] is operated on in this function.
 8098  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
 8099  * This function preserves out, in and k registers.
 8100  * len and offset point to the appropriate index in "in" & "out" correspondingly
 8101  * tmp5 has the carry.
 8102  * other registers are temporary and are modified.
 8103  *
 8104  */
 8105 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
 8106   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
 8107   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8108 
 8109   Label L_first_loop, L_first_loop_exit;
 8110 
 8111   movl(tmp1, len);
 8112   shrl(tmp1, 2);
 8113 
 8114   bind(L_first_loop);
 8115   subl(tmp1, 1);
 8116   jccb(Assembler::negative, L_first_loop_exit);
 8117 
 8118   subl(len, 4);
 8119   subl(offset, 4);
 8120 
 8121   Register op2 = tmp2;
 8122   const Register sum = tmp3;
 8123   const Register op1 = tmp4;
 8124   const Register carry = tmp5;
 8125 
 8126   if (UseBMI2Instructions) {
 8127     op2 = rdxReg;
 8128   }
 8129 
 8130   movq(op1, Address(in, len, Address::times_4,  8));
 8131   rorq(op1, 32);
 8132   movq(sum, Address(out, offset, Address::times_4,  8));
 8133   rorq(sum, 32);
 8134   if (UseBMI2Instructions) {
 8135     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8136   }
 8137   else {
 8138     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8139   }
 8140   // Store back in big endian from little endian
 8141   rorq(sum, 0x20);
 8142   movq(Address(out, offset, Address::times_4,  8), sum);
 8143 
 8144   movq(op1, Address(in, len, Address::times_4,  0));
 8145   rorq(op1, 32);
 8146   movq(sum, Address(out, offset, Address::times_4,  0));
 8147   rorq(sum, 32);
 8148   if (UseBMI2Instructions) {
 8149     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8150   }
 8151   else {
 8152     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8153   }
 8154   // Store back in big endian from little endian
 8155   rorq(sum, 0x20);
 8156   movq(Address(out, offset, Address::times_4,  0), sum);
 8157 
 8158   jmp(L_first_loop);
 8159   bind(L_first_loop_exit);
 8160 }
 8161 
 8162 /**
 8163  * Code for BigInteger::mulAdd() intrinsic
 8164  *
 8165  * rdi: out
 8166  * rsi: in
 8167  * r11: offs (out.length - offset)
 8168  * rcx: len
 8169  * r8:  k
 8170  * r12: tmp1
 8171  * r13: tmp2
 8172  * r14: tmp3
 8173  * r15: tmp4
 8174  * rbx: tmp5
 8175  * Multiply the in[] by word k and add to out[], return the carry in rax
 8176  */
 8177 void MacroAssembler::mul_add(Register out, Register in, Register offs,
 8178    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
 8179    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
 8180 
 8181   Label L_carry, L_last_in, L_done;
 8182 
 8183 // carry = 0;
 8184 // for (int j=len-1; j >= 0; j--) {
 8185 //    long product = (in[j] & LONG_MASK) * kLong +
 8186 //                   (out[offs] & LONG_MASK) + carry;
 8187 //    out[offs--] = (int)product;
 8188 //    carry = product >>> 32;
 8189 // }
 8190 //
 8191   push(tmp1);
 8192   push(tmp2);
 8193   push(tmp3);
 8194   push(tmp4);
 8195   push(tmp5);
 8196 
 8197   Register op2 = tmp2;
 8198   const Register sum = tmp3;
 8199   const Register op1 = tmp4;
 8200   const Register carry =  tmp5;
 8201 
 8202   if (UseBMI2Instructions) {
 8203     op2 = rdxReg;
 8204     movl(op2, k);
 8205   }
 8206   else {
 8207     movl(op2, k);
 8208   }
 8209 
 8210   xorq(carry, carry);
 8211 
 8212   //First loop
 8213 
 8214   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
 8215   //The carry is in tmp5
 8216   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
 8217 
 8218   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
 8219   decrementl(len);
 8220   jccb(Assembler::negative, L_carry);
 8221   decrementl(len);
 8222   jccb(Assembler::negative, L_last_in);
 8223 
 8224   movq(op1, Address(in, len, Address::times_4,  0));
 8225   rorq(op1, 32);
 8226 
 8227   subl(offs, 2);
 8228   movq(sum, Address(out, offs, Address::times_4,  0));
 8229   rorq(sum, 32);
 8230 
 8231   if (UseBMI2Instructions) {
 8232     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
 8233   }
 8234   else {
 8235     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
 8236   }
 8237 
 8238   // Store back in big endian from little endian
 8239   rorq(sum, 0x20);
 8240   movq(Address(out, offs, Address::times_4,  0), sum);
 8241 
 8242   testl(len, len);
 8243   jccb(Assembler::zero, L_carry);
 8244 
 8245   //Multiply the last in[] entry, if any
 8246   bind(L_last_in);
 8247   movl(op1, Address(in, 0));
 8248   movl(sum, Address(out, offs, Address::times_4,  -4));
 8249 
 8250   movl(raxReg, k);
 8251   mull(op1); //tmp4 * eax -> edx:eax
 8252   addl(sum, carry);
 8253   adcl(rdxReg, 0);
 8254   addl(sum, raxReg);
 8255   adcl(rdxReg, 0);
 8256   movl(carry, rdxReg);
 8257 
 8258   movl(Address(out, offs, Address::times_4,  -4), sum);
 8259 
 8260   bind(L_carry);
 8261   //return tmp5/carry as carry in rax
 8262   movl(rax, carry);
 8263 
 8264   bind(L_done);
 8265   pop(tmp5);
 8266   pop(tmp4);
 8267   pop(tmp3);
 8268   pop(tmp2);
 8269   pop(tmp1);
 8270 }
 8271 #endif
 8272 
 8273 /**
 8274  * Emits code to update CRC-32 with a byte value according to constants in table
 8275  *
 8276  * @param [in,out]crc   Register containing the crc.
 8277  * @param [in]val       Register containing the byte to fold into the CRC.
 8278  * @param [in]table     Register containing the table of crc constants.
 8279  *
 8280  * uint32_t crc;
 8281  * val = crc_table[(val ^ crc) & 0xFF];
 8282  * crc = val ^ (crc >> 8);
 8283  *
 8284  */
 8285 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
 8286   xorl(val, crc);
 8287   andl(val, 0xFF);
 8288   shrl(crc, 8); // unsigned shift
 8289   xorl(crc, Address(table, val, Address::times_4, 0));
 8290 }
 8291 
 8292 /**
 8293  * Fold 128-bit data chunk
 8294  */
 8295 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
 8296   if (UseAVX > 0) {
 8297     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
 8298     vpclmulldq(xcrc, xK, xcrc); // [63:0]
 8299     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
 8300     pxor(xcrc, xtmp);
 8301   } else {
 8302     movdqa(xtmp, xcrc);
 8303     pclmulhdq(xtmp, xK);   // [123:64]
 8304     pclmulldq(xcrc, xK);   // [63:0]
 8305     pxor(xcrc, xtmp);
 8306     movdqu(xtmp, Address(buf, offset));
 8307     pxor(xcrc, xtmp);
 8308   }
 8309 }
 8310 
 8311 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
 8312   if (UseAVX > 0) {
 8313     vpclmulhdq(xtmp, xK, xcrc);
 8314     vpclmulldq(xcrc, xK, xcrc);
 8315     pxor(xcrc, xbuf);
 8316     pxor(xcrc, xtmp);
 8317   } else {
 8318     movdqa(xtmp, xcrc);
 8319     pclmulhdq(xtmp, xK);
 8320     pclmulldq(xcrc, xK);
 8321     pxor(xcrc, xbuf);
 8322     pxor(xcrc, xtmp);
 8323   }
 8324 }
 8325 
 8326 /**
 8327  * 8-bit folds to compute 32-bit CRC
 8328  *
 8329  * uint64_t xcrc;
 8330  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
 8331  */
 8332 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
 8333   movdl(tmp, xcrc);
 8334   andl(tmp, 0xFF);
 8335   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
 8336   psrldq(xcrc, 1); // unsigned shift one byte
 8337   pxor(xcrc, xtmp);
 8338 }
 8339 
 8340 /**
 8341  * uint32_t crc;
 8342  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
 8343  */
 8344 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
 8345   movl(tmp, crc);
 8346   andl(tmp, 0xFF);
 8347   shrl(crc, 8);
 8348   xorl(crc, Address(table, tmp, Address::times_4, 0));
 8349 }
 8350 
 8351 /**
 8352  * @param crc   register containing existing CRC (32-bit)
 8353  * @param buf   register pointing to input byte buffer (byte*)
 8354  * @param len   register containing number of bytes
 8355  * @param table register that will contain address of CRC table
 8356  * @param tmp   scratch register
 8357  */
 8358 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
 8359   assert_different_registers(crc, buf, len, table, tmp, rax);
 8360 
 8361   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 8362   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 8363 
 8364   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 8365   // context for the registers used, where all instructions below are using 128-bit mode
 8366   // On EVEX without VL and BW, these instructions will all be AVX.
 8367   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
 8368   notl(crc); // ~crc
 8369   cmpl(len, 16);
 8370   jcc(Assembler::less, L_tail);
 8371 
 8372   // Align buffer to 16 bytes
 8373   movl(tmp, buf);
 8374   andl(tmp, 0xF);
 8375   jccb(Assembler::zero, L_aligned);
 8376   subl(tmp,  16);
 8377   addl(len, tmp);
 8378 
 8379   align(4);
 8380   BIND(L_align_loop);
 8381   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 8382   update_byte_crc32(crc, rax, table);
 8383   increment(buf);
 8384   incrementl(tmp);
 8385   jccb(Assembler::less, L_align_loop);
 8386 
 8387   BIND(L_aligned);
 8388   movl(tmp, len); // save
 8389   shrl(len, 4);
 8390   jcc(Assembler::zero, L_tail_restore);
 8391 
 8392   // Fold crc into first bytes of vector
 8393   movdqa(xmm1, Address(buf, 0));
 8394   movdl(rax, xmm1);
 8395   xorl(crc, rax);
 8396   if (VM_Version::supports_sse4_1()) {
 8397     pinsrd(xmm1, crc, 0);
 8398   } else {
 8399     pinsrw(xmm1, crc, 0);
 8400     shrl(crc, 16);
 8401     pinsrw(xmm1, crc, 1);
 8402   }
 8403   addptr(buf, 16);
 8404   subl(len, 4); // len > 0
 8405   jcc(Assembler::less, L_fold_tail);
 8406 
 8407   movdqa(xmm2, Address(buf,  0));
 8408   movdqa(xmm3, Address(buf, 16));
 8409   movdqa(xmm4, Address(buf, 32));
 8410   addptr(buf, 48);
 8411   subl(len, 3);
 8412   jcc(Assembler::lessEqual, L_fold_512b);
 8413 
 8414   // Fold total 512 bits of polynomial on each iteration,
 8415   // 128 bits per each of 4 parallel streams.
 8416   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32), rscratch1);
 8417 
 8418   align32();
 8419   BIND(L_fold_512b_loop);
 8420   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8421   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
 8422   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
 8423   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
 8424   addptr(buf, 64);
 8425   subl(len, 4);
 8426   jcc(Assembler::greater, L_fold_512b_loop);
 8427 
 8428   // Fold 512 bits to 128 bits.
 8429   BIND(L_fold_512b);
 8430   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8431   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
 8432   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
 8433   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
 8434 
 8435   // Fold the rest of 128 bits data chunks
 8436   BIND(L_fold_tail);
 8437   addl(len, 3);
 8438   jccb(Assembler::lessEqual, L_fold_128b);
 8439   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16), rscratch1);
 8440 
 8441   BIND(L_fold_tail_loop);
 8442   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
 8443   addptr(buf, 16);
 8444   decrementl(len);
 8445   jccb(Assembler::greater, L_fold_tail_loop);
 8446 
 8447   // Fold 128 bits in xmm1 down into 32 bits in crc register.
 8448   BIND(L_fold_128b);
 8449   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()), rscratch1);
 8450   if (UseAVX > 0) {
 8451     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
 8452     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
 8453     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
 8454   } else {
 8455     movdqa(xmm2, xmm0);
 8456     pclmulqdq(xmm2, xmm1, 0x1);
 8457     movdqa(xmm3, xmm0);
 8458     pand(xmm3, xmm2);
 8459     pclmulqdq(xmm0, xmm3, 0x1);
 8460   }
 8461   psrldq(xmm1, 8);
 8462   psrldq(xmm2, 4);
 8463   pxor(xmm0, xmm1);
 8464   pxor(xmm0, xmm2);
 8465 
 8466   // 8 8-bit folds to compute 32-bit CRC.
 8467   for (int j = 0; j < 4; j++) {
 8468     fold_8bit_crc32(xmm0, table, xmm1, rax);
 8469   }
 8470   movdl(crc, xmm0); // mov 32 bits to general register
 8471   for (int j = 0; j < 4; j++) {
 8472     fold_8bit_crc32(crc, table, rax);
 8473   }
 8474 
 8475   BIND(L_tail_restore);
 8476   movl(len, tmp); // restore
 8477   BIND(L_tail);
 8478   andl(len, 0xf);
 8479   jccb(Assembler::zero, L_exit);
 8480 
 8481   // Fold the rest of bytes
 8482   align(4);
 8483   BIND(L_tail_loop);
 8484   movsbl(rax, Address(buf, 0)); // load byte with sign extension
 8485   update_byte_crc32(crc, rax, table);
 8486   increment(buf);
 8487   decrementl(len);
 8488   jccb(Assembler::greater, L_tail_loop);
 8489 
 8490   BIND(L_exit);
 8491   notl(crc); // ~c
 8492 }
 8493 
 8494 #ifdef _LP64
 8495 // Helper function for AVX 512 CRC32
 8496 // Fold 512-bit data chunks
 8497 void MacroAssembler::fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf,
 8498                                              Register pos, int offset) {
 8499   evmovdquq(xmm3, Address(buf, pos, Address::times_1, offset), Assembler::AVX_512bit);
 8500   evpclmulqdq(xtmp, xcrc, xK, 0x10, Assembler::AVX_512bit); // [123:64]
 8501   evpclmulqdq(xmm2, xcrc, xK, 0x01, Assembler::AVX_512bit); // [63:0]
 8502   evpxorq(xcrc, xtmp, xmm2, Assembler::AVX_512bit /* vector_len */);
 8503   evpxorq(xcrc, xcrc, xmm3, Assembler::AVX_512bit /* vector_len */);
 8504 }
 8505 
 8506 // Helper function for AVX 512 CRC32
 8507 // Compute CRC32 for < 256B buffers
 8508 void MacroAssembler::kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register table, Register pos,
 8509                                               Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
 8510                                               Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup) {
 8511 
 8512   Label L_less_than_32, L_exact_16_left, L_less_than_16_left;
 8513   Label L_less_than_8_left, L_less_than_4_left, L_less_than_2_left, L_zero_left;
 8514   Label L_only_less_than_4, L_only_less_than_3, L_only_less_than_2;
 8515 
 8516   // check if there is enough buffer to be able to fold 16B at a time
 8517   cmpl(len, 32);
 8518   jcc(Assembler::less, L_less_than_32);
 8519 
 8520   // if there is, load the constants
 8521   movdqu(xmm10, Address(table, 1 * 16));    //rk1 and rk2 in xmm10
 8522   movdl(xmm0, crc);                        // get the initial crc value
 8523   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8524   pxor(xmm7, xmm0);
 8525 
 8526   // update the buffer pointer
 8527   addl(pos, 16);
 8528   //update the counter.subtract 32 instead of 16 to save one instruction from the loop
 8529   subl(len, 32);
 8530   jmp(L_16B_reduction_loop);
 8531 
 8532   bind(L_less_than_32);
 8533   //mov initial crc to the return value. this is necessary for zero - length buffers.
 8534   movl(rax, crc);
 8535   testl(len, len);
 8536   jcc(Assembler::equal, L_cleanup);
 8537 
 8538   movdl(xmm0, crc);                        //get the initial crc value
 8539 
 8540   cmpl(len, 16);
 8541   jcc(Assembler::equal, L_exact_16_left);
 8542   jcc(Assembler::less, L_less_than_16_left);
 8543 
 8544   movdqu(xmm7, Address(buf, pos, Address::times_1, 0 * 16)); //load the plaintext
 8545   pxor(xmm7, xmm0);                       //xor the initial crc value
 8546   addl(pos, 16);
 8547   subl(len, 16);
 8548   movdqu(xmm10, Address(table, 1 * 16));    // rk1 and rk2 in xmm10
 8549   jmp(L_get_last_two_xmms);
 8550 
 8551   bind(L_less_than_16_left);
 8552   //use stack space to load data less than 16 bytes, zero - out the 16B in memory first.
 8553   pxor(xmm1, xmm1);
 8554   movptr(tmp1, rsp);
 8555   movdqu(Address(tmp1, 0 * 16), xmm1);
 8556 
 8557   cmpl(len, 4);
 8558   jcc(Assembler::less, L_only_less_than_4);
 8559 
 8560   //backup the counter value
 8561   movl(tmp2, len);
 8562   cmpl(len, 8);
 8563   jcc(Assembler::less, L_less_than_8_left);
 8564 
 8565   //load 8 Bytes
 8566   movq(rax, Address(buf, pos, Address::times_1, 0 * 16));
 8567   movq(Address(tmp1, 0 * 16), rax);
 8568   addptr(tmp1, 8);
 8569   subl(len, 8);
 8570   addl(pos, 8);
 8571 
 8572   bind(L_less_than_8_left);
 8573   cmpl(len, 4);
 8574   jcc(Assembler::less, L_less_than_4_left);
 8575 
 8576   //load 4 Bytes
 8577   movl(rax, Address(buf, pos, Address::times_1, 0));
 8578   movl(Address(tmp1, 0 * 16), rax);
 8579   addptr(tmp1, 4);
 8580   subl(len, 4);
 8581   addl(pos, 4);
 8582 
 8583   bind(L_less_than_4_left);
 8584   cmpl(len, 2);
 8585   jcc(Assembler::less, L_less_than_2_left);
 8586 
 8587   // load 2 Bytes
 8588   movw(rax, Address(buf, pos, Address::times_1, 0));
 8589   movl(Address(tmp1, 0 * 16), rax);
 8590   addptr(tmp1, 2);
 8591   subl(len, 2);
 8592   addl(pos, 2);
 8593 
 8594   bind(L_less_than_2_left);
 8595   cmpl(len, 1);
 8596   jcc(Assembler::less, L_zero_left);
 8597 
 8598   // load 1 Byte
 8599   movb(rax, Address(buf, pos, Address::times_1, 0));
 8600   movb(Address(tmp1, 0 * 16), rax);
 8601 
 8602   bind(L_zero_left);
 8603   movdqu(xmm7, Address(rsp, 0));
 8604   pxor(xmm7, xmm0);                       //xor the initial crc value
 8605 
 8606   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8607   movdqu(xmm0, Address(rax, tmp2));
 8608   pshufb(xmm7, xmm0);
 8609   jmp(L_128_done);
 8610 
 8611   bind(L_exact_16_left);
 8612   movdqu(xmm7, Address(buf, pos, Address::times_1, 0));
 8613   pxor(xmm7, xmm0);                       //xor the initial crc value
 8614   jmp(L_128_done);
 8615 
 8616   bind(L_only_less_than_4);
 8617   cmpl(len, 3);
 8618   jcc(Assembler::less, L_only_less_than_3);
 8619 
 8620   // load 3 Bytes
 8621   movb(rax, Address(buf, pos, Address::times_1, 0));
 8622   movb(Address(tmp1, 0), rax);
 8623 
 8624   movb(rax, Address(buf, pos, Address::times_1, 1));
 8625   movb(Address(tmp1, 1), rax);
 8626 
 8627   movb(rax, Address(buf, pos, Address::times_1, 2));
 8628   movb(Address(tmp1, 2), rax);
 8629 
 8630   movdqu(xmm7, Address(rsp, 0));
 8631   pxor(xmm7, xmm0);                     //xor the initial crc value
 8632 
 8633   pslldq(xmm7, 0x5);
 8634   jmp(L_barrett);
 8635   bind(L_only_less_than_3);
 8636   cmpl(len, 2);
 8637   jcc(Assembler::less, L_only_less_than_2);
 8638 
 8639   // load 2 Bytes
 8640   movb(rax, Address(buf, pos, Address::times_1, 0));
 8641   movb(Address(tmp1, 0), rax);
 8642 
 8643   movb(rax, Address(buf, pos, Address::times_1, 1));
 8644   movb(Address(tmp1, 1), rax);
 8645 
 8646   movdqu(xmm7, Address(rsp, 0));
 8647   pxor(xmm7, xmm0);                     //xor the initial crc value
 8648 
 8649   pslldq(xmm7, 0x6);
 8650   jmp(L_barrett);
 8651 
 8652   bind(L_only_less_than_2);
 8653   //load 1 Byte
 8654   movb(rax, Address(buf, pos, Address::times_1, 0));
 8655   movb(Address(tmp1, 0), rax);
 8656 
 8657   movdqu(xmm7, Address(rsp, 0));
 8658   pxor(xmm7, xmm0);                     //xor the initial crc value
 8659 
 8660   pslldq(xmm7, 0x7);
 8661 }
 8662 
 8663 /**
 8664 * Compute CRC32 using AVX512 instructions
 8665 * param crc   register containing existing CRC (32-bit)
 8666 * param buf   register pointing to input byte buffer (byte*)
 8667 * param len   register containing number of bytes
 8668 * param table address of crc or crc32c table
 8669 * param tmp1  scratch register
 8670 * param tmp2  scratch register
 8671 * return rax  result register
 8672 *
 8673 * This routine is identical for crc32c with the exception of the precomputed constant
 8674 * table which will be passed as the table argument.  The calculation steps are
 8675 * the same for both variants.
 8676 */
 8677 void MacroAssembler::kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2) {
 8678   assert_different_registers(crc, buf, len, table, tmp1, tmp2, rax, r12);
 8679 
 8680   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
 8681   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
 8682   Label L_less_than_256, L_fold_128_B_loop, L_fold_256_B_loop;
 8683   Label L_fold_128_B_register, L_final_reduction_for_128, L_16B_reduction_loop;
 8684   Label L_128_done, L_get_last_two_xmms, L_barrett, L_cleanup;
 8685 
 8686   const Register pos = r12;
 8687   push(r12);
 8688   subptr(rsp, 16 * 2 + 8);
 8689 
 8690   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
 8691   // context for the registers used, where all instructions below are using 128-bit mode
 8692   // On EVEX without VL and BW, these instructions will all be AVX.
 8693   movl(pos, 0);
 8694 
 8695   // check if smaller than 256B
 8696   cmpl(len, 256);
 8697   jcc(Assembler::less, L_less_than_256);
 8698 
 8699   // load the initial crc value
 8700   movdl(xmm10, crc);
 8701 
 8702   // receive the initial 64B data, xor the initial crc value
 8703   evmovdquq(xmm0, Address(buf, pos, Address::times_1, 0 * 64), Assembler::AVX_512bit);
 8704   evmovdquq(xmm4, Address(buf, pos, Address::times_1, 1 * 64), Assembler::AVX_512bit);
 8705   evpxorq(xmm0, xmm0, xmm10, Assembler::AVX_512bit);
 8706   evbroadcasti32x4(xmm10, Address(table, 2 * 16), Assembler::AVX_512bit); //zmm10 has rk3 and rk4
 8707 
 8708   subl(len, 256);
 8709   cmpl(len, 256);
 8710   jcc(Assembler::less, L_fold_128_B_loop);
 8711 
 8712   evmovdquq(xmm7, Address(buf, pos, Address::times_1, 2 * 64), Assembler::AVX_512bit);
 8713   evmovdquq(xmm8, Address(buf, pos, Address::times_1, 3 * 64), Assembler::AVX_512bit);
 8714   evbroadcasti32x4(xmm16, Address(table, 0 * 16), Assembler::AVX_512bit); //zmm16 has rk-1 and rk-2
 8715   subl(len, 256);
 8716 
 8717   bind(L_fold_256_B_loop);
 8718   addl(pos, 256);
 8719   fold512bit_crc32_avx512(xmm0, xmm16, xmm1, buf, pos, 0 * 64);
 8720   fold512bit_crc32_avx512(xmm4, xmm16, xmm1, buf, pos, 1 * 64);
 8721   fold512bit_crc32_avx512(xmm7, xmm16, xmm1, buf, pos, 2 * 64);
 8722   fold512bit_crc32_avx512(xmm8, xmm16, xmm1, buf, pos, 3 * 64);
 8723 
 8724   subl(len, 256);
 8725   jcc(Assembler::greaterEqual, L_fold_256_B_loop);
 8726 
 8727   // Fold 256 into 128
 8728   addl(pos, 256);
 8729   evpclmulqdq(xmm1, xmm0, xmm10, 0x01, Assembler::AVX_512bit);
 8730   evpclmulqdq(xmm2, xmm0, xmm10, 0x10, Assembler::AVX_512bit);
 8731   vpternlogq(xmm7, 0x96, xmm1, xmm2, Assembler::AVX_512bit); // xor ABC
 8732 
 8733   evpclmulqdq(xmm5, xmm4, xmm10, 0x01, Assembler::AVX_512bit);
 8734   evpclmulqdq(xmm6, xmm4, xmm10, 0x10, Assembler::AVX_512bit);
 8735   vpternlogq(xmm8, 0x96, xmm5, xmm6, Assembler::AVX_512bit); // xor ABC
 8736 
 8737   evmovdquq(xmm0, xmm7, Assembler::AVX_512bit);
 8738   evmovdquq(xmm4, xmm8, Assembler::AVX_512bit);
 8739 
 8740   addl(len, 128);
 8741   jmp(L_fold_128_B_register);
 8742 
 8743   // at this section of the code, there is 128 * x + y(0 <= y<128) bytes of buffer.The fold_128_B_loop
 8744   // loop will fold 128B at a time until we have 128 + y Bytes of buffer
 8745 
 8746   // fold 128B at a time.This section of the code folds 8 xmm registers in parallel
 8747   bind(L_fold_128_B_loop);
 8748   addl(pos, 128);
 8749   fold512bit_crc32_avx512(xmm0, xmm10, xmm1, buf, pos, 0 * 64);
 8750   fold512bit_crc32_avx512(xmm4, xmm10, xmm1, buf, pos, 1 * 64);
 8751 
 8752   subl(len, 128);
 8753   jcc(Assembler::greaterEqual, L_fold_128_B_loop);
 8754 
 8755   addl(pos, 128);
 8756 
 8757   // at this point, the buffer pointer is pointing at the last y Bytes of the buffer, where 0 <= y < 128
 8758   // the 128B of folded data is in 8 of the xmm registers : xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7
 8759   bind(L_fold_128_B_register);
 8760   evmovdquq(xmm16, Address(table, 5 * 16), Assembler::AVX_512bit); // multiply by rk9-rk16
 8761   evmovdquq(xmm11, Address(table, 9 * 16), Assembler::AVX_512bit); // multiply by rk17-rk20, rk1,rk2, 0,0
 8762   evpclmulqdq(xmm1, xmm0, xmm16, 0x01, Assembler::AVX_512bit);
 8763   evpclmulqdq(xmm2, xmm0, xmm16, 0x10, Assembler::AVX_512bit);
 8764   // save last that has no multiplicand
 8765   vextracti64x2(xmm7, xmm4, 3);
 8766 
 8767   evpclmulqdq(xmm5, xmm4, xmm11, 0x01, Assembler::AVX_512bit);
 8768   evpclmulqdq(xmm6, xmm4, xmm11, 0x10, Assembler::AVX_512bit);
 8769   // Needed later in reduction loop
 8770   movdqu(xmm10, Address(table, 1 * 16));
 8771   vpternlogq(xmm1, 0x96, xmm2, xmm5, Assembler::AVX_512bit); // xor ABC
 8772   vpternlogq(xmm1, 0x96, xmm6, xmm7, Assembler::AVX_512bit); // xor ABC
 8773 
 8774   // Swap 1,0,3,2 - 01 00 11 10
 8775   evshufi64x2(xmm8, xmm1, xmm1, 0x4e, Assembler::AVX_512bit);
 8776   evpxorq(xmm8, xmm8, xmm1, Assembler::AVX_256bit);
 8777   vextracti128(xmm5, xmm8, 1);
 8778   evpxorq(xmm7, xmm5, xmm8, Assembler::AVX_128bit);
 8779 
 8780   // instead of 128, we add 128 - 16 to the loop counter to save 1 instruction from the loop
 8781   // instead of a cmp instruction, we use the negative flag with the jl instruction
 8782   addl(len, 128 - 16);
 8783   jcc(Assembler::less, L_final_reduction_for_128);
 8784 
 8785   bind(L_16B_reduction_loop);
 8786   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8787   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8788   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8789   movdqu(xmm0, Address(buf, pos, Address::times_1, 0 * 16));
 8790   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8791   addl(pos, 16);
 8792   subl(len, 16);
 8793   jcc(Assembler::greaterEqual, L_16B_reduction_loop);
 8794 
 8795   bind(L_final_reduction_for_128);
 8796   addl(len, 16);
 8797   jcc(Assembler::equal, L_128_done);
 8798 
 8799   bind(L_get_last_two_xmms);
 8800   movdqu(xmm2, xmm7);
 8801   addl(pos, len);
 8802   movdqu(xmm1, Address(buf, pos, Address::times_1, -16));
 8803   subl(pos, len);
 8804 
 8805   // get rid of the extra data that was loaded before
 8806   // load the shift constant
 8807   lea(rax, ExternalAddress(StubRoutines::x86::shuf_table_crc32_avx512_addr()));
 8808   movdqu(xmm0, Address(rax, len));
 8809   addl(rax, len);
 8810 
 8811   vpshufb(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8812   //Change mask to 512
 8813   vpxor(xmm0, xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 2 * 16), Assembler::AVX_128bit, tmp2);
 8814   vpshufb(xmm2, xmm2, xmm0, Assembler::AVX_128bit);
 8815 
 8816   blendvpb(xmm2, xmm2, xmm1, xmm0, Assembler::AVX_128bit);
 8817   vpclmulqdq(xmm8, xmm7, xmm10, 0x01);
 8818   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8819   vpxor(xmm7, xmm7, xmm8, Assembler::AVX_128bit);
 8820   vpxor(xmm7, xmm7, xmm2, Assembler::AVX_128bit);
 8821 
 8822   bind(L_128_done);
 8823   // compute crc of a 128-bit value
 8824   movdqu(xmm10, Address(table, 3 * 16));
 8825   movdqu(xmm0, xmm7);
 8826 
 8827   // 64b fold
 8828   vpclmulqdq(xmm7, xmm7, xmm10, 0x0);
 8829   vpsrldq(xmm0, xmm0, 0x8, Assembler::AVX_128bit);
 8830   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8831 
 8832   // 32b fold
 8833   movdqu(xmm0, xmm7);
 8834   vpslldq(xmm7, xmm7, 0x4, Assembler::AVX_128bit);
 8835   vpclmulqdq(xmm7, xmm7, xmm10, 0x10);
 8836   vpxor(xmm7, xmm7, xmm0, Assembler::AVX_128bit);
 8837   jmp(L_barrett);
 8838 
 8839   bind(L_less_than_256);
 8840   kernel_crc32_avx512_256B(crc, buf, len, table, pos, tmp1, tmp2, L_barrett, L_16B_reduction_loop, L_get_last_two_xmms, L_128_done, L_cleanup);
 8841 
 8842   //barrett reduction
 8843   bind(L_barrett);
 8844   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr() + 1 * 16), Assembler::AVX_128bit, tmp2);
 8845   movdqu(xmm1, xmm7);
 8846   movdqu(xmm2, xmm7);
 8847   movdqu(xmm10, Address(table, 4 * 16));
 8848 
 8849   pclmulqdq(xmm7, xmm10, 0x0);
 8850   pxor(xmm7, xmm2);
 8851   vpand(xmm7, xmm7, ExternalAddress(StubRoutines::x86::crc_by128_masks_avx512_addr()), Assembler::AVX_128bit, tmp2);
 8852   movdqu(xmm2, xmm7);
 8853   pclmulqdq(xmm7, xmm10, 0x10);
 8854   pxor(xmm7, xmm2);
 8855   pxor(xmm7, xmm1);
 8856   pextrd(crc, xmm7, 2);
 8857 
 8858   bind(L_cleanup);
 8859   addptr(rsp, 16 * 2 + 8);
 8860   pop(r12);
 8861 }
 8862 
 8863 // S. Gueron / Information Processing Letters 112 (2012) 184
 8864 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
 8865 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
 8866 // Output: the 64-bit carry-less product of B * CONST
 8867 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
 8868                                      Register tmp1, Register tmp2, Register tmp3) {
 8869   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 8870   if (n > 0) {
 8871     addq(tmp3, n * 256 * 8);
 8872   }
 8873   //    Q1 = TABLEExt[n][B & 0xFF];
 8874   movl(tmp1, in);
 8875   andl(tmp1, 0x000000FF);
 8876   shll(tmp1, 3);
 8877   addq(tmp1, tmp3);
 8878   movq(tmp1, Address(tmp1, 0));
 8879 
 8880   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 8881   movl(tmp2, in);
 8882   shrl(tmp2, 8);
 8883   andl(tmp2, 0x000000FF);
 8884   shll(tmp2, 3);
 8885   addq(tmp2, tmp3);
 8886   movq(tmp2, Address(tmp2, 0));
 8887 
 8888   shlq(tmp2, 8);
 8889   xorq(tmp1, tmp2);
 8890 
 8891   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 8892   movl(tmp2, in);
 8893   shrl(tmp2, 16);
 8894   andl(tmp2, 0x000000FF);
 8895   shll(tmp2, 3);
 8896   addq(tmp2, tmp3);
 8897   movq(tmp2, Address(tmp2, 0));
 8898 
 8899   shlq(tmp2, 16);
 8900   xorq(tmp1, tmp2);
 8901 
 8902   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 8903   shrl(in, 24);
 8904   andl(in, 0x000000FF);
 8905   shll(in, 3);
 8906   addq(in, tmp3);
 8907   movq(in, Address(in, 0));
 8908 
 8909   shlq(in, 24);
 8910   xorq(in, tmp1);
 8911   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 8912 }
 8913 
 8914 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 8915                                       Register in_out,
 8916                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 8917                                       XMMRegister w_xtmp2,
 8918                                       Register tmp1,
 8919                                       Register n_tmp2, Register n_tmp3) {
 8920   if (is_pclmulqdq_supported) {
 8921     movdl(w_xtmp1, in_out); // modified blindly
 8922 
 8923     movl(tmp1, const_or_pre_comp_const_index);
 8924     movdl(w_xtmp2, tmp1);
 8925     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 8926 
 8927     movdq(in_out, w_xtmp1);
 8928   } else {
 8929     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
 8930   }
 8931 }
 8932 
 8933 // Recombination Alternative 2: No bit-reflections
 8934 // T1 = (CRC_A * U1) << 1
 8935 // T2 = (CRC_B * U2) << 1
 8936 // C1 = T1 >> 32
 8937 // C2 = T2 >> 32
 8938 // T1 = T1 & 0xFFFFFFFF
 8939 // T2 = T2 & 0xFFFFFFFF
 8940 // T1 = CRC32(0, T1)
 8941 // T2 = CRC32(0, T2)
 8942 // C1 = C1 ^ T1
 8943 // C2 = C2 ^ T2
 8944 // CRC = C1 ^ C2 ^ CRC_C
 8945 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 8946                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8947                                      Register tmp1, Register tmp2,
 8948                                      Register n_tmp3) {
 8949   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8950   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 8951   shlq(in_out, 1);
 8952   movl(tmp1, in_out);
 8953   shrq(in_out, 32);
 8954   xorl(tmp2, tmp2);
 8955   crc32(tmp2, tmp1, 4);
 8956   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
 8957   shlq(in1, 1);
 8958   movl(tmp1, in1);
 8959   shrq(in1, 32);
 8960   xorl(tmp2, tmp2);
 8961   crc32(tmp2, tmp1, 4);
 8962   xorl(in1, tmp2);
 8963   xorl(in_out, in1);
 8964   xorl(in_out, in2);
 8965 }
 8966 
 8967 // Set N to predefined value
 8968 // Subtract from a length of a buffer
 8969 // execute in a loop:
 8970 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
 8971 // for i = 1 to N do
 8972 //  CRC_A = CRC32(CRC_A, A[i])
 8973 //  CRC_B = CRC32(CRC_B, B[i])
 8974 //  CRC_C = CRC32(CRC_C, C[i])
 8975 // end for
 8976 // Recombine
 8977 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 8978                                        Register in_out1, Register in_out2, Register in_out3,
 8979                                        Register tmp1, Register tmp2, Register tmp3,
 8980                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 8981                                        Register tmp4, Register tmp5,
 8982                                        Register n_tmp6) {
 8983   Label L_processPartitions;
 8984   Label L_processPartition;
 8985   Label L_exit;
 8986 
 8987   bind(L_processPartitions);
 8988   cmpl(in_out1, 3 * size);
 8989   jcc(Assembler::less, L_exit);
 8990     xorl(tmp1, tmp1);
 8991     xorl(tmp2, tmp2);
 8992     movq(tmp3, in_out2);
 8993     addq(tmp3, size);
 8994 
 8995     bind(L_processPartition);
 8996       crc32(in_out3, Address(in_out2, 0), 8);
 8997       crc32(tmp1, Address(in_out2, size), 8);
 8998       crc32(tmp2, Address(in_out2, size * 2), 8);
 8999       addq(in_out2, 8);
 9000       cmpq(in_out2, tmp3);
 9001       jcc(Assembler::less, L_processPartition);
 9002     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 9003             w_xtmp1, w_xtmp2, w_xtmp3,
 9004             tmp4, tmp5,
 9005             n_tmp6);
 9006     addq(in_out2, 2 * size);
 9007     subl(in_out1, 3 * size);
 9008     jmp(L_processPartitions);
 9009 
 9010   bind(L_exit);
 9011 }
 9012 #else
 9013 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
 9014                                      Register tmp1, Register tmp2, Register tmp3,
 9015                                      XMMRegister xtmp1, XMMRegister xtmp2) {
 9016   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
 9017   if (n > 0) {
 9018     addl(tmp3, n * 256 * 8);
 9019   }
 9020   //    Q1 = TABLEExt[n][B & 0xFF];
 9021   movl(tmp1, in_out);
 9022   andl(tmp1, 0x000000FF);
 9023   shll(tmp1, 3);
 9024   addl(tmp1, tmp3);
 9025   movq(xtmp1, Address(tmp1, 0));
 9026 
 9027   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
 9028   movl(tmp2, in_out);
 9029   shrl(tmp2, 8);
 9030   andl(tmp2, 0x000000FF);
 9031   shll(tmp2, 3);
 9032   addl(tmp2, tmp3);
 9033   movq(xtmp2, Address(tmp2, 0));
 9034 
 9035   psllq(xtmp2, 8);
 9036   pxor(xtmp1, xtmp2);
 9037 
 9038   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
 9039   movl(tmp2, in_out);
 9040   shrl(tmp2, 16);
 9041   andl(tmp2, 0x000000FF);
 9042   shll(tmp2, 3);
 9043   addl(tmp2, tmp3);
 9044   movq(xtmp2, Address(tmp2, 0));
 9045 
 9046   psllq(xtmp2, 16);
 9047   pxor(xtmp1, xtmp2);
 9048 
 9049   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
 9050   shrl(in_out, 24);
 9051   andl(in_out, 0x000000FF);
 9052   shll(in_out, 3);
 9053   addl(in_out, tmp3);
 9054   movq(xtmp2, Address(in_out, 0));
 9055 
 9056   psllq(xtmp2, 24);
 9057   pxor(xtmp1, xtmp2); // Result in CXMM
 9058   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
 9059 }
 9060 
 9061 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
 9062                                       Register in_out,
 9063                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
 9064                                       XMMRegister w_xtmp2,
 9065                                       Register tmp1,
 9066                                       Register n_tmp2, Register n_tmp3) {
 9067   if (is_pclmulqdq_supported) {
 9068     movdl(w_xtmp1, in_out);
 9069 
 9070     movl(tmp1, const_or_pre_comp_const_index);
 9071     movdl(w_xtmp2, tmp1);
 9072     pclmulqdq(w_xtmp1, w_xtmp2, 0);
 9073     // Keep result in XMM since GPR is 32 bit in length
 9074   } else {
 9075     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
 9076   }
 9077 }
 9078 
 9079 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
 9080                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9081                                      Register tmp1, Register tmp2,
 9082                                      Register n_tmp3) {
 9083   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9084   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
 9085 
 9086   psllq(w_xtmp1, 1);
 9087   movdl(tmp1, w_xtmp1);
 9088   psrlq(w_xtmp1, 32);
 9089   movdl(in_out, w_xtmp1);
 9090 
 9091   xorl(tmp2, tmp2);
 9092   crc32(tmp2, tmp1, 4);
 9093   xorl(in_out, tmp2);
 9094 
 9095   psllq(w_xtmp2, 1);
 9096   movdl(tmp1, w_xtmp2);
 9097   psrlq(w_xtmp2, 32);
 9098   movdl(in1, w_xtmp2);
 9099 
 9100   xorl(tmp2, tmp2);
 9101   crc32(tmp2, tmp1, 4);
 9102   xorl(in1, tmp2);
 9103   xorl(in_out, in1);
 9104   xorl(in_out, in2);
 9105 }
 9106 
 9107 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
 9108                                        Register in_out1, Register in_out2, Register in_out3,
 9109                                        Register tmp1, Register tmp2, Register tmp3,
 9110                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9111                                        Register tmp4, Register tmp5,
 9112                                        Register n_tmp6) {
 9113   Label L_processPartitions;
 9114   Label L_processPartition;
 9115   Label L_exit;
 9116 
 9117   bind(L_processPartitions);
 9118   cmpl(in_out1, 3 * size);
 9119   jcc(Assembler::less, L_exit);
 9120     xorl(tmp1, tmp1);
 9121     xorl(tmp2, tmp2);
 9122     movl(tmp3, in_out2);
 9123     addl(tmp3, size);
 9124 
 9125     bind(L_processPartition);
 9126       crc32(in_out3, Address(in_out2, 0), 4);
 9127       crc32(tmp1, Address(in_out2, size), 4);
 9128       crc32(tmp2, Address(in_out2, size*2), 4);
 9129       crc32(in_out3, Address(in_out2, 0+4), 4);
 9130       crc32(tmp1, Address(in_out2, size+4), 4);
 9131       crc32(tmp2, Address(in_out2, size*2+4), 4);
 9132       addl(in_out2, 8);
 9133       cmpl(in_out2, tmp3);
 9134       jcc(Assembler::less, L_processPartition);
 9135 
 9136         push(tmp3);
 9137         push(in_out1);
 9138         push(in_out2);
 9139         tmp4 = tmp3;
 9140         tmp5 = in_out1;
 9141         n_tmp6 = in_out2;
 9142 
 9143       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
 9144             w_xtmp1, w_xtmp2, w_xtmp3,
 9145             tmp4, tmp5,
 9146             n_tmp6);
 9147 
 9148         pop(in_out2);
 9149         pop(in_out1);
 9150         pop(tmp3);
 9151 
 9152     addl(in_out2, 2 * size);
 9153     subl(in_out1, 3 * size);
 9154     jmp(L_processPartitions);
 9155 
 9156   bind(L_exit);
 9157 }
 9158 #endif //LP64
 9159 
 9160 #ifdef _LP64
 9161 // Algorithm 2: Pipelined usage of the CRC32 instruction.
 9162 // Input: A buffer I of L bytes.
 9163 // Output: the CRC32C value of the buffer.
 9164 // Notations:
 9165 // Write L = 24N + r, with N = floor (L/24).
 9166 // r = L mod 24 (0 <= r < 24).
 9167 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
 9168 // N quadwords, and R consists of r bytes.
 9169 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
 9170 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
 9171 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
 9172 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
 9173 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 9174                                           Register tmp1, Register tmp2, Register tmp3,
 9175                                           Register tmp4, Register tmp5, Register tmp6,
 9176                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9177                                           bool is_pclmulqdq_supported) {
 9178   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 9179   Label L_wordByWord;
 9180   Label L_byteByByteProlog;
 9181   Label L_byteByByte;
 9182   Label L_exit;
 9183 
 9184   if (is_pclmulqdq_supported ) {
 9185     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr();
 9186     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1);
 9187 
 9188     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2);
 9189     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3);
 9190 
 9191     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4);
 9192     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5);
 9193     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
 9194   } else {
 9195     const_or_pre_comp_const_index[0] = 1;
 9196     const_or_pre_comp_const_index[1] = 0;
 9197 
 9198     const_or_pre_comp_const_index[2] = 3;
 9199     const_or_pre_comp_const_index[3] = 2;
 9200 
 9201     const_or_pre_comp_const_index[4] = 5;
 9202     const_or_pre_comp_const_index[5] = 4;
 9203    }
 9204   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 9205                     in2, in1, in_out,
 9206                     tmp1, tmp2, tmp3,
 9207                     w_xtmp1, w_xtmp2, w_xtmp3,
 9208                     tmp4, tmp5,
 9209                     tmp6);
 9210   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 9211                     in2, in1, in_out,
 9212                     tmp1, tmp2, tmp3,
 9213                     w_xtmp1, w_xtmp2, w_xtmp3,
 9214                     tmp4, tmp5,
 9215                     tmp6);
 9216   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 9217                     in2, in1, in_out,
 9218                     tmp1, tmp2, tmp3,
 9219                     w_xtmp1, w_xtmp2, w_xtmp3,
 9220                     tmp4, tmp5,
 9221                     tmp6);
 9222   movl(tmp1, in2);
 9223   andl(tmp1, 0x00000007);
 9224   negl(tmp1);
 9225   addl(tmp1, in2);
 9226   addq(tmp1, in1);
 9227 
 9228   cmpq(in1, tmp1);
 9229   jccb(Assembler::greaterEqual, L_byteByByteProlog);
 9230   align(16);
 9231   BIND(L_wordByWord);
 9232     crc32(in_out, Address(in1, 0), 8);
 9233     addq(in1, 8);
 9234     cmpq(in1, tmp1);
 9235     jcc(Assembler::less, L_wordByWord);
 9236 
 9237   BIND(L_byteByByteProlog);
 9238   andl(in2, 0x00000007);
 9239   movl(tmp2, 1);
 9240 
 9241   cmpl(tmp2, in2);
 9242   jccb(Assembler::greater, L_exit);
 9243   BIND(L_byteByByte);
 9244     crc32(in_out, Address(in1, 0), 1);
 9245     incq(in1);
 9246     incl(tmp2);
 9247     cmpl(tmp2, in2);
 9248     jcc(Assembler::lessEqual, L_byteByByte);
 9249 
 9250   BIND(L_exit);
 9251 }
 9252 #else
 9253 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
 9254                                           Register tmp1, Register  tmp2, Register tmp3,
 9255                                           Register tmp4, Register  tmp5, Register tmp6,
 9256                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
 9257                                           bool is_pclmulqdq_supported) {
 9258   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
 9259   Label L_wordByWord;
 9260   Label L_byteByByteProlog;
 9261   Label L_byteByByte;
 9262   Label L_exit;
 9263 
 9264   if (is_pclmulqdq_supported) {
 9265     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::crc32c_table_addr();
 9266     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 1);
 9267 
 9268     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 2);
 9269     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 3);
 9270 
 9271     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 4);
 9272     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::crc32c_table_addr() + 5);
 9273   } else {
 9274     const_or_pre_comp_const_index[0] = 1;
 9275     const_or_pre_comp_const_index[1] = 0;
 9276 
 9277     const_or_pre_comp_const_index[2] = 3;
 9278     const_or_pre_comp_const_index[3] = 2;
 9279 
 9280     const_or_pre_comp_const_index[4] = 5;
 9281     const_or_pre_comp_const_index[5] = 4;
 9282   }
 9283   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
 9284                     in2, in1, in_out,
 9285                     tmp1, tmp2, tmp3,
 9286                     w_xtmp1, w_xtmp2, w_xtmp3,
 9287                     tmp4, tmp5,
 9288                     tmp6);
 9289   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
 9290                     in2, in1, in_out,
 9291                     tmp1, tmp2, tmp3,
 9292                     w_xtmp1, w_xtmp2, w_xtmp3,
 9293                     tmp4, tmp5,
 9294                     tmp6);
 9295   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
 9296                     in2, in1, in_out,
 9297                     tmp1, tmp2, tmp3,
 9298                     w_xtmp1, w_xtmp2, w_xtmp3,
 9299                     tmp4, tmp5,
 9300                     tmp6);
 9301   movl(tmp1, in2);
 9302   andl(tmp1, 0x00000007);
 9303   negl(tmp1);
 9304   addl(tmp1, in2);
 9305   addl(tmp1, in1);
 9306 
 9307   BIND(L_wordByWord);
 9308   cmpl(in1, tmp1);
 9309   jcc(Assembler::greaterEqual, L_byteByByteProlog);
 9310     crc32(in_out, Address(in1,0), 4);
 9311     addl(in1, 4);
 9312     jmp(L_wordByWord);
 9313 
 9314   BIND(L_byteByByteProlog);
 9315   andl(in2, 0x00000007);
 9316   movl(tmp2, 1);
 9317 
 9318   BIND(L_byteByByte);
 9319   cmpl(tmp2, in2);
 9320   jccb(Assembler::greater, L_exit);
 9321     movb(tmp1, Address(in1, 0));
 9322     crc32(in_out, tmp1, 1);
 9323     incl(in1);
 9324     incl(tmp2);
 9325     jmp(L_byteByByte);
 9326 
 9327   BIND(L_exit);
 9328 }
 9329 #endif // LP64
 9330 #undef BIND
 9331 #undef BLOCK_COMMENT
 9332 
 9333 // Compress char[] array to byte[].
 9334 // Intrinsic for java.lang.StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len)
 9335 // Return the array length if every element in array can be encoded,
 9336 // otherwise, the index of first non-latin1 (> 0xff) character.
 9337 //   @IntrinsicCandidate
 9338 //   public static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
 9339 //     for (int i = 0; i < len; i++) {
 9340 //       char c = src[srcOff];
 9341 //       if (c > 0xff) {
 9342 //           return i;  // return index of non-latin1 char
 9343 //       }
 9344 //       dst[dstOff] = (byte)c;
 9345 //       srcOff++;
 9346 //       dstOff++;
 9347 //     }
 9348 //     return len;
 9349 //   }
 9350 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
 9351   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
 9352   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
 9353   Register tmp5, Register result, KRegister mask1, KRegister mask2) {
 9354   Label copy_chars_loop, done, reset_sp, copy_tail;
 9355 
 9356   // rsi: src
 9357   // rdi: dst
 9358   // rdx: len
 9359   // rcx: tmp5
 9360   // rax: result
 9361 
 9362   // rsi holds start addr of source char[] to be compressed
 9363   // rdi holds start addr of destination byte[]
 9364   // rdx holds length
 9365 
 9366   assert(len != result, "");
 9367 
 9368   // save length for return
 9369   movl(result, len);
 9370 
 9371   if ((AVX3Threshold == 0) && (UseAVX > 2) && // AVX512
 9372     VM_Version::supports_avx512vlbw() &&
 9373     VM_Version::supports_bmi2()) {
 9374 
 9375     Label copy_32_loop, copy_loop_tail, below_threshold, reset_for_copy_tail;
 9376 
 9377     // alignment
 9378     Label post_alignment;
 9379 
 9380     // if length of the string is less than 32, handle it the old fashioned way
 9381     testl(len, -32);
 9382     jcc(Assembler::zero, below_threshold);
 9383 
 9384     // First check whether a character is compressible ( <= 0xFF).
 9385     // Create mask to test for Unicode chars inside zmm vector
 9386     movl(tmp5, 0x00FF);
 9387     evpbroadcastw(tmp2Reg, tmp5, Assembler::AVX_512bit);
 9388 
 9389     testl(len, -64);
 9390     jccb(Assembler::zero, post_alignment);
 9391 
 9392     movl(tmp5, dst);
 9393     andl(tmp5, (32 - 1));
 9394     negl(tmp5);
 9395     andl(tmp5, (32 - 1));
 9396 
 9397     // bail out when there is nothing to be done
 9398     testl(tmp5, 0xFFFFFFFF);
 9399     jccb(Assembler::zero, post_alignment);
 9400 
 9401     // ~(~0 << len), where len is the # of remaining elements to process
 9402     movl(len, 0xFFFFFFFF);
 9403     shlxl(len, len, tmp5);
 9404     notl(len);
 9405     kmovdl(mask2, len);
 9406     movl(len, result);
 9407 
 9408     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9409     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9410     ktestd(mask1, mask2);
 9411     jcc(Assembler::carryClear, copy_tail);
 9412 
 9413     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9414 
 9415     addptr(src, tmp5);
 9416     addptr(src, tmp5);
 9417     addptr(dst, tmp5);
 9418     subl(len, tmp5);
 9419 
 9420     bind(post_alignment);
 9421     // end of alignment
 9422 
 9423     movl(tmp5, len);
 9424     andl(tmp5, (32 - 1));    // tail count (in chars)
 9425     andl(len, ~(32 - 1));    // vector count (in chars)
 9426     jccb(Assembler::zero, copy_loop_tail);
 9427 
 9428     lea(src, Address(src, len, Address::times_2));
 9429     lea(dst, Address(dst, len, Address::times_1));
 9430     negptr(len);
 9431 
 9432     bind(copy_32_loop);
 9433     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
 9434     evpcmpuw(mask1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
 9435     kortestdl(mask1, mask1);
 9436     jccb(Assembler::carryClear, reset_for_copy_tail);
 9437 
 9438     // All elements in current processed chunk are valid candidates for
 9439     // compression. Write a truncated byte elements to the memory.
 9440     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
 9441     addptr(len, 32);
 9442     jccb(Assembler::notZero, copy_32_loop);
 9443 
 9444     bind(copy_loop_tail);
 9445     // bail out when there is nothing to be done
 9446     testl(tmp5, 0xFFFFFFFF);
 9447     jcc(Assembler::zero, done);
 9448 
 9449     movl(len, tmp5);
 9450 
 9451     // ~(~0 << len), where len is the # of remaining elements to process
 9452     movl(tmp5, 0xFFFFFFFF);
 9453     shlxl(tmp5, tmp5, len);
 9454     notl(tmp5);
 9455 
 9456     kmovdl(mask2, tmp5);
 9457 
 9458     evmovdquw(tmp1Reg, mask2, Address(src, 0), /*merge*/ false, Assembler::AVX_512bit);
 9459     evpcmpw(mask1, mask2, tmp1Reg, tmp2Reg, Assembler::le, /*signed*/ false, Assembler::AVX_512bit);
 9460     ktestd(mask1, mask2);
 9461     jcc(Assembler::carryClear, copy_tail);
 9462 
 9463     evpmovwb(Address(dst, 0), mask2, tmp1Reg, Assembler::AVX_512bit);
 9464     jmp(done);
 9465 
 9466     bind(reset_for_copy_tail);
 9467     lea(src, Address(src, tmp5, Address::times_2));
 9468     lea(dst, Address(dst, tmp5, Address::times_1));
 9469     subptr(len, tmp5);
 9470     jmp(copy_chars_loop);
 9471 
 9472     bind(below_threshold);
 9473   }
 9474 
 9475   if (UseSSE42Intrinsics) {
 9476     Label copy_32_loop, copy_16, copy_tail_sse, reset_for_copy_tail;
 9477 
 9478     // vectored compression
 9479     testl(len, 0xfffffff8);
 9480     jcc(Assembler::zero, copy_tail);
 9481 
 9482     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
 9483     movdl(tmp1Reg, tmp5);
 9484     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
 9485 
 9486     andl(len, 0xfffffff0);
 9487     jccb(Assembler::zero, copy_16);
 9488 
 9489     // compress 16 chars per iter
 9490     pxor(tmp4Reg, tmp4Reg);
 9491 
 9492     lea(src, Address(src, len, Address::times_2));
 9493     lea(dst, Address(dst, len, Address::times_1));
 9494     negptr(len);
 9495 
 9496     bind(copy_32_loop);
 9497     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
 9498     por(tmp4Reg, tmp2Reg);
 9499     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
 9500     por(tmp4Reg, tmp3Reg);
 9501     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
 9502     jccb(Assembler::notZero, reset_for_copy_tail);
 9503     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
 9504     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
 9505     addptr(len, 16);
 9506     jccb(Assembler::notZero, copy_32_loop);
 9507 
 9508     // compress next vector of 8 chars (if any)
 9509     bind(copy_16);
 9510     // len = 0
 9511     testl(result, 0x00000008);     // check if there's a block of 8 chars to compress
 9512     jccb(Assembler::zero, copy_tail_sse);
 9513 
 9514     pxor(tmp3Reg, tmp3Reg);
 9515 
 9516     movdqu(tmp2Reg, Address(src, 0));
 9517     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
 9518     jccb(Assembler::notZero, reset_for_copy_tail);
 9519     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
 9520     movq(Address(dst, 0), tmp2Reg);
 9521     addptr(src, 16);
 9522     addptr(dst, 8);
 9523     jmpb(copy_tail_sse);
 9524 
 9525     bind(reset_for_copy_tail);
 9526     movl(tmp5, result);
 9527     andl(tmp5, 0x0000000f);
 9528     lea(src, Address(src, tmp5, Address::times_2));
 9529     lea(dst, Address(dst, tmp5, Address::times_1));
 9530     subptr(len, tmp5);
 9531     jmpb(copy_chars_loop);
 9532 
 9533     bind(copy_tail_sse);
 9534     movl(len, result);
 9535     andl(len, 0x00000007);    // tail count (in chars)
 9536   }
 9537   // compress 1 char per iter
 9538   bind(copy_tail);
 9539   testl(len, len);
 9540   jccb(Assembler::zero, done);
 9541   lea(src, Address(src, len, Address::times_2));
 9542   lea(dst, Address(dst, len, Address::times_1));
 9543   negptr(len);
 9544 
 9545   bind(copy_chars_loop);
 9546   load_unsigned_short(tmp5, Address(src, len, Address::times_2));
 9547   testl(tmp5, 0xff00);      // check if Unicode char
 9548   jccb(Assembler::notZero, reset_sp);
 9549   movb(Address(dst, len, Address::times_1), tmp5);  // ASCII char; compress to 1 byte
 9550   increment(len);
 9551   jccb(Assembler::notZero, copy_chars_loop);
 9552 
 9553   // add len then return (len will be zero if compress succeeded, otherwise negative)
 9554   bind(reset_sp);
 9555   addl(result, len);
 9556 
 9557   bind(done);
 9558 }
 9559 
 9560 // Inflate byte[] array to char[].
 9561 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
 9562 //   @IntrinsicCandidate
 9563 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
 9564 //     for (int i = 0; i < len; i++) {
 9565 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
 9566 //     }
 9567 //   }
 9568 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
 9569   XMMRegister tmp1, Register tmp2, KRegister mask) {
 9570   Label copy_chars_loop, done, below_threshold, avx3_threshold;
 9571   // rsi: src
 9572   // rdi: dst
 9573   // rdx: len
 9574   // rcx: tmp2
 9575 
 9576   // rsi holds start addr of source byte[] to be inflated
 9577   // rdi holds start addr of destination char[]
 9578   // rdx holds length
 9579   assert_different_registers(src, dst, len, tmp2);
 9580   movl(tmp2, len);
 9581   if ((UseAVX > 2) && // AVX512
 9582     VM_Version::supports_avx512vlbw() &&
 9583     VM_Version::supports_bmi2()) {
 9584 
 9585     Label copy_32_loop, copy_tail;
 9586     Register tmp3_aliased = len;
 9587 
 9588     // if length of the string is less than 16, handle it in an old fashioned way
 9589     testl(len, -16);
 9590     jcc(Assembler::zero, below_threshold);
 9591 
 9592     testl(len, -1 * AVX3Threshold);
 9593     jcc(Assembler::zero, avx3_threshold);
 9594 
 9595     // In order to use only one arithmetic operation for the main loop we use
 9596     // this pre-calculation
 9597     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
 9598     andl(len, -32);     // vector count
 9599     jccb(Assembler::zero, copy_tail);
 9600 
 9601     lea(src, Address(src, len, Address::times_1));
 9602     lea(dst, Address(dst, len, Address::times_2));
 9603     negptr(len);
 9604 
 9605 
 9606     // inflate 32 chars per iter
 9607     bind(copy_32_loop);
 9608     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
 9609     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
 9610     addptr(len, 32);
 9611     jcc(Assembler::notZero, copy_32_loop);
 9612 
 9613     bind(copy_tail);
 9614     // bail out when there is nothing to be done
 9615     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
 9616     jcc(Assembler::zero, done);
 9617 
 9618     // ~(~0 << length), where length is the # of remaining elements to process
 9619     movl(tmp3_aliased, -1);
 9620     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
 9621     notl(tmp3_aliased);
 9622     kmovdl(mask, tmp3_aliased);
 9623     evpmovzxbw(tmp1, mask, Address(src, 0), Assembler::AVX_512bit);
 9624     evmovdquw(Address(dst, 0), mask, tmp1, /*merge*/ true, Assembler::AVX_512bit);
 9625 
 9626     jmp(done);
 9627     bind(avx3_threshold);
 9628   }
 9629   if (UseSSE42Intrinsics) {
 9630     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
 9631 
 9632     if (UseAVX > 1) {
 9633       andl(tmp2, (16 - 1));
 9634       andl(len, -16);
 9635       jccb(Assembler::zero, copy_new_tail);
 9636     } else {
 9637       andl(tmp2, 0x00000007);   // tail count (in chars)
 9638       andl(len, 0xfffffff8);    // vector count (in chars)
 9639       jccb(Assembler::zero, copy_tail);
 9640     }
 9641 
 9642     // vectored inflation
 9643     lea(src, Address(src, len, Address::times_1));
 9644     lea(dst, Address(dst, len, Address::times_2));
 9645     negptr(len);
 9646 
 9647     if (UseAVX > 1) {
 9648       bind(copy_16_loop);
 9649       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
 9650       vmovdqu(Address(dst, len, Address::times_2), tmp1);
 9651       addptr(len, 16);
 9652       jcc(Assembler::notZero, copy_16_loop);
 9653 
 9654       bind(below_threshold);
 9655       bind(copy_new_tail);
 9656       movl(len, tmp2);
 9657       andl(tmp2, 0x00000007);
 9658       andl(len, 0xFFFFFFF8);
 9659       jccb(Assembler::zero, copy_tail);
 9660 
 9661       pmovzxbw(tmp1, Address(src, 0));
 9662       movdqu(Address(dst, 0), tmp1);
 9663       addptr(src, 8);
 9664       addptr(dst, 2 * 8);
 9665 
 9666       jmp(copy_tail, true);
 9667     }
 9668 
 9669     // inflate 8 chars per iter
 9670     bind(copy_8_loop);
 9671     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
 9672     movdqu(Address(dst, len, Address::times_2), tmp1);
 9673     addptr(len, 8);
 9674     jcc(Assembler::notZero, copy_8_loop);
 9675 
 9676     bind(copy_tail);
 9677     movl(len, tmp2);
 9678 
 9679     cmpl(len, 4);
 9680     jccb(Assembler::less, copy_bytes);
 9681 
 9682     movdl(tmp1, Address(src, 0));  // load 4 byte chars
 9683     pmovzxbw(tmp1, tmp1);
 9684     movq(Address(dst, 0), tmp1);
 9685     subptr(len, 4);
 9686     addptr(src, 4);
 9687     addptr(dst, 8);
 9688 
 9689     bind(copy_bytes);
 9690   } else {
 9691     bind(below_threshold);
 9692   }
 9693 
 9694   testl(len, len);
 9695   jccb(Assembler::zero, done);
 9696   lea(src, Address(src, len, Address::times_1));
 9697   lea(dst, Address(dst, len, Address::times_2));
 9698   negptr(len);
 9699 
 9700   // inflate 1 char per iter
 9701   bind(copy_chars_loop);
 9702   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
 9703   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
 9704   increment(len);
 9705   jcc(Assembler::notZero, copy_chars_loop);
 9706 
 9707   bind(done);
 9708 }
 9709 
 9710 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, XMMRegister src, bool merge, int vector_len) {
 9711   switch(type) {
 9712     case T_BYTE:
 9713     case T_BOOLEAN:
 9714       evmovdqub(dst, kmask, src, merge, vector_len);
 9715       break;
 9716     case T_CHAR:
 9717     case T_SHORT:
 9718       evmovdquw(dst, kmask, src, merge, vector_len);
 9719       break;
 9720     case T_INT:
 9721     case T_FLOAT:
 9722       evmovdqul(dst, kmask, src, merge, vector_len);
 9723       break;
 9724     case T_LONG:
 9725     case T_DOUBLE:
 9726       evmovdquq(dst, kmask, src, merge, vector_len);
 9727       break;
 9728     default:
 9729       fatal("Unexpected type argument %s", type2name(type));
 9730       break;
 9731   }
 9732 }
 9733 
 9734 
 9735 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, XMMRegister dst, Address src, bool merge, int vector_len) {
 9736   switch(type) {
 9737     case T_BYTE:
 9738     case T_BOOLEAN:
 9739       evmovdqub(dst, kmask, src, merge, vector_len);
 9740       break;
 9741     case T_CHAR:
 9742     case T_SHORT:
 9743       evmovdquw(dst, kmask, src, merge, vector_len);
 9744       break;
 9745     case T_INT:
 9746     case T_FLOAT:
 9747       evmovdqul(dst, kmask, src, merge, vector_len);
 9748       break;
 9749     case T_LONG:
 9750     case T_DOUBLE:
 9751       evmovdquq(dst, kmask, src, merge, vector_len);
 9752       break;
 9753     default:
 9754       fatal("Unexpected type argument %s", type2name(type));
 9755       break;
 9756   }
 9757 }
 9758 
 9759 void MacroAssembler::evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len) {
 9760   switch(type) {
 9761     case T_BYTE:
 9762     case T_BOOLEAN:
 9763       evmovdqub(dst, kmask, src, merge, vector_len);
 9764       break;
 9765     case T_CHAR:
 9766     case T_SHORT:
 9767       evmovdquw(dst, kmask, src, merge, vector_len);
 9768       break;
 9769     case T_INT:
 9770     case T_FLOAT:
 9771       evmovdqul(dst, kmask, src, merge, vector_len);
 9772       break;
 9773     case T_LONG:
 9774     case T_DOUBLE:
 9775       evmovdquq(dst, kmask, src, merge, vector_len);
 9776       break;
 9777     default:
 9778       fatal("Unexpected type argument %s", type2name(type));
 9779       break;
 9780   }
 9781 }
 9782 
 9783 void MacroAssembler::knot(uint masklen, KRegister dst, KRegister src, KRegister ktmp, Register rtmp) {
 9784   switch(masklen) {
 9785     case 2:
 9786        knotbl(dst, src);
 9787        movl(rtmp, 3);
 9788        kmovbl(ktmp, rtmp);
 9789        kandbl(dst, ktmp, dst);
 9790        break;
 9791     case 4:
 9792        knotbl(dst, src);
 9793        movl(rtmp, 15);
 9794        kmovbl(ktmp, rtmp);
 9795        kandbl(dst, ktmp, dst);
 9796        break;
 9797     case 8:
 9798        knotbl(dst, src);
 9799        break;
 9800     case 16:
 9801        knotwl(dst, src);
 9802        break;
 9803     case 32:
 9804        knotdl(dst, src);
 9805        break;
 9806     case 64:
 9807        knotql(dst, src);
 9808        break;
 9809     default:
 9810       fatal("Unexpected vector length %d", masklen);
 9811       break;
 9812   }
 9813 }
 9814 
 9815 void MacroAssembler::kand(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9816   switch(type) {
 9817     case T_BOOLEAN:
 9818     case T_BYTE:
 9819        kandbl(dst, src1, src2);
 9820        break;
 9821     case T_CHAR:
 9822     case T_SHORT:
 9823        kandwl(dst, src1, src2);
 9824        break;
 9825     case T_INT:
 9826     case T_FLOAT:
 9827        kanddl(dst, src1, src2);
 9828        break;
 9829     case T_LONG:
 9830     case T_DOUBLE:
 9831        kandql(dst, src1, src2);
 9832        break;
 9833     default:
 9834       fatal("Unexpected type argument %s", type2name(type));
 9835       break;
 9836   }
 9837 }
 9838 
 9839 void MacroAssembler::kor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9840   switch(type) {
 9841     case T_BOOLEAN:
 9842     case T_BYTE:
 9843        korbl(dst, src1, src2);
 9844        break;
 9845     case T_CHAR:
 9846     case T_SHORT:
 9847        korwl(dst, src1, src2);
 9848        break;
 9849     case T_INT:
 9850     case T_FLOAT:
 9851        kordl(dst, src1, src2);
 9852        break;
 9853     case T_LONG:
 9854     case T_DOUBLE:
 9855        korql(dst, src1, src2);
 9856        break;
 9857     default:
 9858       fatal("Unexpected type argument %s", type2name(type));
 9859       break;
 9860   }
 9861 }
 9862 
 9863 void MacroAssembler::kxor(BasicType type, KRegister dst, KRegister src1, KRegister src2) {
 9864   switch(type) {
 9865     case T_BOOLEAN:
 9866     case T_BYTE:
 9867        kxorbl(dst, src1, src2);
 9868        break;
 9869     case T_CHAR:
 9870     case T_SHORT:
 9871        kxorwl(dst, src1, src2);
 9872        break;
 9873     case T_INT:
 9874     case T_FLOAT:
 9875        kxordl(dst, src1, src2);
 9876        break;
 9877     case T_LONG:
 9878     case T_DOUBLE:
 9879        kxorql(dst, src1, src2);
 9880        break;
 9881     default:
 9882       fatal("Unexpected type argument %s", type2name(type));
 9883       break;
 9884   }
 9885 }
 9886 
 9887 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9888   switch(type) {
 9889     case T_BOOLEAN:
 9890     case T_BYTE:
 9891       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9892     case T_CHAR:
 9893     case T_SHORT:
 9894       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9895     case T_INT:
 9896     case T_FLOAT:
 9897       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9898     case T_LONG:
 9899     case T_DOUBLE:
 9900       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9901     default:
 9902       fatal("Unexpected type argument %s", type2name(type)); break;
 9903   }
 9904 }
 9905 
 9906 void MacroAssembler::evperm(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9907   switch(type) {
 9908     case T_BOOLEAN:
 9909     case T_BYTE:
 9910       evpermb(dst, mask, nds, src, merge, vector_len); break;
 9911     case T_CHAR:
 9912     case T_SHORT:
 9913       evpermw(dst, mask, nds, src, merge, vector_len); break;
 9914     case T_INT:
 9915     case T_FLOAT:
 9916       evpermd(dst, mask, nds, src, merge, vector_len); break;
 9917     case T_LONG:
 9918     case T_DOUBLE:
 9919       evpermq(dst, mask, nds, src, merge, vector_len); break;
 9920     default:
 9921       fatal("Unexpected type argument %s", type2name(type)); break;
 9922   }
 9923 }
 9924 
 9925 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9926   switch(type) {
 9927     case T_BYTE:
 9928       evpminub(dst, mask, nds, src, merge, vector_len); break;
 9929     case T_SHORT:
 9930       evpminuw(dst, mask, nds, src, merge, vector_len); break;
 9931     case T_INT:
 9932       evpminud(dst, mask, nds, src, merge, vector_len); break;
 9933     case T_LONG:
 9934       evpminuq(dst, mask, nds, src, merge, vector_len); break;
 9935     default:
 9936       fatal("Unexpected type argument %s", type2name(type)); break;
 9937   }
 9938 }
 9939 
 9940 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9941   switch(type) {
 9942     case T_BYTE:
 9943       evpmaxub(dst, mask, nds, src, merge, vector_len); break;
 9944     case T_SHORT:
 9945       evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
 9946     case T_INT:
 9947       evpmaxud(dst, mask, nds, src, merge, vector_len); break;
 9948     case T_LONG:
 9949       evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
 9950     default:
 9951       fatal("Unexpected type argument %s", type2name(type)); break;
 9952   }
 9953 }
 9954 
 9955 void MacroAssembler::evpminu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9956   switch(type) {
 9957     case T_BYTE:
 9958       evpminub(dst, mask, nds, src, merge, vector_len); break;
 9959     case T_SHORT:
 9960       evpminuw(dst, mask, nds, src, merge, vector_len); break;
 9961     case T_INT:
 9962       evpminud(dst, mask, nds, src, merge, vector_len); break;
 9963     case T_LONG:
 9964       evpminuq(dst, mask, nds, src, merge, vector_len); break;
 9965     default:
 9966       fatal("Unexpected type argument %s", type2name(type)); break;
 9967   }
 9968 }
 9969 
 9970 void MacroAssembler::evpmaxu(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
 9971   switch(type) {
 9972     case T_BYTE:
 9973       evpmaxub(dst, mask, nds, src, merge, vector_len); break;
 9974     case T_SHORT:
 9975       evpmaxuw(dst, mask, nds, src, merge, vector_len); break;
 9976     case T_INT:
 9977       evpmaxud(dst, mask, nds, src, merge, vector_len); break;
 9978     case T_LONG:
 9979       evpmaxuq(dst, mask, nds, src, merge, vector_len); break;
 9980     default:
 9981       fatal("Unexpected type argument %s", type2name(type)); break;
 9982   }
 9983 }
 9984 
 9985 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
 9986   switch(type) {
 9987     case T_BYTE:
 9988       evpminsb(dst, mask, nds, src, merge, vector_len); break;
 9989     case T_SHORT:
 9990       evpminsw(dst, mask, nds, src, merge, vector_len); break;
 9991     case T_INT:
 9992       evpminsd(dst, mask, nds, src, merge, vector_len); break;
 9993     case T_LONG:
 9994       evpminsq(dst, mask, nds, src, merge, vector_len); break;
 9995     default:
 9996       fatal("Unexpected type argument %s", type2name(type)); break;
 9997   }
 9998 }
 9999 
10000 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10001   switch(type) {
10002     case T_BYTE:
10003       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
10004     case T_SHORT:
10005       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
10006     case T_INT:
10007       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
10008     case T_LONG:
10009       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
10010     default:
10011       fatal("Unexpected type argument %s", type2name(type)); break;
10012   }
10013 }
10014 
10015 void MacroAssembler::evpmins(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10016   switch(type) {
10017     case T_BYTE:
10018       evpminsb(dst, mask, nds, src, merge, vector_len); break;
10019     case T_SHORT:
10020       evpminsw(dst, mask, nds, src, merge, vector_len); break;
10021     case T_INT:
10022       evpminsd(dst, mask, nds, src, merge, vector_len); break;
10023     case T_LONG:
10024       evpminsq(dst, mask, nds, src, merge, vector_len); break;
10025     default:
10026       fatal("Unexpected type argument %s", type2name(type)); break;
10027   }
10028 }
10029 
10030 void MacroAssembler::evpmaxs(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10031   switch(type) {
10032     case T_BYTE:
10033       evpmaxsb(dst, mask, nds, src, merge, vector_len); break;
10034     case T_SHORT:
10035       evpmaxsw(dst, mask, nds, src, merge, vector_len); break;
10036     case T_INT:
10037       evpmaxsd(dst, mask, nds, src, merge, vector_len); break;
10038     case T_LONG:
10039       evpmaxsq(dst, mask, nds, src, merge, vector_len); break;
10040     default:
10041       fatal("Unexpected type argument %s", type2name(type)); break;
10042   }
10043 }
10044 
10045 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10046   switch(type) {
10047     case T_INT:
10048       evpxord(dst, mask, nds, src, merge, vector_len); break;
10049     case T_LONG:
10050       evpxorq(dst, mask, nds, src, merge, vector_len); break;
10051     default:
10052       fatal("Unexpected type argument %s", type2name(type)); break;
10053   }
10054 }
10055 
10056 void MacroAssembler::evxor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10057   switch(type) {
10058     case T_INT:
10059       evpxord(dst, mask, nds, src, merge, vector_len); break;
10060     case T_LONG:
10061       evpxorq(dst, mask, nds, src, merge, vector_len); break;
10062     default:
10063       fatal("Unexpected type argument %s", type2name(type)); break;
10064   }
10065 }
10066 
10067 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10068   switch(type) {
10069     case T_INT:
10070       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
10071     case T_LONG:
10072       evporq(dst, mask, nds, src, merge, vector_len); break;
10073     default:
10074       fatal("Unexpected type argument %s", type2name(type)); break;
10075   }
10076 }
10077 
10078 void MacroAssembler::evor(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10079   switch(type) {
10080     case T_INT:
10081       Assembler::evpord(dst, mask, nds, src, merge, vector_len); break;
10082     case T_LONG:
10083       evporq(dst, mask, nds, src, merge, vector_len); break;
10084     default:
10085       fatal("Unexpected type argument %s", type2name(type)); break;
10086   }
10087 }
10088 
10089 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len) {
10090   switch(type) {
10091     case T_INT:
10092       evpandd(dst, mask, nds, src, merge, vector_len); break;
10093     case T_LONG:
10094       evpandq(dst, mask, nds, src, merge, vector_len); break;
10095     default:
10096       fatal("Unexpected type argument %s", type2name(type)); break;
10097   }
10098 }
10099 
10100 void MacroAssembler::evand(BasicType type, XMMRegister dst, KRegister mask, XMMRegister nds, Address src, bool merge, int vector_len) {
10101   switch(type) {
10102     case T_INT:
10103       evpandd(dst, mask, nds, src, merge, vector_len); break;
10104     case T_LONG:
10105       evpandq(dst, mask, nds, src, merge, vector_len); break;
10106     default:
10107       fatal("Unexpected type argument %s", type2name(type)); break;
10108   }
10109 }
10110 
10111 void MacroAssembler::kortest(uint masklen, KRegister src1, KRegister src2) {
10112   switch(masklen) {
10113     case 8:
10114        kortestbl(src1, src2);
10115        break;
10116     case 16:
10117        kortestwl(src1, src2);
10118        break;
10119     case 32:
10120        kortestdl(src1, src2);
10121        break;
10122     case 64:
10123        kortestql(src1, src2);
10124        break;
10125     default:
10126       fatal("Unexpected mask length %d", masklen);
10127       break;
10128   }
10129 }
10130 
10131 
10132 void MacroAssembler::ktest(uint masklen, KRegister src1, KRegister src2) {
10133   switch(masklen)  {
10134     case 8:
10135        ktestbl(src1, src2);
10136        break;
10137     case 16:
10138        ktestwl(src1, src2);
10139        break;
10140     case 32:
10141        ktestdl(src1, src2);
10142        break;
10143     case 64:
10144        ktestql(src1, src2);
10145        break;
10146     default:
10147       fatal("Unexpected mask length %d", masklen);
10148       break;
10149   }
10150 }
10151 
10152 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
10153   switch(type) {
10154     case T_INT:
10155       evprold(dst, mask, src, shift, merge, vlen_enc); break;
10156     case T_LONG:
10157       evprolq(dst, mask, src, shift, merge, vlen_enc); break;
10158     default:
10159       fatal("Unexpected type argument %s", type2name(type)); break;
10160       break;
10161   }
10162 }
10163 
10164 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src, int shift, bool merge, int vlen_enc) {
10165   switch(type) {
10166     case T_INT:
10167       evprord(dst, mask, src, shift, merge, vlen_enc); break;
10168     case T_LONG:
10169       evprorq(dst, mask, src, shift, merge, vlen_enc); break;
10170     default:
10171       fatal("Unexpected type argument %s", type2name(type)); break;
10172   }
10173 }
10174 
10175 void MacroAssembler::evrold(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
10176   switch(type) {
10177     case T_INT:
10178       evprolvd(dst, mask, src1, src2, merge, vlen_enc); break;
10179     case T_LONG:
10180       evprolvq(dst, mask, src1, src2, merge, vlen_enc); break;
10181     default:
10182       fatal("Unexpected type argument %s", type2name(type)); break;
10183   }
10184 }
10185 
10186 void MacroAssembler::evrord(BasicType type, XMMRegister dst, KRegister mask, XMMRegister src1, XMMRegister src2, bool merge, int vlen_enc) {
10187   switch(type) {
10188     case T_INT:
10189       evprorvd(dst, mask, src1, src2, merge, vlen_enc); break;
10190     case T_LONG:
10191       evprorvq(dst, mask, src1, src2, merge, vlen_enc); break;
10192     default:
10193       fatal("Unexpected type argument %s", type2name(type)); break;
10194   }
10195 }
10196 
10197 void MacroAssembler::evpandq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10198   assert(rscratch != noreg || always_reachable(src), "missing");
10199 
10200   if (reachable(src)) {
10201     evpandq(dst, nds, as_Address(src), vector_len);
10202   } else {
10203     lea(rscratch, src);
10204     evpandq(dst, nds, Address(rscratch, 0), vector_len);
10205   }
10206 }
10207 
10208 void MacroAssembler::evpaddq(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch) {
10209   assert(rscratch != noreg || always_reachable(src), "missing");
10210 
10211   if (reachable(src)) {
10212     Assembler::evpaddq(dst, mask, nds, as_Address(src), merge, vector_len);
10213   } else {
10214     lea(rscratch, src);
10215     Assembler::evpaddq(dst, mask, nds, Address(rscratch, 0), merge, vector_len);
10216   }
10217 }
10218 
10219 void MacroAssembler::evporq(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10220   assert(rscratch != noreg || always_reachable(src), "missing");
10221 
10222   if (reachable(src)) {
10223     evporq(dst, nds, as_Address(src), vector_len);
10224   } else {
10225     lea(rscratch, src);
10226     evporq(dst, nds, Address(rscratch, 0), vector_len);
10227   }
10228 }
10229 
10230 void MacroAssembler::vpshufb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10231   assert(rscratch != noreg || always_reachable(src), "missing");
10232 
10233   if (reachable(src)) {
10234     vpshufb(dst, nds, as_Address(src), vector_len);
10235   } else {
10236     lea(rscratch, src);
10237     vpshufb(dst, nds, Address(rscratch, 0), vector_len);
10238   }
10239 }
10240 
10241 void MacroAssembler::vpor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch) {
10242   assert(rscratch != noreg || always_reachable(src), "missing");
10243 
10244   if (reachable(src)) {
10245     Assembler::vpor(dst, nds, as_Address(src), vector_len);
10246   } else {
10247     lea(rscratch, src);
10248     Assembler::vpor(dst, nds, Address(rscratch, 0), vector_len);
10249   }
10250 }
10251 
10252 void MacroAssembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, AddressLiteral src3, int vector_len, Register rscratch) {
10253   assert(rscratch != noreg || always_reachable(src3), "missing");
10254 
10255   if (reachable(src3)) {
10256     vpternlogq(dst, imm8, src2, as_Address(src3), vector_len);
10257   } else {
10258     lea(rscratch, src3);
10259     vpternlogq(dst, imm8, src2, Address(rscratch, 0), vector_len);
10260   }
10261 }
10262 
10263 #if COMPILER2_OR_JVMCI
10264 
10265 void MacroAssembler::fill_masked(BasicType bt, Address dst, XMMRegister xmm, KRegister mask,
10266                                  Register length, Register temp, int vec_enc) {
10267   // Computing mask for predicated vector store.
10268   movptr(temp, -1);
10269   bzhiq(temp, temp, length);
10270   kmov(mask, temp);
10271   evmovdqu(bt, mask, dst, xmm, true, vec_enc);
10272 }
10273 
10274 // Set memory operation for length "less than" 64 bytes.
10275 void MacroAssembler::fill64_masked(uint shift, Register dst, int disp,
10276                                        XMMRegister xmm, KRegister mask, Register length,
10277                                        Register temp, bool use64byteVector) {
10278   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10279   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
10280   if (!use64byteVector) {
10281     fill32(dst, disp, xmm);
10282     subptr(length, 32 >> shift);
10283     fill32_masked(shift, dst, disp + 32, xmm, mask, length, temp);
10284   } else {
10285     assert(MaxVectorSize == 64, "vector length != 64");
10286     fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_512bit);
10287   }
10288 }
10289 
10290 
10291 void MacroAssembler::fill32_masked(uint shift, Register dst, int disp,
10292                                        XMMRegister xmm, KRegister mask, Register length,
10293                                        Register temp) {
10294   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10295   const BasicType type[] = { T_BYTE, T_SHORT, T_INT, T_LONG};
10296   fill_masked(type[shift], Address(dst, disp), xmm, mask, length, temp, Assembler::AVX_256bit);
10297 }
10298 
10299 
10300 void MacroAssembler::fill32(Address dst, XMMRegister xmm) {
10301   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10302   vmovdqu(dst, xmm);
10303 }
10304 
10305 void MacroAssembler::fill32(Register dst, int disp, XMMRegister xmm) {
10306   fill32(Address(dst, disp), xmm);
10307 }
10308 
10309 void MacroAssembler::fill64(Address dst, XMMRegister xmm, bool use64byteVector) {
10310   assert(MaxVectorSize >= 32, "vector length should be >= 32");
10311   if (!use64byteVector) {
10312     fill32(dst, xmm);
10313     fill32(dst.plus_disp(32), xmm);
10314   } else {
10315     evmovdquq(dst, xmm, Assembler::AVX_512bit);
10316   }
10317 }
10318 
10319 void MacroAssembler::fill64(Register dst, int disp, XMMRegister xmm, bool use64byteVector) {
10320   fill64(Address(dst, disp), xmm, use64byteVector);
10321 }
10322 
10323 #ifdef _LP64
10324 void MacroAssembler::generate_fill_avx3(BasicType type, Register to, Register value,
10325                                         Register count, Register rtmp, XMMRegister xtmp) {
10326   Label L_exit;
10327   Label L_fill_start;
10328   Label L_fill_64_bytes;
10329   Label L_fill_96_bytes;
10330   Label L_fill_128_bytes;
10331   Label L_fill_128_bytes_loop;
10332   Label L_fill_128_loop_header;
10333   Label L_fill_128_bytes_loop_header;
10334   Label L_fill_128_bytes_loop_pre_header;
10335   Label L_fill_zmm_sequence;
10336 
10337   int shift = -1;
10338   int avx3threshold = VM_Version::avx3_threshold();
10339   switch(type) {
10340     case T_BYTE:  shift = 0;
10341       break;
10342     case T_SHORT: shift = 1;
10343       break;
10344     case T_INT:   shift = 2;
10345       break;
10346     /* Uncomment when LONG fill stubs are supported.
10347     case T_LONG:  shift = 3;
10348       break;
10349     */
10350     default:
10351       fatal("Unhandled type: %s\n", type2name(type));
10352   }
10353 
10354   if ((avx3threshold != 0)  || (MaxVectorSize == 32)) {
10355 
10356     if (MaxVectorSize == 64) {
10357       cmpq(count, avx3threshold >> shift);
10358       jcc(Assembler::greater, L_fill_zmm_sequence);
10359     }
10360 
10361     evpbroadcast(type, xtmp, value, Assembler::AVX_256bit);
10362 
10363     bind(L_fill_start);
10364 
10365     cmpq(count, 32 >> shift);
10366     jccb(Assembler::greater, L_fill_64_bytes);
10367     fill32_masked(shift, to, 0, xtmp, k2, count, rtmp);
10368     jmp(L_exit);
10369 
10370     bind(L_fill_64_bytes);
10371     cmpq(count, 64 >> shift);
10372     jccb(Assembler::greater, L_fill_96_bytes);
10373     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp);
10374     jmp(L_exit);
10375 
10376     bind(L_fill_96_bytes);
10377     cmpq(count, 96 >> shift);
10378     jccb(Assembler::greater, L_fill_128_bytes);
10379     fill64(to, 0, xtmp);
10380     subq(count, 64 >> shift);
10381     fill32_masked(shift, to, 64, xtmp, k2, count, rtmp);
10382     jmp(L_exit);
10383 
10384     bind(L_fill_128_bytes);
10385     cmpq(count, 128 >> shift);
10386     jccb(Assembler::greater, L_fill_128_bytes_loop_pre_header);
10387     fill64(to, 0, xtmp);
10388     fill32(to, 64, xtmp);
10389     subq(count, 96 >> shift);
10390     fill32_masked(shift, to, 96, xtmp, k2, count, rtmp);
10391     jmp(L_exit);
10392 
10393     bind(L_fill_128_bytes_loop_pre_header);
10394     {
10395       mov(rtmp, to);
10396       andq(rtmp, 31);
10397       jccb(Assembler::zero, L_fill_128_bytes_loop_header);
10398       negq(rtmp);
10399       addq(rtmp, 32);
10400       mov64(r8, -1L);
10401       bzhiq(r8, r8, rtmp);
10402       kmovql(k2, r8);
10403       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_256bit);
10404       addq(to, rtmp);
10405       shrq(rtmp, shift);
10406       subq(count, rtmp);
10407     }
10408 
10409     cmpq(count, 128 >> shift);
10410     jcc(Assembler::less, L_fill_start);
10411 
10412     bind(L_fill_128_bytes_loop_header);
10413     subq(count, 128 >> shift);
10414 
10415     align32();
10416     bind(L_fill_128_bytes_loop);
10417       fill64(to, 0, xtmp);
10418       fill64(to, 64, xtmp);
10419       addq(to, 128);
10420       subq(count, 128 >> shift);
10421       jccb(Assembler::greaterEqual, L_fill_128_bytes_loop);
10422 
10423     addq(count, 128 >> shift);
10424     jcc(Assembler::zero, L_exit);
10425     jmp(L_fill_start);
10426   }
10427 
10428   if (MaxVectorSize == 64) {
10429     // Sequence using 64 byte ZMM register.
10430     Label L_fill_128_bytes_zmm;
10431     Label L_fill_192_bytes_zmm;
10432     Label L_fill_192_bytes_loop_zmm;
10433     Label L_fill_192_bytes_loop_header_zmm;
10434     Label L_fill_192_bytes_loop_pre_header_zmm;
10435     Label L_fill_start_zmm_sequence;
10436 
10437     bind(L_fill_zmm_sequence);
10438     evpbroadcast(type, xtmp, value, Assembler::AVX_512bit);
10439 
10440     bind(L_fill_start_zmm_sequence);
10441     cmpq(count, 64 >> shift);
10442     jccb(Assembler::greater, L_fill_128_bytes_zmm);
10443     fill64_masked(shift, to, 0, xtmp, k2, count, rtmp, true);
10444     jmp(L_exit);
10445 
10446     bind(L_fill_128_bytes_zmm);
10447     cmpq(count, 128 >> shift);
10448     jccb(Assembler::greater, L_fill_192_bytes_zmm);
10449     fill64(to, 0, xtmp, true);
10450     subq(count, 64 >> shift);
10451     fill64_masked(shift, to, 64, xtmp, k2, count, rtmp, true);
10452     jmp(L_exit);
10453 
10454     bind(L_fill_192_bytes_zmm);
10455     cmpq(count, 192 >> shift);
10456     jccb(Assembler::greater, L_fill_192_bytes_loop_pre_header_zmm);
10457     fill64(to, 0, xtmp, true);
10458     fill64(to, 64, xtmp, true);
10459     subq(count, 128 >> shift);
10460     fill64_masked(shift, to, 128, xtmp, k2, count, rtmp, true);
10461     jmp(L_exit);
10462 
10463     bind(L_fill_192_bytes_loop_pre_header_zmm);
10464     {
10465       movq(rtmp, to);
10466       andq(rtmp, 63);
10467       jccb(Assembler::zero, L_fill_192_bytes_loop_header_zmm);
10468       negq(rtmp);
10469       addq(rtmp, 64);
10470       mov64(r8, -1L);
10471       bzhiq(r8, r8, rtmp);
10472       kmovql(k2, r8);
10473       evmovdqu(T_BYTE, k2, Address(to, 0), xtmp, true, Assembler::AVX_512bit);
10474       addq(to, rtmp);
10475       shrq(rtmp, shift);
10476       subq(count, rtmp);
10477     }
10478 
10479     cmpq(count, 192 >> shift);
10480     jcc(Assembler::less, L_fill_start_zmm_sequence);
10481 
10482     bind(L_fill_192_bytes_loop_header_zmm);
10483     subq(count, 192 >> shift);
10484 
10485     align32();
10486     bind(L_fill_192_bytes_loop_zmm);
10487       fill64(to, 0, xtmp, true);
10488       fill64(to, 64, xtmp, true);
10489       fill64(to, 128, xtmp, true);
10490       addq(to, 192);
10491       subq(count, 192 >> shift);
10492       jccb(Assembler::greaterEqual, L_fill_192_bytes_loop_zmm);
10493 
10494     addq(count, 192 >> shift);
10495     jcc(Assembler::zero, L_exit);
10496     jmp(L_fill_start_zmm_sequence);
10497   }
10498   bind(L_exit);
10499 }
10500 #endif
10501 #endif //COMPILER2_OR_JVMCI
10502 
10503 
10504 #ifdef _LP64
10505 void MacroAssembler::convert_f2i(Register dst, XMMRegister src) {
10506   Label done;
10507   cvttss2sil(dst, src);
10508   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10509   cmpl(dst, 0x80000000); // float_sign_flip
10510   jccb(Assembler::notEqual, done);
10511   subptr(rsp, 8);
10512   movflt(Address(rsp, 0), src);
10513   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
10514   pop(dst);
10515   bind(done);
10516 }
10517 
10518 void MacroAssembler::convert_d2i(Register dst, XMMRegister src) {
10519   Label done;
10520   cvttsd2sil(dst, src);
10521   // Conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
10522   cmpl(dst, 0x80000000); // float_sign_flip
10523   jccb(Assembler::notEqual, done);
10524   subptr(rsp, 8);
10525   movdbl(Address(rsp, 0), src);
10526   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
10527   pop(dst);
10528   bind(done);
10529 }
10530 
10531 void MacroAssembler::convert_f2l(Register dst, XMMRegister src) {
10532   Label done;
10533   cvttss2siq(dst, src);
10534   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10535   jccb(Assembler::notEqual, done);
10536   subptr(rsp, 8);
10537   movflt(Address(rsp, 0), src);
10538   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
10539   pop(dst);
10540   bind(done);
10541 }
10542 
10543 void MacroAssembler::round_float(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10544   // Following code is line by line assembly translation rounding algorithm.
10545   // Please refer to java.lang.Math.round(float) algorithm for details.
10546   const int32_t FloatConsts_EXP_BIT_MASK = 0x7F800000;
10547   const int32_t FloatConsts_SIGNIFICAND_WIDTH = 24;
10548   const int32_t FloatConsts_EXP_BIAS = 127;
10549   const int32_t FloatConsts_SIGNIF_BIT_MASK = 0x007FFFFF;
10550   const int32_t MINUS_32 = 0xFFFFFFE0;
10551   Label L_special_case, L_block1, L_exit;
10552   movl(rtmp, FloatConsts_EXP_BIT_MASK);
10553   movdl(dst, src);
10554   andl(dst, rtmp);
10555   sarl(dst, FloatConsts_SIGNIFICAND_WIDTH - 1);
10556   movl(rtmp, FloatConsts_SIGNIFICAND_WIDTH - 2 + FloatConsts_EXP_BIAS);
10557   subl(rtmp, dst);
10558   movl(rcx, rtmp);
10559   movl(dst, MINUS_32);
10560   testl(rtmp, dst);
10561   jccb(Assembler::notEqual, L_special_case);
10562   movdl(dst, src);
10563   andl(dst, FloatConsts_SIGNIF_BIT_MASK);
10564   orl(dst, FloatConsts_SIGNIF_BIT_MASK + 1);
10565   movdl(rtmp, src);
10566   testl(rtmp, rtmp);
10567   jccb(Assembler::greaterEqual, L_block1);
10568   negl(dst);
10569   bind(L_block1);
10570   sarl(dst);
10571   addl(dst, 0x1);
10572   sarl(dst, 0x1);
10573   jmp(L_exit);
10574   bind(L_special_case);
10575   convert_f2i(dst, src);
10576   bind(L_exit);
10577 }
10578 
10579 void MacroAssembler::round_double(Register dst, XMMRegister src, Register rtmp, Register rcx) {
10580   // Following code is line by line assembly translation rounding algorithm.
10581   // Please refer to java.lang.Math.round(double) algorithm for details.
10582   const int64_t DoubleConsts_EXP_BIT_MASK = 0x7FF0000000000000L;
10583   const int64_t DoubleConsts_SIGNIFICAND_WIDTH = 53;
10584   const int64_t DoubleConsts_EXP_BIAS = 1023;
10585   const int64_t DoubleConsts_SIGNIF_BIT_MASK = 0x000FFFFFFFFFFFFFL;
10586   const int64_t MINUS_64 = 0xFFFFFFFFFFFFFFC0L;
10587   Label L_special_case, L_block1, L_exit;
10588   mov64(rtmp, DoubleConsts_EXP_BIT_MASK);
10589   movq(dst, src);
10590   andq(dst, rtmp);
10591   sarq(dst, DoubleConsts_SIGNIFICAND_WIDTH - 1);
10592   mov64(rtmp, DoubleConsts_SIGNIFICAND_WIDTH - 2 + DoubleConsts_EXP_BIAS);
10593   subq(rtmp, dst);
10594   movq(rcx, rtmp);
10595   mov64(dst, MINUS_64);
10596   testq(rtmp, dst);
10597   jccb(Assembler::notEqual, L_special_case);
10598   movq(dst, src);
10599   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK);
10600   andq(dst, rtmp);
10601   mov64(rtmp, DoubleConsts_SIGNIF_BIT_MASK + 1);
10602   orq(dst, rtmp);
10603   movq(rtmp, src);
10604   testq(rtmp, rtmp);
10605   jccb(Assembler::greaterEqual, L_block1);
10606   negq(dst);
10607   bind(L_block1);
10608   sarq(dst);
10609   addq(dst, 0x1);
10610   sarq(dst, 0x1);
10611   jmp(L_exit);
10612   bind(L_special_case);
10613   convert_d2l(dst, src);
10614   bind(L_exit);
10615 }
10616 
10617 void MacroAssembler::convert_d2l(Register dst, XMMRegister src) {
10618   Label done;
10619   cvttsd2siq(dst, src);
10620   cmp64(dst, ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
10621   jccb(Assembler::notEqual, done);
10622   subptr(rsp, 8);
10623   movdbl(Address(rsp, 0), src);
10624   call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
10625   pop(dst);
10626   bind(done);
10627 }
10628 
10629 void MacroAssembler::cache_wb(Address line)
10630 {
10631   // 64 bit cpus always support clflush
10632   assert(VM_Version::supports_clflush(), "clflush should be available");
10633   bool optimized = VM_Version::supports_clflushopt();
10634   bool no_evict = VM_Version::supports_clwb();
10635 
10636   // prefer clwb (writeback without evict) otherwise
10637   // prefer clflushopt (potentially parallel writeback with evict)
10638   // otherwise fallback on clflush (serial writeback with evict)
10639 
10640   if (optimized) {
10641     if (no_evict) {
10642       clwb(line);
10643     } else {
10644       clflushopt(line);
10645     }
10646   } else {
10647     // no need for fence when using CLFLUSH
10648     clflush(line);
10649   }
10650 }
10651 
10652 void MacroAssembler::cache_wbsync(bool is_pre)
10653 {
10654   assert(VM_Version::supports_clflush(), "clflush should be available");
10655   bool optimized = VM_Version::supports_clflushopt();
10656   bool no_evict = VM_Version::supports_clwb();
10657 
10658   // pick the correct implementation
10659 
10660   if (!is_pre && (optimized || no_evict)) {
10661     // need an sfence for post flush when using clflushopt or clwb
10662     // otherwise no no need for any synchroniaztion
10663 
10664     sfence();
10665   }
10666 }
10667 
10668 #endif // _LP64
10669 
10670 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
10671   switch (cond) {
10672     // Note some conditions are synonyms for others
10673     case Assembler::zero:         return Assembler::notZero;
10674     case Assembler::notZero:      return Assembler::zero;
10675     case Assembler::less:         return Assembler::greaterEqual;
10676     case Assembler::lessEqual:    return Assembler::greater;
10677     case Assembler::greater:      return Assembler::lessEqual;
10678     case Assembler::greaterEqual: return Assembler::less;
10679     case Assembler::below:        return Assembler::aboveEqual;
10680     case Assembler::belowEqual:   return Assembler::above;
10681     case Assembler::above:        return Assembler::belowEqual;
10682     case Assembler::aboveEqual:   return Assembler::below;
10683     case Assembler::overflow:     return Assembler::noOverflow;
10684     case Assembler::noOverflow:   return Assembler::overflow;
10685     case Assembler::negative:     return Assembler::positive;
10686     case Assembler::positive:     return Assembler::negative;
10687     case Assembler::parity:       return Assembler::noParity;
10688     case Assembler::noParity:     return Assembler::parity;
10689   }
10690   ShouldNotReachHere(); return Assembler::overflow;
10691 }
10692 
10693 // This is simply a call to Thread::current()
10694 void MacroAssembler::get_thread_slow(Register thread) {
10695   if (thread != rax) {
10696     push(rax);
10697   }
10698   push(rdi);
10699   push(rsi);
10700   push(rdx);
10701   push(rcx);
10702   push(r8);
10703   push(r9);
10704   push(r10);
10705   push(r11);
10706 
10707   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
10708 
10709   pop(r11);
10710   pop(r10);
10711   pop(r9);
10712   pop(r8);
10713   pop(rcx);
10714   pop(rdx);
10715   pop(rsi);
10716   pop(rdi);
10717   if (thread != rax) {
10718     mov(thread, rax);
10719     pop(rax);
10720   }
10721 }
10722 
10723 void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigned bias, Register tmp) {
10724   Label L_stack_ok;
10725   if (bias == 0) {
10726     testptr(sp, 2 * wordSize - 1);
10727   } else {
10728     // lea(tmp, Address(rsp, bias);
10729     mov(tmp, sp);
10730     addptr(tmp, bias);
10731     testptr(tmp, 2 * wordSize - 1);
10732   }
10733   jcc(Assembler::equal, L_stack_ok);
10734   block_comment(msg);
10735   stop(msg);
10736   bind(L_stack_ok);
10737 }
10738 
10739 // Implements lightweight-locking.
10740 //
10741 // obj: the object to be locked
10742 // reg_rax: rax
10743 // thread: the thread which attempts to lock obj
10744 // tmp: a temporary register
10745 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register reg_rax, Register tmp, Label& slow) {
10746   Register thread = r15_thread;
10747 
10748   assert(reg_rax == rax, "");
10749   assert_different_registers(basic_lock, obj, reg_rax, thread, tmp);
10750 
10751   Label push;
10752   const Register top = tmp;
10753 
10754   // Preload the markWord. It is important that this is the first
10755   // instruction emitted as it is part of C1's null check semantics.
10756   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
10757 
10758   if (UseObjectMonitorTable) {
10759     // Clear cache in case fast locking succeeds.
10760     movptr(Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes()))), 0);
10761   }
10762 
10763   // Load top.
10764   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10765 
10766   // Check if the lock-stack is full.
10767   cmpl(top, LockStack::end_offset());
10768   jcc(Assembler::greaterEqual, slow);
10769 
10770   // Check for recursion.
10771   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
10772   jcc(Assembler::equal, push);
10773 
10774   // Check header for monitor (0b10).
10775   testptr(reg_rax, markWord::monitor_value);
10776   jcc(Assembler::notZero, slow);
10777 
10778   // Try to lock. Transition lock bits 0b01 => 0b00
10779   movptr(tmp, reg_rax);
10780   andptr(tmp, ~(int32_t)markWord::unlocked_value);
10781   orptr(reg_rax, markWord::unlocked_value);
10782   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10783   jcc(Assembler::notEqual, slow);
10784 
10785   // Restore top, CAS clobbers register.
10786   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10787 
10788   bind(push);
10789   // After successful lock, push object on lock-stack.
10790   movptr(Address(thread, top), obj);
10791   incrementl(top, oopSize);
10792   movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
10793 }
10794 
10795 // Implements lightweight-unlocking.
10796 //
10797 // obj: the object to be unlocked
10798 // reg_rax: rax
10799 // thread: the thread
10800 // tmp: a temporary register
10801 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register tmp, Label& slow) {
10802   Register thread = r15_thread;
10803 
10804   assert(reg_rax == rax, "");
10805   assert_different_registers(obj, reg_rax, thread, tmp);
10806 
10807   Label unlocked, push_and_slow;
10808   const Register top = tmp;
10809 
10810   // Check if obj is top of lock-stack.
10811   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10812   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
10813   jcc(Assembler::notEqual, slow);
10814 
10815   // Pop lock-stack.
10816   DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
10817   subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10818 
10819   // Check if recursive.
10820   cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
10821   jcc(Assembler::equal, unlocked);
10822 
10823   // Not recursive. Check header for monitor (0b10).
10824   movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
10825   testptr(reg_rax, markWord::monitor_value);
10826   jcc(Assembler::notZero, push_and_slow);
10827 
10828 #ifdef ASSERT
10829   // Check header not unlocked (0b01).
10830   Label not_unlocked;
10831   testptr(reg_rax, markWord::unlocked_value);
10832   jcc(Assembler::zero, not_unlocked);
10833   stop("lightweight_unlock already unlocked");
10834   bind(not_unlocked);
10835 #endif
10836 
10837   // Try to unlock. Transition lock bits 0b00 => 0b01
10838   movptr(tmp, reg_rax);
10839   orptr(tmp, markWord::unlocked_value);
10840   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10841   jcc(Assembler::equal, unlocked);
10842 
10843   bind(push_and_slow);
10844   // Restore lock-stack and handle the unlock in runtime.
10845 #ifdef ASSERT
10846   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10847   movptr(Address(thread, top), obj);
10848 #endif
10849   addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
10850   jmp(slow);
10851 
10852   bind(unlocked);
10853 }
10854 
10855 #ifdef _LP64
10856 // Saves legacy GPRs state on stack.
10857 void MacroAssembler::save_legacy_gprs() {
10858   subq(rsp, 16 * wordSize);
10859   movq(Address(rsp, 15 * wordSize), rax);
10860   movq(Address(rsp, 14 * wordSize), rcx);
10861   movq(Address(rsp, 13 * wordSize), rdx);
10862   movq(Address(rsp, 12 * wordSize), rbx);
10863   movq(Address(rsp, 10 * wordSize), rbp);
10864   movq(Address(rsp, 9 * wordSize), rsi);
10865   movq(Address(rsp, 8 * wordSize), rdi);
10866   movq(Address(rsp, 7 * wordSize), r8);
10867   movq(Address(rsp, 6 * wordSize), r9);
10868   movq(Address(rsp, 5 * wordSize), r10);
10869   movq(Address(rsp, 4 * wordSize), r11);
10870   movq(Address(rsp, 3 * wordSize), r12);
10871   movq(Address(rsp, 2 * wordSize), r13);
10872   movq(Address(rsp, wordSize), r14);
10873   movq(Address(rsp, 0), r15);
10874 }
10875 
10876 // Resotres back legacy GPRs state from stack.
10877 void MacroAssembler::restore_legacy_gprs() {
10878   movq(r15, Address(rsp, 0));
10879   movq(r14, Address(rsp, wordSize));
10880   movq(r13, Address(rsp, 2 * wordSize));
10881   movq(r12, Address(rsp, 3 * wordSize));
10882   movq(r11, Address(rsp, 4 * wordSize));
10883   movq(r10, Address(rsp, 5 * wordSize));
10884   movq(r9,  Address(rsp, 6 * wordSize));
10885   movq(r8,  Address(rsp, 7 * wordSize));
10886   movq(rdi, Address(rsp, 8 * wordSize));
10887   movq(rsi, Address(rsp, 9 * wordSize));
10888   movq(rbp, Address(rsp, 10 * wordSize));
10889   movq(rbx, Address(rsp, 12 * wordSize));
10890   movq(rdx, Address(rsp, 13 * wordSize));
10891   movq(rcx, Address(rsp, 14 * wordSize));
10892   movq(rax, Address(rsp, 15 * wordSize));
10893   addq(rsp, 16 * wordSize);
10894 }
10895 
10896 void MacroAssembler::load_aotrc_address(Register reg, address a) {
10897 #if INCLUDE_CDS
10898   assert(AOTRuntimeConstants::contains(a), "address out of range for data area");
10899   if (SCCache::is_on_for_write()) {
10900     // all aotrc field addresses should be registered in the SCC address table
10901     lea(reg, ExternalAddress(a));
10902   } else {
10903     mov64(reg, (uint64_t)a);
10904   }
10905 #else
10906   ShouldNotReachHere();
10907 #endif
10908 }
10909 
10910 void MacroAssembler::setcc(Assembler::Condition comparison, Register dst) {
10911   if (VM_Version::supports_apx_f()) {
10912     esetzucc(comparison, dst);
10913   } else {
10914     setb(comparison, dst);
10915     movzbl(dst, dst);
10916   }
10917 }
10918 
10919 #endif